Fill in descriptors on the render thread in the PPSSPP UI.

Common/Data/Collections/FastVec.h

@@ -117,6 +117,12 @@ public:
 		IncreaseCapacityTo(newCapacity);
 	}
 
+	void extend(const T *newData, size_t count) {
+		IncreaseCapacityTo(size_ + count);
+		memcpy(data_ + size_, newData, count * sizeof(T));
+		size_ += count;
+	}
+
 	void LockCapacity() {
 #ifdef _DEBUG
 		capacityLocked_ = true;

Common/GPU/Vulkan/VulkanDescSet.cpp

@@ -75,7 +75,9 @@ VkDescriptorSet VulkanDescSetPool::Allocate(int n, const VkDescriptorSetLayout *
 
 	usage_++;
 
-	vulkan_->SetDebugName(desc, VK_OBJECT_TYPE_DESCRIPTOR_SET, tag);
+	if (tag) {
+		vulkan_->SetDebugName(desc, VK_OBJECT_TYPE_DESCRIPTOR_SET, tag);
+	}
 	return desc;
 }
 

Common/GPU/Vulkan/VulkanDescSet.h

@@ -18,7 +18,7 @@ enum class BindingType {
 // Only appropriate for use in a per-frame pool.
 class VulkanDescSetPool {
 public:
-	VulkanDescSetPool(const char *tag, bool grow) : tag_(tag), grow_(grow) {}
+	VulkanDescSetPool(const char *tag = "", bool grow = true) : tag_(tag), grow_(grow) {}
 	~VulkanDescSetPool();
 
 	// Must call this before use: defines how to clear cache of ANY returned values from Allocate().
@@ -32,6 +32,13 @@ public:
 	void Reset();
 	void Destroy();
 
+	void SetTag(const char *tag) {
+		tag_ = tag;
+	}
+	bool IsDestroyed() const {
+		return !descPool_;
+	}
+
 private:
 	VkResult Recreate(bool grow);
 

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -338,7 +338,7 @@ void VulkanQueueRunner::PreprocessSteps(std::vector<VKRStep *> &steps) {
 	}
 }
 
-void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) {
+void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) {
 	QueueProfileContext *profile = frameData.profile.enabled ? &frameData.profile : nullptr;
 
 	if (profile)
@@ -394,7 +394,7 @@ void VulkanQueueRunner::RunSteps(std::vector<VKRStep *> &steps, FrameData &frame
 					vkCmdBeginDebugUtilsLabelEXT(cmd, &labelInfo);
 				}
 			}
-			PerformRenderPass(step, cmd);
+			PerformRenderPass(step, cmd, curFrame);
 			break;
 		case VKRStepType::COPY:
 			PerformCopy(step, cmd);
@@ -1103,7 +1103,7 @@ void TransitionFromOptimal(VkCommandBuffer cmd, VkImage colorImage, VkImageLayou
 	}
 }
 
-void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd) {
+void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd, int curFrame) {
 	for (size_t i = 0; i < step.preTransitions.size(); i++) {
 		const TransitionRequest &iter = step.preTransitions[i];
 		if (iter.aspect == VK_IMAGE_ASPECT_COLOR_BIT && iter.fb->color.layout != iter.targetLayout) {
@@ -1199,6 +1199,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 	// The stencil ones are very commonly mostly redundant so let's eliminate them where possible.
 	// Might also want to consider scissor and viewport.
 	VkPipeline lastPipeline = VK_NULL_HANDLE;
+	VKRPipelineLayout *vkrPipelineLayout = nullptr;
 	VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
 
 	bool pipelineOK = false;
@@ -1241,7 +1242,8 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 				if (pipeline != VK_NULL_HANDLE) {
 					vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
-					pipelineLayout = c.pipeline.pipelineLayout->pipelineLayout;
+					vkrPipelineLayout = c.pipeline.pipelineLayout;
+					pipelineLayout = vkrPipelineLayout->pipelineLayout;
 					lastGraphicsPipeline = graphicsPipeline;
 					pipelineOK = true;
 				} else {
@@ -1336,7 +1338,12 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::DRAW_INDEXED:
 			if (pipelineOK) {
-				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.drawIndexed.ds, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets);
+				VkDescriptorSet set;
+				if (c.drawIndexed.ds)
+					set = c.drawIndexed.ds;
+				else
+					set = vkrPipelineLayout->descSets_[curFrame][c.drawIndexed.descSetIndex].set;
+				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets);
 				vkCmdBindIndexBuffer(cmd, c.drawIndexed.ibuffer, c.drawIndexed.ioffset, VK_INDEX_TYPE_UINT16);
 				VkDeviceSize voffset = c.drawIndexed.voffset;
 				vkCmdBindVertexBuffers(cmd, 0, 1, &c.drawIndexed.vbuffer, &voffset);
@@ -1346,7 +1353,13 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::DRAW:
 			if (pipelineOK) {
-				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.draw.ds, c.draw.numUboOffsets, c.draw.uboOffsets);
+				VkDescriptorSet set;
+				if (c.drawIndexed.ds)
+					set = c.drawIndexed.ds;
+				else
+					set = vkrPipelineLayout->descSets_[curFrame][c.drawIndexed.descSetIndex].set;
+				_dbg_assert_(set != VK_NULL_HANDLE);
+				vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.draw.numUboOffsets, c.draw.uboOffsets);
 				if (c.draw.vbuffer) {
 					vkCmdBindVertexBuffers(cmd, 0, 1, &c.draw.vbuffer, &c.draw.voffset);
 				}

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -71,6 +71,7 @@ struct VkRenderData {
 		} compute_pipeline;
 		struct {
 			VkDescriptorSet ds;
+			uint32_t descSetIndex;
 			int numUboOffsets;
 			uint32_t uboOffsets[3];
 			VkBuffer vbuffer;
@@ -80,6 +81,7 @@ struct VkRenderData {
 		} draw;
 		struct {
 			VkDescriptorSet ds;
+			uint32_t descSetIndex;
 			uint32_t uboOffsets[3];
 			uint16_t numUboOffsets;
 			uint16_t instances;
@@ -119,9 +121,7 @@ struct VkRenderData {
 			const char *annotation;
 		} debugAnnotation;
 		struct {
-			int setNumber;
-			VkDescriptorSet set;
-			VkPipelineLayout pipelineLayout;
+			int setIndex;
 		} bindDescSet;
 	};
 };
@@ -231,7 +231,7 @@ public:
 	}
 
 	void PreprocessSteps(std::vector<VKRStep *> &steps);
-	void RunSteps(std::vector<VKRStep *> &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false);
+	void RunSteps(std::vector<VKRStep *> &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false);
 	void LogSteps(const std::vector<VKRStep *> &steps, bool verbose);
 
 	static std::string StepToString(VulkanContext *vulkan, const VKRStep &step);
@@ -291,7 +291,7 @@ private:
 	bool InitDepthStencilBuffer(VkCommandBuffer cmd);  // Used for non-buffered rendering.
 
 	VKRRenderPass *PerformBindFramebufferAsRenderTarget(const VKRStep &pass, VkCommandBuffer cmd);
-	void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd);
+	void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd, int curFrame);
 	void PerformCopy(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformBlit(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformReadback(const VKRStep &pass, VkCommandBuffer cmd, FrameData &frameData);

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -265,6 +265,7 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan, bool useThread,
 	initTimeMs_("initTimeMs"),
 	totalGPUTimeMs_("totalGPUTimeMs"),
 	renderCPUTimeMs_("renderCPUTimeMs"),
+	descUpdateTimeMs_("descUpdateCPUTimeMs"),
 	useRenderThread_(useThread),
 	frameTimeHistory_(frameTimeHistory)
 {
@@ -645,6 +646,8 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile
 
 	PollPresentTiming();
 
+	ResetDescriptorLists(curFrame);
+
 	int validBits = vulkan_->GetQueueFamilyProperties(vulkan_->GetGraphicsQueueFamilyIndex()).timestampValidBits;
 
 	FrameTimeData &frameTimeData = frameTimeHistory_.Add(frameId);
@@ -679,6 +682,9 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile
 				renderCPUTimeMs_.Update((frameData.profile.cpuEndTime - frameData.profile.cpuStartTime) * 1000.0);
 				renderCPUTimeMs_.Format(line, sizeof(line));
 				str << line;
+				descUpdateTimeMs_.Update(frameData.profile.descWriteTime * 1000.0);
+				descUpdateTimeMs_.Format(line, sizeof(line));
+				str << line;
 				for (int i = 0; i < numQueries - 1; i++) {
 					uint64_t diff = (queryResults[i + 1] - queryResults[i]) & timestampDiffMask;
 					double milliseconds = (double)diff * timestampConversionFactor;
@@ -1462,6 +1468,11 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
 	}
 	frameData.SubmitPending(vulkan_, FrameSubmitType::Pending, frameDataShared_);
 
+	// Flush descriptors.
+	double descStart = time_now_d();
+	FlushDescriptors(task.frame);
+	frameData.profile.descWriteTime = time_now_d() - descStart;
+
 	if (!frameData.hasMainCommands) {
 		// Effectively resets both main and present command buffers, since they both live in this pool.
 		// We always record main commands first, so we don't need to reset the present command buffer separately.
@@ -1483,11 +1494,11 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) {
 		int passes = GetVRPassesCount();
 		for (int i = 0; i < passes; i++) {
 			PreVRFrameRender(i);
-			queueRunner_.RunSteps(task.steps, frameData, frameDataShared_, i < passes - 1);
+			queueRunner_.RunSteps(task.steps, task.frame, frameData, frameDataShared_, i < passes - 1);
 			PostVRFrameRender();
 		}
 	} else {
-		queueRunner_.RunSteps(task.steps, frameData, frameDataShared_);
+		queueRunner_.RunSteps(task.steps, task.frame, frameData, frameDataShared_);
 	}
 
 	switch (task.runType) {
@@ -1562,16 +1573,16 @@ void VulkanRenderManager::ResetStats() {
 	renderCPUTimeMs_.Reset();
 }
 
-VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindingTypes, size_t bindingCount, bool geoShadersEnabled, const char *tag) {
+VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindingTypes, size_t bindingTypesCount, bool geoShadersEnabled, const char *tag) {
 	VKRPipelineLayout *layout = new VKRPipelineLayout();
 	layout->tag = tag;
-	layout->bindingCount = (uint32_t)bindingCount;
+	layout->bindingTypesCount = (uint32_t)bindingTypesCount;
 
-	_dbg_assert_(bindingCount <= ARRAY_SIZE(layout->bindingTypes));
-	memcpy(layout->bindingTypes, bindingTypes, sizeof(BindingType) * bindingCount);
+	_dbg_assert_(bindingTypesCount <= ARRAY_SIZE(layout->bindingTypes));
+	memcpy(layout->bindingTypes, bindingTypes, sizeof(BindingType) * bindingTypesCount);
 
 	VkDescriptorSetLayoutBinding bindings[VKRPipelineLayout::MAX_DESC_SET_BINDINGS];
-	for (int i = 0; i < bindingCount; i++) {
+	for (int i = 0; i < bindingTypesCount; i++) {
 		bindings[i].binding = i;
 		bindings[i].descriptorCount = 1;
 		bindings[i].pImmutableSamplers = nullptr;
@@ -1611,7 +1622,7 @@ VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindin
 	}
 
 	VkDescriptorSetLayoutCreateInfo dsl = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO };
-	dsl.bindingCount = (uint32_t)bindingCount;
+	dsl.bindingCount = (uint32_t)bindingTypesCount;
 	dsl.pBindings = bindings;
 	VkResult res = vkCreateDescriptorSetLayout(vulkan_->GetDevice(), &dsl, nullptr, &layout->descriptorSetLayout);
 	_assert_(VK_SUCCESS == res && layout->descriptorSetLayout);
@@ -1625,10 +1636,135 @@ VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindin
 
 	vulkan_->SetDebugName(layout->descriptorSetLayout, VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT, tag);
 	vulkan_->SetDebugName(layout->pipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT, tag);
+
+	for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) {
+		layout->descPools[i].Create(vulkan_, bindingTypes, (uint32_t)bindingTypesCount, 512);
+		layout->descPools[i].Setup([]() {});
+	}
+
+	pipelineLayouts_.push_back(layout);
 	return layout;
 }
 
 void VulkanRenderManager::DestroyPipelineLayout(VKRPipelineLayout *layout) {
+	for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) {
+		layout->descPools[i].Destroy();
+	}
+
 	vulkan_->Delete().QueueDeletePipelineLayout(layout->pipelineLayout);
 	vulkan_->Delete().QueueDeleteDescriptorSetLayout(layout->descriptorSetLayout);
+	for (auto iter = pipelineLayouts_.begin(); iter != pipelineLayouts_.end(); iter++) {
+		if (*iter == layout) {
+			pipelineLayouts_.erase(iter);
+			break;
+		}
+	}
+}
+
+void VulkanRenderManager::FlushDescriptors(int frame) {
+	for (auto iter : pipelineLayouts_) {
+		iter->FlushDescSets(vulkan_, frame);
+	}
+}
+
+void VulkanRenderManager::ResetDescriptorLists(int frame) {
+	for (auto iter : pipelineLayouts_) {
+		iter->flushedDescriptors_[frame] = 0;
+		iter->descSets_[frame].clear();
+		iter->descData_[frame].clear();
+	}
+}
+
+void VKRPipelineLayout::FlushDescSets(VulkanContext *vulkan, int frame) {
+	_dbg_assert_(frame < VulkanContext::MAX_INFLIGHT_FRAMES);
+
+	VulkanDescSetPool &pool = descPools[frame];
+	FastVec<PackedDescriptor> &descData = descData_[frame];
+	FastVec<PendingDescSet> &descSets = descSets_[frame];
+
+	pool.Reset();
+
+	// This will write all descriptors.
+	// Initially, we won't do any de-duplication, so no hashmap lookups but also extra cost of writing additional descriptors.
+	// A short look-back might be enough?
+
+	// Initially, let's do naive single desc set writes.
+	VkWriteDescriptorSet writes[MAX_DESC_SET_BINDINGS];
+	VkDescriptorImageInfo imageInfo[MAX_DESC_SET_BINDINGS];  // just picked a practical number
+	VkDescriptorBufferInfo bufferInfo[MAX_DESC_SET_BINDINGS];
+
+	for (size_t index = flushedDescriptors_[frame]; index < descSets.size(); index++) {
+		auto &d = descSets[index];
+
+		// TODO: This is where to look up to see if we already have an identical descriptor previously in the array.
+		// We can do this with a simple custom hash map here that doesn't handle collisions, since false positives aren't too bad.
+		// Should probably check history, one or two items, then fall back to lookup. Or we should do the history lookup in BindDescriptors...
+
+		// For now we just allocate unconditionally.
+		d.set = pool.Allocate(1, &descriptorSetLayout, nullptr);
+
+		const PackedDescriptor *data = descData.begin() + d.offset;
+		int numWrites = 0;
+		int numBuffers = 0;
+		int numImages = 0;
+		for (int i = 0; i < d.count; i++) {
+			if (!data[i].image.view) {  // This automatically also checks for an null buffer.
+				continue;
+			}
+
+			switch (this->bindingTypes[i]) {
+			case BindingType::COMBINED_IMAGE_SAMPLER:
+				_dbg_assert_(data[i].image.sampler != VK_NULL_HANDLE);
+				imageInfo[numImages].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+				imageInfo[numImages].imageView = data[i].image.view;
+				imageInfo[numImages].sampler = data[i].image.sampler;
+				writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
+				writes[numWrites].pImageInfo = &imageInfo[numImages];
+				writes[numWrites].pBufferInfo = nullptr;
+				numImages++;
+				break;
+			case BindingType::STORAGE_IMAGE_COMPUTE:
+				imageInfo[numImages].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+				imageInfo[numImages].imageView = data[i].image.view;
+				imageInfo[numImages].sampler = VK_NULL_HANDLE;
+				writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
+				writes[numWrites].pImageInfo = &imageInfo[numImages];
+				writes[numWrites].pBufferInfo = nullptr;
+				numImages++;
+				break;
+			case BindingType::STORAGE_BUFFER_VERTEX:
+			case BindingType::STORAGE_BUFFER_COMPUTE:
+				bufferInfo[numBuffers].buffer = data[i].buffer.buffer;
+				bufferInfo[numBuffers].offset = data[i].buffer.offset;
+				bufferInfo[numBuffers].range = data[i].buffer.range;
+				writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
+				writes[numWrites].pBufferInfo = &bufferInfo[numBuffers];
+				writes[numWrites].pImageInfo = nullptr;
+				numBuffers++;
+				break;
+			case BindingType::UNIFORM_BUFFER_DYNAMIC_ALL:
+			case BindingType::UNIFORM_BUFFER_DYNAMIC_VERTEX:
+				bufferInfo[numBuffers].buffer = data[i].buffer.buffer;
+				bufferInfo[numBuffers].offset = 0;
+				bufferInfo[numBuffers].range = data[i].buffer.range;
+				writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
+				writes[numWrites].pBufferInfo = &bufferInfo[numBuffers];
+				writes[numWrites].pImageInfo = nullptr;
+				numBuffers++;
+				break;
+			}
+			writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
+			writes[numWrites].pNext = nullptr;
+			writes[numWrites].descriptorCount = 1;
+			writes[numWrites].dstArrayElement = 0;
+			writes[numWrites].dstBinding = i;
+			writes[numWrites].dstSet = d.set;
+			writes[numWrites].pTexelBufferView = nullptr;
+			numWrites++;
+		}
+
+		vkUpdateDescriptorSets(vulkan->GetDevice(), numWrites, writes, 0, nullptr);
+	}
+
+	flushedDescriptors_[frame] = (int)descSets.size();
 }

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -185,19 +185,55 @@ struct CompileQueueEntry {
 	VkSampleCountFlagBits sampleCount;
 };
 
+// Pending descriptor sets.
+// TODO: Sort these by VKRPipelineLayout to avoid storing it for each element.
+struct PendingDescSet {
+	int offset;  // probably enough with a u16.
+	u8 count;
+	VkDescriptorSet set;
+};
+
+struct PackedDescriptor {
+	union {
+		struct {
+			VkImageView view;
+			VkSampler sampler;
+		} image;
+		struct {
+			VkBuffer buffer;
+			uint32_t offset;
+			uint32_t range;
+		} buffer;
+	};
+};
+
 // Note that we only support a single descriptor set due to compatibility with some ancient devices.
 // We should probably eventually give that up.
 struct VKRPipelineLayout {
+	VKRPipelineLayout() {}
 	~VKRPipelineLayout() {
 		_assert_(!pipelineLayout && !descriptorSetLayout);
+		_assert_(descPools[0].IsDestroyed());
 	}
 	enum { MAX_DESC_SET_BINDINGS = 10 };
 	BindingType bindingTypes[MAX_DESC_SET_BINDINGS];
-	uint32_t bindingCount;
-	VkPipelineLayout pipelineLayout;
-	VkDescriptorSetLayout descriptorSetLayout;  // only support 1 for now.
+
+	uint32_t bindingTypesCount = 0;
+	VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
+	VkDescriptorSetLayout descriptorSetLayout = VK_NULL_HANDLE;  // only support 1 for now.
 	int pushConstSize = 0;
-	const char *tag;
+	const char *tag = nullptr;
+
+	// The pipeline layout owns the descriptor set pools. Don't go create excessive layouts.
+	VulkanDescSetPool descPools[VulkanContext::MAX_INFLIGHT_FRAMES];
+
+	// TODO: We should be able to get away with a single descData_/descSets_ and then send it along,
+	// but it's easier to just segregate by frame id.
+	FastVec<PackedDescriptor> descData_[VulkanContext::MAX_INFLIGHT_FRAMES];
+	FastVec<PendingDescSet> descSets_[VulkanContext::MAX_INFLIGHT_FRAMES];
+	int flushedDescriptors_[VulkanContext::MAX_INFLIGHT_FRAMES]{};
+
+	void FlushDescSets(VulkanContext *vulkan, int frame);
 };
 
 class VulkanRenderManager {
@@ -283,6 +319,7 @@ public:
 		//     DebugBreak();
 		// }
 		curPipelineFlags_ |= flags;
+		curPipelineLayout_ = pipelineLayout;
 		return true;
 	}
 
@@ -412,6 +449,46 @@ public:
 			curRenderStep_->render.stencilStore = VKRRenderPassStoreAction::DONT_CARE;
 	}
 
+private:
+	// Descriptors will match the current pipeline layout, set by the last call to BindPipeline.
+	// Count is the count of void*s. Two are needed for COMBINED_IMAGE_SAMPLER, everything else is a single one.
+	// The goal is to keep this function very small and fast, and do the expensive work on the render thread or
+	// another thread.
+	int BindDescriptors(const PackedDescriptor *desc, int count) {
+		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
+
+		int curFrame = vulkan_->GetCurFrame();
+
+		size_t offset = curPipelineLayout_->descData_[curFrame].size();
+		curPipelineLayout_->descData_[curFrame].extend(desc, count);
+
+		int setIndex = (int)curPipelineLayout_->descSets_[curFrame].size();
+		PendingDescSet &descSet = curPipelineLayout_->descSets_[curFrame].push_uninitialized();
+		descSet.offset = (uint32_t)offset;
+		descSet.count = count;
+		descSet.set = VK_NULL_HANDLE;  // to be filled in
+		return setIndex;
+	}
+
+public:
+	void Draw(const PackedDescriptor *desc, int descCount, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, int count, int offset = 0) {
+		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_);
+		int setIndex = BindDescriptors(desc, descCount);
+		VkRenderData &data = curRenderStep_->commands.push_uninitialized();
+		data.cmd = VKRRenderCommand::DRAW;
+		data.draw.count = count;
+		data.draw.offset = offset;
+		data.draw.ds = VK_NULL_HANDLE;
+		data.draw.descSetIndex = setIndex;
+		data.draw.vbuffer = vbuffer;
+		data.draw.voffset = voffset;
+		data.draw.numUboOffsets = numUboOffsets;
+		_dbg_assert_(numUboOffsets <= ARRAY_SIZE(data.draw.uboOffsets));
+		for (int i = 0; i < numUboOffsets; i++)
+			data.draw.uboOffsets[i] = uboOffsets[i];
+		curRenderStep_->render.numDraws++;
+	}
+
 	void Draw(VkDescriptorSet descSet, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, int count, int offset = 0) {
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_);
 		VkRenderData &data = curRenderStep_->commands.push_uninitialized();
@@ -428,6 +505,26 @@ public:
 		curRenderStep_->render.numDraws++;
 	}
 
+	void DrawIndexed(const PackedDescriptor *desc, int descCount, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, VkBuffer ibuffer, int ioffset, int count, int numInstances) {
+		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_);
+		int setIndex = BindDescriptors(desc, descCount);
+		VkRenderData &data = curRenderStep_->commands.push_uninitialized();
+		data.cmd = VKRRenderCommand::DRAW_INDEXED;
+		data.drawIndexed.count = count;
+		data.drawIndexed.instances = numInstances;
+		data.drawIndexed.ds = VK_NULL_HANDLE;
+		data.drawIndexed.descSetIndex = setIndex;
+		data.drawIndexed.vbuffer = vbuffer;
+		data.drawIndexed.voffset = voffset;
+		data.drawIndexed.ibuffer = ibuffer;
+		data.drawIndexed.ioffset = ioffset;
+		data.drawIndexed.numUboOffsets = numUboOffsets;
+		_dbg_assert_(numUboOffsets <= ARRAY_SIZE(data.drawIndexed.uboOffsets));
+		for (int i = 0; i < numUboOffsets; i++)
+			data.drawIndexed.uboOffsets[i] = uboOffsets[i];
+		curRenderStep_->render.numDraws++;
+	}
+
 	void DrawIndexed(VkDescriptorSet descSet, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, VkBuffer ibuffer, int ioffset, int count, int numInstances) {
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_);
 		VkRenderData &data = curRenderStep_->commands.push_uninitialized();
@@ -505,6 +602,9 @@ private:
 	void PresentWaitThreadFunc();
 	void PollPresentTiming();
 
+	void ResetDescriptorLists(int frame);
+	void FlushDescriptors(int frame);
+
 	FrameDataShared frameDataShared_;
 
 	FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES];
@@ -572,9 +672,13 @@ private:
 	SimpleStat initTimeMs_;
 	SimpleStat totalGPUTimeMs_;
 	SimpleStat renderCPUTimeMs_;
+	SimpleStat descUpdateTimeMs_;
 
 	std::function<void(InvalidationCallbackFlags)> invalidationCallback_;
 
 	uint64_t frameIdGen_ = FRAME_TIME_HISTORY_LENGTH;
 	HistoryBuffer<FrameTimeData, FRAME_TIME_HISTORY_LENGTH> &frameTimeHistory_;
+
+	VKRPipelineLayout *curPipelineLayout_ = nullptr;
+	std::vector<VKRPipelineLayout *> pipelineLayouts_;
 };

Common/GPU/Vulkan/thin3d_vulkan.cpp

@@ -27,19 +27,13 @@
 #include "Common/Data/Convert/SmallDataConvert.h"
 #include "Common/GPU/thin3d.h"
 #include "Common/GPU/Vulkan/VulkanRenderManager.h"
-#include "Common/GPU/Vulkan/VulkanDescSet.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
 #include "Common/GPU/Vulkan/VulkanImage.h"
 #include "Common/GPU/Vulkan/VulkanMemory.h"
 #include "Common/GPU/Vulkan/VulkanLoader.h"
 #include "Common/Thread/Promise.h"
 
-// We support a frame-global descriptor set, which can be optionally used by other code,
-// but is not directly used by thin3d. It has to be defined here though, be in set 0
-// and specified in every pipeline layout, otherwise it can't sit undisturbed when other
-// descriptor sets are bound on top.
-
-// For descriptor set 1, we use a simple descriptor set for all thin3d rendering: 1 UBO binding point, 3 combined texture/samples.
+// For descriptor set 0 (the only one), we use a simple descriptor set for all thin3d rendering: 1 UBO binding point, 3 combined texture/samples.
 //
 // binding 0 - uniform buffer
 // binding 1 - texture/sampler
@@ -510,7 +504,7 @@ public:
 		}
 	}
 
-	VkDescriptorSet GetOrCreateDescriptorSet(VkBuffer buffer);
+	void BindDescriptors(VkBuffer buffer, PackedDescriptor descriptors[4]);
 
 	std::vector<std::string> GetFeatureList() const override;
 	std::vector<std::string> GetExtensionList(bool device, bool enabledOnly) const override;
@@ -562,19 +556,6 @@ private:
 
 	VulkanPushPool *push_ = nullptr;
 
-	struct FrameData {
-		FrameData() : descriptorPool("VKContext", false) {
-			descriptorPool.Setup([this] { descSets_.clear(); });
-		}
-		// Per-frame descriptor set cache. As it's per frame and reset every frame, we don't need to
-		// worry about invalidating descriptors pointing to deleted textures.
-		// However! ARM is not a fan of doing it this way.
-		std::map<DescriptorSetKey, VkDescriptorSet> descSets_;
-		VulkanDescSetPool descriptorPool;
-	};
-
-	FrameData frame_[VulkanContext::MAX_INFLIGHT_FRAMES];
-
 	DeviceCaps caps_{};
 
 	uint8_t stencilRef_ = 0;
@@ -1053,10 +1034,6 @@ VKContext::VKContext(VulkanContext *vulkan, bool useRenderThread)
 	}
 	pipelineLayout_ = renderManager_.CreatePipelineLayout(bindings, ARRAY_SIZE(bindings), caps_.geometryShaderSupported, "thin3d_layout");
 
-	for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) {
-		frame_[i].descriptorPool.Create(vulkan_, bindings, ARRAY_SIZE(bindings), 1024);
-	}
-
 	VkPipelineCacheCreateInfo pc{ VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO };
 	VkResult res = vkCreatePipelineCache(vulkan_->GetDevice(), &pc, nullptr, &pipelineCache_);
 	_assert_(VK_SUCCESS == res);
@@ -1066,10 +1043,6 @@ VKContext::~VKContext() {
 	DestroyPresets();
 
 	delete nullTexture_;
-	// This also destroys all descriptor sets.
-	for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) {
-		frame_[i].descriptorPool.Destroy();
-	}
 	push_->Destroy();
 	delete push_;
 	renderManager_.DestroyPipelineLayout(pipelineLayout_);
@@ -1078,12 +1051,7 @@ VKContext::~VKContext() {
 
 void VKContext::BeginFrame(DebugFlags debugFlags) {
 	renderManager_.BeginFrame(debugFlags & DebugFlags::PROFILE_TIMESTAMPS, debugFlags & DebugFlags::PROFILE_SCOPES);
-
-	FrameData &frame = frame_[vulkan_->GetCurFrame()];
-
 	push_->BeginFrame();
-
-	frame.descriptorPool.Reset();
 }
 
 void VKContext::EndFrame() {
@@ -1121,81 +1089,30 @@ void VKContext::WipeQueue() {
 	renderManager_.Wipe();
 }
 
-VkDescriptorSet VKContext::GetOrCreateDescriptorSet(VkBuffer buf) {
-	DescriptorSetKey key{};
-
-	FrameData *frame = &frame_[vulkan_->GetCurFrame()];
+void VKContext::BindDescriptors(VkBuffer buf, PackedDescriptor descriptors[4]) {
+	descriptors[0].buffer.buffer = buf;
+	descriptors[0].buffer.offset = 0;  // dynamic
+	descriptors[0].buffer.range = curPipeline_->GetUBOSize();
 
+	int numDescs = 1;
 	for (int i = 0; i < MAX_BOUND_TEXTURES; ++i) {
+		VkImageView view;
+		VkSampler sampler;
 		if (boundTextures_[i]) {
-			key.imageViews_[i] = (boundTextureFlags_[i] & TextureBindFlags::VULKAN_BIND_ARRAY) ? boundTextures_[i]->GetImageArrayView() : boundTextures_[i]->GetImageView();
+			view = (boundTextureFlags_[i] & TextureBindFlags::VULKAN_BIND_ARRAY) ? boundTextures_[i]->GetImageArrayView() : boundTextures_[i]->GetImageView();
 		} else {
-			key.imageViews_[i] = boundImageView_[i];
+			view = boundImageView_[i];
 		}
-		key.samplers_[i] = boundSamplers_[i];
-	}
-	key.buffer_ = buf;
+		sampler = boundSamplers_[i] ? boundSamplers_[i]->GetSampler() : VK_NULL_HANDLE;
 
-	auto iter = frame->descSets_.find(key);
-	if (iter != frame->descSets_.end()) {
-		return iter->second;
-	}
-
-	VkDescriptorSet descSet = frame->descriptorPool.Allocate(1, &pipelineLayout_->descriptorSetLayout, "thin3d_descset");
-	if (descSet == VK_NULL_HANDLE) {
-		ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed");
-		return VK_NULL_HANDLE;
-	}
-
-	vulkan_->SetDebugName(descSet, VK_OBJECT_TYPE_DESCRIPTOR_SET, "(thin3d desc set)");
-
-	VkDescriptorBufferInfo bufferDesc;
-	bufferDesc.buffer = buf;
-	bufferDesc.offset = 0;
-	bufferDesc.range = curPipeline_->GetUBOSize();
-
-	VkDescriptorImageInfo imageDesc[MAX_BOUND_TEXTURES]{};
-	VkWriteDescriptorSet writes[1 + MAX_BOUND_TEXTURES]{};
-
-	// If handles are NULL for whatever buggy reason, it's best to leave the descriptors
-	// unwritten instead of trying to write a zero, which is not legal.
-
-	int numWrites = 0;
-	if (buf) {
-		writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
-		writes[numWrites].dstSet = descSet;
-		writes[numWrites].dstArrayElement = 0;
-		writes[numWrites].dstBinding = 0;
-		writes[numWrites].pBufferInfo = &bufferDesc;
-		writes[numWrites].pImageInfo = nullptr;
-		writes[numWrites].pTexelBufferView = nullptr;
-		writes[numWrites].descriptorCount = 1;
-		writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
-		numWrites++;
-	}
-
-	for (int i = 0; i < MAX_BOUND_TEXTURES; ++i) {
-		if (key.imageViews_[i] && key.samplers_[i] && key.samplers_[i]->GetSampler()) {
-			imageDesc[i].imageView = key.imageViews_[i];
-			imageDesc[i].sampler = key.samplers_[i]->GetSampler();
-			imageDesc[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
-			writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
-			writes[numWrites].dstSet = descSet;
-			writes[numWrites].dstArrayElement = 0;
-			writes[numWrites].dstBinding = i + 1;
-			writes[numWrites].pBufferInfo = nullptr;
-			writes[numWrites].pImageInfo = &imageDesc[i];
-			writes[numWrites].pTexelBufferView = nullptr;
-			writes[numWrites].descriptorCount = 1;
-			writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
-			numWrites++;
+		if (view && sampler) {
+			descriptors[i + 1].image.view = view;
+			descriptors[i + 1].image.sampler = sampler;
+		} else {
+			descriptors[i + 1].image.view = VK_NULL_HANDLE;
+			descriptors[i + 1].image.sampler = VK_NULL_HANDLE;
 		}
 	}
-
-	vkUpdateDescriptorSets(device_, numWrites, writes, 0, nullptr);
-
-	frame->descSets_[key] = descSet;
-	return descSet;
 }
 
 Pipeline *VKContext::CreateGraphicsPipeline(const PipelineDesc &desc, const char *tag) {
@@ -1445,7 +1362,7 @@ void VKContext::BindTextures(int start, int count, Texture **textures, TextureBi
 		boundTextures_[i] = static_cast<VKTexture *>(textures[i - start]);
 		boundTextureFlags_[i] = flags;
 		if (boundTextures_[i]) {
-			// If a texture is bound, we set these up in GetOrCreateDescriptorSet too.
+			// If a texture is bound, we set these up in BindDescriptors too.
 			// But we might need to set the view here anyway so it can be queried using GetNativeObject.
 			if (flags & TextureBindFlags::VULKAN_BIND_ARRAY) {
 				boundImageView_[i] = boundTextures_[i]->GetImageArrayView();
@@ -1498,15 +1415,11 @@ void VKContext::Draw(int vertexCount, int offset) {
 	uint32_t ubo_offset = (uint32_t)curPipeline_->PushUBO(push_, vulkan_, &vulkanUBObuf);
 	size_t vbBindOffset = push_->Push(vbuf->GetData(), vbuf->GetSize(), 4, &vulkanVbuf);
 
-	VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf);
-	if (descSet == VK_NULL_HANDLE) {
-		ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__);
-		return;
-	}
-
 	BindCurrentPipeline();
 	ApplyDynamicState();
-	renderManager_.Draw(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount, offset);
+	PackedDescriptor descriptors[4];
+	BindDescriptors(vulkanUBObuf, descriptors);
+	renderManager_.Draw(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount, offset);
 }
 
 void VKContext::DrawIndexed(int vertexCount, int offset) {
@@ -1518,15 +1431,11 @@ void VKContext::DrawIndexed(int vertexCount, int offset) {
 	size_t vbBindOffset = push_->Push(vbuf->GetData(), vbuf->GetSize(), 4, &vulkanVbuf);
 	size_t ibBindOffset = push_->Push(ibuf->GetData(), ibuf->GetSize(), 4, &vulkanIbuf);
 
-	VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf);
-	if (descSet == VK_NULL_HANDLE) {
-		ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__);
-		return;
-	}
-
 	BindCurrentPipeline();
 	ApplyDynamicState();
-	renderManager_.DrawIndexed(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vulkanIbuf, (int)ibBindOffset + offset * sizeof(uint32_t), vertexCount, 1);
+	PackedDescriptor descriptors[4];
+	BindDescriptors(vulkanUBObuf, descriptors);
+	renderManager_.DrawIndexed(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vulkanIbuf, (int)ibBindOffset + offset * sizeof(uint32_t), vertexCount, 1);
 }
 
 void VKContext::DrawUP(const void *vdata, int vertexCount) {
@@ -1544,15 +1453,11 @@ void VKContext::DrawUP(const void *vdata, int vertexCount) {
 
 	uint32_t ubo_offset = (uint32_t)curPipeline_->PushUBO(push_, vulkan_, &vulkanUBObuf);
 
-	VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf);
-	if (descSet == VK_NULL_HANDLE) {
-		ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__);
-		return;
-	}
-
 	BindCurrentPipeline();
 	ApplyDynamicState();
-	renderManager_.Draw(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount);
+	PackedDescriptor descriptors[4];
+	BindDescriptors(vulkanUBObuf, descriptors);
+	renderManager_.Draw(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount);
 }
 
 void VKContext::BindCurrentPipeline() {

Common/UI/Context.cpp

@@ -83,8 +83,6 @@ void UIContext::BeginPipeline(Draw::Pipeline *pipeline, Draw::SamplerState *samp
 	// Also clear out any other textures bound.
 	Draw::SamplerState *samplers[3]{ samplerState };
 	draw_->BindSamplerStates(0, 3, samplers);
-	Draw::Texture *textures[2]{};
-	draw_->BindTextures(1, 2, textures);
 	RebindTexture();
 	UIBegin(pipeline);
 }