From f0ee3b8daae91c61258759de12d0cf702eb5d2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Mon, 9 Oct 2023 09:44:58 +0200 Subject: [PATCH] Fill in descriptors on the render thread in the PPSSPP UI. --- Common/Data/Collections/FastVec.h | 6 + Common/GPU/Vulkan/VulkanDescSet.cpp | 4 +- Common/GPU/Vulkan/VulkanDescSet.h | 9 +- Common/GPU/Vulkan/VulkanQueueRunner.cpp | 25 +++- Common/GPU/Vulkan/VulkanQueueRunner.h | 10 +- Common/GPU/Vulkan/VulkanRenderManager.cpp | 152 ++++++++++++++++++++-- Common/GPU/Vulkan/VulkanRenderManager.h | 112 +++++++++++++++- Common/GPU/Vulkan/thin3d_vulkan.cpp | 151 ++++----------------- Common/UI/Context.cpp | 2 - 9 files changed, 321 insertions(+), 150 deletions(-) diff --git a/Common/Data/Collections/FastVec.h b/Common/Data/Collections/FastVec.h index 819e68799b..b78cab6f77 100644 --- a/Common/Data/Collections/FastVec.h +++ b/Common/Data/Collections/FastVec.h @@ -117,6 +117,12 @@ public: IncreaseCapacityTo(newCapacity); } + void extend(const T *newData, size_t count) { + IncreaseCapacityTo(size_ + count); + memcpy(data_ + size_, newData, count * sizeof(T)); + size_ += count; + } + void LockCapacity() { #ifdef _DEBUG capacityLocked_ = true; diff --git a/Common/GPU/Vulkan/VulkanDescSet.cpp b/Common/GPU/Vulkan/VulkanDescSet.cpp index 06c2cc48fd..444a3670dc 100644 --- a/Common/GPU/Vulkan/VulkanDescSet.cpp +++ b/Common/GPU/Vulkan/VulkanDescSet.cpp @@ -75,7 +75,9 @@ VkDescriptorSet VulkanDescSetPool::Allocate(int n, const VkDescriptorSetLayout * usage_++; - vulkan_->SetDebugName(desc, VK_OBJECT_TYPE_DESCRIPTOR_SET, tag); + if (tag) { + vulkan_->SetDebugName(desc, VK_OBJECT_TYPE_DESCRIPTOR_SET, tag); + } return desc; } diff --git a/Common/GPU/Vulkan/VulkanDescSet.h b/Common/GPU/Vulkan/VulkanDescSet.h index e42e24b30c..82007962a3 100644 --- a/Common/GPU/Vulkan/VulkanDescSet.h +++ b/Common/GPU/Vulkan/VulkanDescSet.h @@ -18,7 +18,7 @@ enum class BindingType { // Only appropriate for use in a per-frame pool. class VulkanDescSetPool { public: - VulkanDescSetPool(const char *tag, bool grow) : tag_(tag), grow_(grow) {} + VulkanDescSetPool(const char *tag = "", bool grow = true) : tag_(tag), grow_(grow) {} ~VulkanDescSetPool(); // Must call this before use: defines how to clear cache of ANY returned values from Allocate(). @@ -32,6 +32,13 @@ public: void Reset(); void Destroy(); + void SetTag(const char *tag) { + tag_ = tag; + } + bool IsDestroyed() const { + return !descPool_; + } + private: VkResult Recreate(bool grow); diff --git a/Common/GPU/Vulkan/VulkanQueueRunner.cpp b/Common/GPU/Vulkan/VulkanQueueRunner.cpp index c38a2bd6d9..0b774b2569 100644 --- a/Common/GPU/Vulkan/VulkanQueueRunner.cpp +++ b/Common/GPU/Vulkan/VulkanQueueRunner.cpp @@ -338,7 +338,7 @@ void VulkanQueueRunner::PreprocessSteps(std::vector &steps) { } } -void VulkanQueueRunner::RunSteps(std::vector &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) { +void VulkanQueueRunner::RunSteps(std::vector &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps) { QueueProfileContext *profile = frameData.profile.enabled ? &frameData.profile : nullptr; if (profile) @@ -394,7 +394,7 @@ void VulkanQueueRunner::RunSteps(std::vector &steps, FrameData &frame vkCmdBeginDebugUtilsLabelEXT(cmd, &labelInfo); } } - PerformRenderPass(step, cmd); + PerformRenderPass(step, cmd, curFrame); break; case VKRStepType::COPY: PerformCopy(step, cmd); @@ -1103,7 +1103,7 @@ void TransitionFromOptimal(VkCommandBuffer cmd, VkImage colorImage, VkImageLayou } } -void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd) { +void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd, int curFrame) { for (size_t i = 0; i < step.preTransitions.size(); i++) { const TransitionRequest &iter = step.preTransitions[i]; if (iter.aspect == VK_IMAGE_ASPECT_COLOR_BIT && iter.fb->color.layout != iter.targetLayout) { @@ -1199,6 +1199,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c // The stencil ones are very commonly mostly redundant so let's eliminate them where possible. // Might also want to consider scissor and viewport. VkPipeline lastPipeline = VK_NULL_HANDLE; + VKRPipelineLayout *vkrPipelineLayout = nullptr; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; bool pipelineOK = false; @@ -1241,7 +1242,8 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c if (pipeline != VK_NULL_HANDLE) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); - pipelineLayout = c.pipeline.pipelineLayout->pipelineLayout; + vkrPipelineLayout = c.pipeline.pipelineLayout; + pipelineLayout = vkrPipelineLayout->pipelineLayout; lastGraphicsPipeline = graphicsPipeline; pipelineOK = true; } else { @@ -1336,7 +1338,12 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c case VKRRenderCommand::DRAW_INDEXED: if (pipelineOK) { - vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.drawIndexed.ds, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets); + VkDescriptorSet set; + if (c.drawIndexed.ds) + set = c.drawIndexed.ds; + else + set = vkrPipelineLayout->descSets_[curFrame][c.drawIndexed.descSetIndex].set; + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets); vkCmdBindIndexBuffer(cmd, c.drawIndexed.ibuffer, c.drawIndexed.ioffset, VK_INDEX_TYPE_UINT16); VkDeviceSize voffset = c.drawIndexed.voffset; vkCmdBindVertexBuffers(cmd, 0, 1, &c.drawIndexed.vbuffer, &voffset); @@ -1346,7 +1353,13 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c case VKRRenderCommand::DRAW: if (pipelineOK) { - vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &c.draw.ds, c.draw.numUboOffsets, c.draw.uboOffsets); + VkDescriptorSet set; + if (c.drawIndexed.ds) + set = c.drawIndexed.ds; + else + set = vkrPipelineLayout->descSets_[curFrame][c.drawIndexed.descSetIndex].set; + _dbg_assert_(set != VK_NULL_HANDLE); + vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, c.draw.numUboOffsets, c.draw.uboOffsets); if (c.draw.vbuffer) { vkCmdBindVertexBuffers(cmd, 0, 1, &c.draw.vbuffer, &c.draw.voffset); } diff --git a/Common/GPU/Vulkan/VulkanQueueRunner.h b/Common/GPU/Vulkan/VulkanQueueRunner.h index 4ce030e370..e26b33cf4e 100644 --- a/Common/GPU/Vulkan/VulkanQueueRunner.h +++ b/Common/GPU/Vulkan/VulkanQueueRunner.h @@ -71,6 +71,7 @@ struct VkRenderData { } compute_pipeline; struct { VkDescriptorSet ds; + uint32_t descSetIndex; int numUboOffsets; uint32_t uboOffsets[3]; VkBuffer vbuffer; @@ -80,6 +81,7 @@ struct VkRenderData { } draw; struct { VkDescriptorSet ds; + uint32_t descSetIndex; uint32_t uboOffsets[3]; uint16_t numUboOffsets; uint16_t instances; @@ -119,9 +121,7 @@ struct VkRenderData { const char *annotation; } debugAnnotation; struct { - int setNumber; - VkDescriptorSet set; - VkPipelineLayout pipelineLayout; + int setIndex; } bindDescSet; }; }; @@ -231,7 +231,7 @@ public: } void PreprocessSteps(std::vector &steps); - void RunSteps(std::vector &steps, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false); + void RunSteps(std::vector &steps, int curFrame, FrameData &frameData, FrameDataShared &frameDataShared, bool keepSteps = false); void LogSteps(const std::vector &steps, bool verbose); static std::string StepToString(VulkanContext *vulkan, const VKRStep &step); @@ -291,7 +291,7 @@ private: bool InitDepthStencilBuffer(VkCommandBuffer cmd); // Used for non-buffered rendering. VKRRenderPass *PerformBindFramebufferAsRenderTarget(const VKRStep &pass, VkCommandBuffer cmd); - void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd); + void PerformRenderPass(const VKRStep &pass, VkCommandBuffer cmd, int curFrame); void PerformCopy(const VKRStep &pass, VkCommandBuffer cmd); void PerformBlit(const VKRStep &pass, VkCommandBuffer cmd); void PerformReadback(const VKRStep &pass, VkCommandBuffer cmd, FrameData &frameData); diff --git a/Common/GPU/Vulkan/VulkanRenderManager.cpp b/Common/GPU/Vulkan/VulkanRenderManager.cpp index 92ff72ae18..b8c3937f5c 100644 --- a/Common/GPU/Vulkan/VulkanRenderManager.cpp +++ b/Common/GPU/Vulkan/VulkanRenderManager.cpp @@ -265,6 +265,7 @@ VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan, bool useThread, initTimeMs_("initTimeMs"), totalGPUTimeMs_("totalGPUTimeMs"), renderCPUTimeMs_("renderCPUTimeMs"), + descUpdateTimeMs_("descUpdateCPUTimeMs"), useRenderThread_(useThread), frameTimeHistory_(frameTimeHistory) { @@ -645,6 +646,8 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile PollPresentTiming(); + ResetDescriptorLists(curFrame); + int validBits = vulkan_->GetQueueFamilyProperties(vulkan_->GetGraphicsQueueFamilyIndex()).timestampValidBits; FrameTimeData &frameTimeData = frameTimeHistory_.Add(frameId); @@ -679,6 +682,9 @@ void VulkanRenderManager::BeginFrame(bool enableProfiling, bool enableLogProfile renderCPUTimeMs_.Update((frameData.profile.cpuEndTime - frameData.profile.cpuStartTime) * 1000.0); renderCPUTimeMs_.Format(line, sizeof(line)); str << line; + descUpdateTimeMs_.Update(frameData.profile.descWriteTime * 1000.0); + descUpdateTimeMs_.Format(line, sizeof(line)); + str << line; for (int i = 0; i < numQueries - 1; i++) { uint64_t diff = (queryResults[i + 1] - queryResults[i]) & timestampDiffMask; double milliseconds = (double)diff * timestampConversionFactor; @@ -1462,6 +1468,11 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) { } frameData.SubmitPending(vulkan_, FrameSubmitType::Pending, frameDataShared_); + // Flush descriptors. + double descStart = time_now_d(); + FlushDescriptors(task.frame); + frameData.profile.descWriteTime = time_now_d() - descStart; + if (!frameData.hasMainCommands) { // Effectively resets both main and present command buffers, since they both live in this pool. // We always record main commands first, so we don't need to reset the present command buffer separately. @@ -1483,11 +1494,11 @@ void VulkanRenderManager::Run(VKRRenderThreadTask &task) { int passes = GetVRPassesCount(); for (int i = 0; i < passes; i++) { PreVRFrameRender(i); - queueRunner_.RunSteps(task.steps, frameData, frameDataShared_, i < passes - 1); + queueRunner_.RunSteps(task.steps, task.frame, frameData, frameDataShared_, i < passes - 1); PostVRFrameRender(); } } else { - queueRunner_.RunSteps(task.steps, frameData, frameDataShared_); + queueRunner_.RunSteps(task.steps, task.frame, frameData, frameDataShared_); } switch (task.runType) { @@ -1562,16 +1573,16 @@ void VulkanRenderManager::ResetStats() { renderCPUTimeMs_.Reset(); } -VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindingTypes, size_t bindingCount, bool geoShadersEnabled, const char *tag) { +VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindingTypes, size_t bindingTypesCount, bool geoShadersEnabled, const char *tag) { VKRPipelineLayout *layout = new VKRPipelineLayout(); layout->tag = tag; - layout->bindingCount = (uint32_t)bindingCount; + layout->bindingTypesCount = (uint32_t)bindingTypesCount; - _dbg_assert_(bindingCount <= ARRAY_SIZE(layout->bindingTypes)); - memcpy(layout->bindingTypes, bindingTypes, sizeof(BindingType) * bindingCount); + _dbg_assert_(bindingTypesCount <= ARRAY_SIZE(layout->bindingTypes)); + memcpy(layout->bindingTypes, bindingTypes, sizeof(BindingType) * bindingTypesCount); VkDescriptorSetLayoutBinding bindings[VKRPipelineLayout::MAX_DESC_SET_BINDINGS]; - for (int i = 0; i < bindingCount; i++) { + for (int i = 0; i < bindingTypesCount; i++) { bindings[i].binding = i; bindings[i].descriptorCount = 1; bindings[i].pImmutableSamplers = nullptr; @@ -1611,7 +1622,7 @@ VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindin } VkDescriptorSetLayoutCreateInfo dsl = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO }; - dsl.bindingCount = (uint32_t)bindingCount; + dsl.bindingCount = (uint32_t)bindingTypesCount; dsl.pBindings = bindings; VkResult res = vkCreateDescriptorSetLayout(vulkan_->GetDevice(), &dsl, nullptr, &layout->descriptorSetLayout); _assert_(VK_SUCCESS == res && layout->descriptorSetLayout); @@ -1625,10 +1636,135 @@ VKRPipelineLayout *VulkanRenderManager::CreatePipelineLayout(BindingType *bindin vulkan_->SetDebugName(layout->descriptorSetLayout, VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT, tag); vulkan_->SetDebugName(layout->pipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT, tag); + + for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) { + layout->descPools[i].Create(vulkan_, bindingTypes, (uint32_t)bindingTypesCount, 512); + layout->descPools[i].Setup([]() {}); + } + + pipelineLayouts_.push_back(layout); return layout; } void VulkanRenderManager::DestroyPipelineLayout(VKRPipelineLayout *layout) { + for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) { + layout->descPools[i].Destroy(); + } + vulkan_->Delete().QueueDeletePipelineLayout(layout->pipelineLayout); vulkan_->Delete().QueueDeleteDescriptorSetLayout(layout->descriptorSetLayout); + for (auto iter = pipelineLayouts_.begin(); iter != pipelineLayouts_.end(); iter++) { + if (*iter == layout) { + pipelineLayouts_.erase(iter); + break; + } + } +} + +void VulkanRenderManager::FlushDescriptors(int frame) { + for (auto iter : pipelineLayouts_) { + iter->FlushDescSets(vulkan_, frame); + } +} + +void VulkanRenderManager::ResetDescriptorLists(int frame) { + for (auto iter : pipelineLayouts_) { + iter->flushedDescriptors_[frame] = 0; + iter->descSets_[frame].clear(); + iter->descData_[frame].clear(); + } +} + +void VKRPipelineLayout::FlushDescSets(VulkanContext *vulkan, int frame) { + _dbg_assert_(frame < VulkanContext::MAX_INFLIGHT_FRAMES); + + VulkanDescSetPool &pool = descPools[frame]; + FastVec &descData = descData_[frame]; + FastVec &descSets = descSets_[frame]; + + pool.Reset(); + + // This will write all descriptors. + // Initially, we won't do any de-duplication, so no hashmap lookups but also extra cost of writing additional descriptors. + // A short look-back might be enough? + + // Initially, let's do naive single desc set writes. + VkWriteDescriptorSet writes[MAX_DESC_SET_BINDINGS]; + VkDescriptorImageInfo imageInfo[MAX_DESC_SET_BINDINGS]; // just picked a practical number + VkDescriptorBufferInfo bufferInfo[MAX_DESC_SET_BINDINGS]; + + for (size_t index = flushedDescriptors_[frame]; index < descSets.size(); index++) { + auto &d = descSets[index]; + + // TODO: This is where to look up to see if we already have an identical descriptor previously in the array. + // We can do this with a simple custom hash map here that doesn't handle collisions, since false positives aren't too bad. + // Should probably check history, one or two items, then fall back to lookup. Or we should do the history lookup in BindDescriptors... + + // For now we just allocate unconditionally. + d.set = pool.Allocate(1, &descriptorSetLayout, nullptr); + + const PackedDescriptor *data = descData.begin() + d.offset; + int numWrites = 0; + int numBuffers = 0; + int numImages = 0; + for (int i = 0; i < d.count; i++) { + if (!data[i].image.view) { // This automatically also checks for an null buffer. + continue; + } + + switch (this->bindingTypes[i]) { + case BindingType::COMBINED_IMAGE_SAMPLER: + _dbg_assert_(data[i].image.sampler != VK_NULL_HANDLE); + imageInfo[numImages].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + imageInfo[numImages].imageView = data[i].image.view; + imageInfo[numImages].sampler = data[i].image.sampler; + writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + writes[numWrites].pImageInfo = &imageInfo[numImages]; + writes[numWrites].pBufferInfo = nullptr; + numImages++; + break; + case BindingType::STORAGE_IMAGE_COMPUTE: + imageInfo[numImages].imageLayout = VK_IMAGE_LAYOUT_GENERAL; + imageInfo[numImages].imageView = data[i].image.view; + imageInfo[numImages].sampler = VK_NULL_HANDLE; + writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[numWrites].pImageInfo = &imageInfo[numImages]; + writes[numWrites].pBufferInfo = nullptr; + numImages++; + break; + case BindingType::STORAGE_BUFFER_VERTEX: + case BindingType::STORAGE_BUFFER_COMPUTE: + bufferInfo[numBuffers].buffer = data[i].buffer.buffer; + bufferInfo[numBuffers].offset = data[i].buffer.offset; + bufferInfo[numBuffers].range = data[i].buffer.range; + writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[numWrites].pBufferInfo = &bufferInfo[numBuffers]; + writes[numWrites].pImageInfo = nullptr; + numBuffers++; + break; + case BindingType::UNIFORM_BUFFER_DYNAMIC_ALL: + case BindingType::UNIFORM_BUFFER_DYNAMIC_VERTEX: + bufferInfo[numBuffers].buffer = data[i].buffer.buffer; + bufferInfo[numBuffers].offset = 0; + bufferInfo[numBuffers].range = data[i].buffer.range; + writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC; + writes[numWrites].pBufferInfo = &bufferInfo[numBuffers]; + writes[numWrites].pImageInfo = nullptr; + numBuffers++; + break; + } + writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[numWrites].pNext = nullptr; + writes[numWrites].descriptorCount = 1; + writes[numWrites].dstArrayElement = 0; + writes[numWrites].dstBinding = i; + writes[numWrites].dstSet = d.set; + writes[numWrites].pTexelBufferView = nullptr; + numWrites++; + } + + vkUpdateDescriptorSets(vulkan->GetDevice(), numWrites, writes, 0, nullptr); + } + + flushedDescriptors_[frame] = (int)descSets.size(); } diff --git a/Common/GPU/Vulkan/VulkanRenderManager.h b/Common/GPU/Vulkan/VulkanRenderManager.h index 9a7ef0f795..d4d7be5bf7 100644 --- a/Common/GPU/Vulkan/VulkanRenderManager.h +++ b/Common/GPU/Vulkan/VulkanRenderManager.h @@ -185,19 +185,55 @@ struct CompileQueueEntry { VkSampleCountFlagBits sampleCount; }; +// Pending descriptor sets. +// TODO: Sort these by VKRPipelineLayout to avoid storing it for each element. +struct PendingDescSet { + int offset; // probably enough with a u16. + u8 count; + VkDescriptorSet set; +}; + +struct PackedDescriptor { + union { + struct { + VkImageView view; + VkSampler sampler; + } image; + struct { + VkBuffer buffer; + uint32_t offset; + uint32_t range; + } buffer; + }; +}; + // Note that we only support a single descriptor set due to compatibility with some ancient devices. // We should probably eventually give that up. struct VKRPipelineLayout { + VKRPipelineLayout() {} ~VKRPipelineLayout() { _assert_(!pipelineLayout && !descriptorSetLayout); + _assert_(descPools[0].IsDestroyed()); } enum { MAX_DESC_SET_BINDINGS = 10 }; BindingType bindingTypes[MAX_DESC_SET_BINDINGS]; - uint32_t bindingCount; - VkPipelineLayout pipelineLayout; - VkDescriptorSetLayout descriptorSetLayout; // only support 1 for now. + + uint32_t bindingTypesCount = 0; + VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; + VkDescriptorSetLayout descriptorSetLayout = VK_NULL_HANDLE; // only support 1 for now. int pushConstSize = 0; - const char *tag; + const char *tag = nullptr; + + // The pipeline layout owns the descriptor set pools. Don't go create excessive layouts. + VulkanDescSetPool descPools[VulkanContext::MAX_INFLIGHT_FRAMES]; + + // TODO: We should be able to get away with a single descData_/descSets_ and then send it along, + // but it's easier to just segregate by frame id. + FastVec descData_[VulkanContext::MAX_INFLIGHT_FRAMES]; + FastVec descSets_[VulkanContext::MAX_INFLIGHT_FRAMES]; + int flushedDescriptors_[VulkanContext::MAX_INFLIGHT_FRAMES]{}; + + void FlushDescSets(VulkanContext *vulkan, int frame); }; class VulkanRenderManager { @@ -283,6 +319,7 @@ public: // DebugBreak(); // } curPipelineFlags_ |= flags; + curPipelineLayout_ = pipelineLayout; return true; } @@ -412,6 +449,46 @@ public: curRenderStep_->render.stencilStore = VKRRenderPassStoreAction::DONT_CARE; } +private: + // Descriptors will match the current pipeline layout, set by the last call to BindPipeline. + // Count is the count of void*s. Two are needed for COMBINED_IMAGE_SAMPLER, everything else is a single one. + // The goal is to keep this function very small and fast, and do the expensive work on the render thread or + // another thread. + int BindDescriptors(const PackedDescriptor *desc, int count) { + _dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER); + + int curFrame = vulkan_->GetCurFrame(); + + size_t offset = curPipelineLayout_->descData_[curFrame].size(); + curPipelineLayout_->descData_[curFrame].extend(desc, count); + + int setIndex = (int)curPipelineLayout_->descSets_[curFrame].size(); + PendingDescSet &descSet = curPipelineLayout_->descSets_[curFrame].push_uninitialized(); + descSet.offset = (uint32_t)offset; + descSet.count = count; + descSet.set = VK_NULL_HANDLE; // to be filled in + return setIndex; + } + +public: + void Draw(const PackedDescriptor *desc, int descCount, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, int count, int offset = 0) { + _dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_); + int setIndex = BindDescriptors(desc, descCount); + VkRenderData &data = curRenderStep_->commands.push_uninitialized(); + data.cmd = VKRRenderCommand::DRAW; + data.draw.count = count; + data.draw.offset = offset; + data.draw.ds = VK_NULL_HANDLE; + data.draw.descSetIndex = setIndex; + data.draw.vbuffer = vbuffer; + data.draw.voffset = voffset; + data.draw.numUboOffsets = numUboOffsets; + _dbg_assert_(numUboOffsets <= ARRAY_SIZE(data.draw.uboOffsets)); + for (int i = 0; i < numUboOffsets; i++) + data.draw.uboOffsets[i] = uboOffsets[i]; + curRenderStep_->render.numDraws++; + } + void Draw(VkDescriptorSet descSet, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, int count, int offset = 0) { _dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_); VkRenderData &data = curRenderStep_->commands.push_uninitialized(); @@ -428,6 +505,26 @@ public: curRenderStep_->render.numDraws++; } + void DrawIndexed(const PackedDescriptor *desc, int descCount, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, VkBuffer ibuffer, int ioffset, int count, int numInstances) { + _dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_); + int setIndex = BindDescriptors(desc, descCount); + VkRenderData &data = curRenderStep_->commands.push_uninitialized(); + data.cmd = VKRRenderCommand::DRAW_INDEXED; + data.drawIndexed.count = count; + data.drawIndexed.instances = numInstances; + data.drawIndexed.ds = VK_NULL_HANDLE; + data.drawIndexed.descSetIndex = setIndex; + data.drawIndexed.vbuffer = vbuffer; + data.drawIndexed.voffset = voffset; + data.drawIndexed.ibuffer = ibuffer; + data.drawIndexed.ioffset = ioffset; + data.drawIndexed.numUboOffsets = numUboOffsets; + _dbg_assert_(numUboOffsets <= ARRAY_SIZE(data.drawIndexed.uboOffsets)); + for (int i = 0; i < numUboOffsets; i++) + data.drawIndexed.uboOffsets[i] = uboOffsets[i]; + curRenderStep_->render.numDraws++; + } + void DrawIndexed(VkDescriptorSet descSet, int numUboOffsets, const uint32_t *uboOffsets, VkBuffer vbuffer, int voffset, VkBuffer ibuffer, int ioffset, int count, int numInstances) { _dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER && curStepHasViewport_ && curStepHasScissor_); VkRenderData &data = curRenderStep_->commands.push_uninitialized(); @@ -505,6 +602,9 @@ private: void PresentWaitThreadFunc(); void PollPresentTiming(); + void ResetDescriptorLists(int frame); + void FlushDescriptors(int frame); + FrameDataShared frameDataShared_; FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES]; @@ -572,9 +672,13 @@ private: SimpleStat initTimeMs_; SimpleStat totalGPUTimeMs_; SimpleStat renderCPUTimeMs_; + SimpleStat descUpdateTimeMs_; std::function invalidationCallback_; uint64_t frameIdGen_ = FRAME_TIME_HISTORY_LENGTH; HistoryBuffer &frameTimeHistory_; + + VKRPipelineLayout *curPipelineLayout_ = nullptr; + std::vector pipelineLayouts_; }; diff --git a/Common/GPU/Vulkan/thin3d_vulkan.cpp b/Common/GPU/Vulkan/thin3d_vulkan.cpp index 118b38b8ce..6e376617e9 100644 --- a/Common/GPU/Vulkan/thin3d_vulkan.cpp +++ b/Common/GPU/Vulkan/thin3d_vulkan.cpp @@ -27,19 +27,13 @@ #include "Common/Data/Convert/SmallDataConvert.h" #include "Common/GPU/thin3d.h" #include "Common/GPU/Vulkan/VulkanRenderManager.h" -#include "Common/GPU/Vulkan/VulkanDescSet.h" #include "Common/GPU/Vulkan/VulkanContext.h" #include "Common/GPU/Vulkan/VulkanImage.h" #include "Common/GPU/Vulkan/VulkanMemory.h" #include "Common/GPU/Vulkan/VulkanLoader.h" #include "Common/Thread/Promise.h" -// We support a frame-global descriptor set, which can be optionally used by other code, -// but is not directly used by thin3d. It has to be defined here though, be in set 0 -// and specified in every pipeline layout, otherwise it can't sit undisturbed when other -// descriptor sets are bound on top. - -// For descriptor set 1, we use a simple descriptor set for all thin3d rendering: 1 UBO binding point, 3 combined texture/samples. +// For descriptor set 0 (the only one), we use a simple descriptor set for all thin3d rendering: 1 UBO binding point, 3 combined texture/samples. // // binding 0 - uniform buffer // binding 1 - texture/sampler @@ -510,7 +504,7 @@ public: } } - VkDescriptorSet GetOrCreateDescriptorSet(VkBuffer buffer); + void BindDescriptors(VkBuffer buffer, PackedDescriptor descriptors[4]); std::vector GetFeatureList() const override; std::vector GetExtensionList(bool device, bool enabledOnly) const override; @@ -562,19 +556,6 @@ private: VulkanPushPool *push_ = nullptr; - struct FrameData { - FrameData() : descriptorPool("VKContext", false) { - descriptorPool.Setup([this] { descSets_.clear(); }); - } - // Per-frame descriptor set cache. As it's per frame and reset every frame, we don't need to - // worry about invalidating descriptors pointing to deleted textures. - // However! ARM is not a fan of doing it this way. - std::map descSets_; - VulkanDescSetPool descriptorPool; - }; - - FrameData frame_[VulkanContext::MAX_INFLIGHT_FRAMES]; - DeviceCaps caps_{}; uint8_t stencilRef_ = 0; @@ -1053,10 +1034,6 @@ VKContext::VKContext(VulkanContext *vulkan, bool useRenderThread) } pipelineLayout_ = renderManager_.CreatePipelineLayout(bindings, ARRAY_SIZE(bindings), caps_.geometryShaderSupported, "thin3d_layout"); - for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) { - frame_[i].descriptorPool.Create(vulkan_, bindings, ARRAY_SIZE(bindings), 1024); - } - VkPipelineCacheCreateInfo pc{ VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO }; VkResult res = vkCreatePipelineCache(vulkan_->GetDevice(), &pc, nullptr, &pipelineCache_); _assert_(VK_SUCCESS == res); @@ -1066,10 +1043,6 @@ VKContext::~VKContext() { DestroyPresets(); delete nullTexture_; - // This also destroys all descriptor sets. - for (int i = 0; i < VulkanContext::MAX_INFLIGHT_FRAMES; i++) { - frame_[i].descriptorPool.Destroy(); - } push_->Destroy(); delete push_; renderManager_.DestroyPipelineLayout(pipelineLayout_); @@ -1078,12 +1051,7 @@ VKContext::~VKContext() { void VKContext::BeginFrame(DebugFlags debugFlags) { renderManager_.BeginFrame(debugFlags & DebugFlags::PROFILE_TIMESTAMPS, debugFlags & DebugFlags::PROFILE_SCOPES); - - FrameData &frame = frame_[vulkan_->GetCurFrame()]; - push_->BeginFrame(); - - frame.descriptorPool.Reset(); } void VKContext::EndFrame() { @@ -1121,81 +1089,30 @@ void VKContext::WipeQueue() { renderManager_.Wipe(); } -VkDescriptorSet VKContext::GetOrCreateDescriptorSet(VkBuffer buf) { - DescriptorSetKey key{}; - - FrameData *frame = &frame_[vulkan_->GetCurFrame()]; +void VKContext::BindDescriptors(VkBuffer buf, PackedDescriptor descriptors[4]) { + descriptors[0].buffer.buffer = buf; + descriptors[0].buffer.offset = 0; // dynamic + descriptors[0].buffer.range = curPipeline_->GetUBOSize(); + int numDescs = 1; for (int i = 0; i < MAX_BOUND_TEXTURES; ++i) { + VkImageView view; + VkSampler sampler; if (boundTextures_[i]) { - key.imageViews_[i] = (boundTextureFlags_[i] & TextureBindFlags::VULKAN_BIND_ARRAY) ? boundTextures_[i]->GetImageArrayView() : boundTextures_[i]->GetImageView(); + view = (boundTextureFlags_[i] & TextureBindFlags::VULKAN_BIND_ARRAY) ? boundTextures_[i]->GetImageArrayView() : boundTextures_[i]->GetImageView(); } else { - key.imageViews_[i] = boundImageView_[i]; + view = boundImageView_[i]; } - key.samplers_[i] = boundSamplers_[i]; - } - key.buffer_ = buf; + sampler = boundSamplers_[i] ? boundSamplers_[i]->GetSampler() : VK_NULL_HANDLE; - auto iter = frame->descSets_.find(key); - if (iter != frame->descSets_.end()) { - return iter->second; - } - - VkDescriptorSet descSet = frame->descriptorPool.Allocate(1, &pipelineLayout_->descriptorSetLayout, "thin3d_descset"); - if (descSet == VK_NULL_HANDLE) { - ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed"); - return VK_NULL_HANDLE; - } - - vulkan_->SetDebugName(descSet, VK_OBJECT_TYPE_DESCRIPTOR_SET, "(thin3d desc set)"); - - VkDescriptorBufferInfo bufferDesc; - bufferDesc.buffer = buf; - bufferDesc.offset = 0; - bufferDesc.range = curPipeline_->GetUBOSize(); - - VkDescriptorImageInfo imageDesc[MAX_BOUND_TEXTURES]{}; - VkWriteDescriptorSet writes[1 + MAX_BOUND_TEXTURES]{}; - - // If handles are NULL for whatever buggy reason, it's best to leave the descriptors - // unwritten instead of trying to write a zero, which is not legal. - - int numWrites = 0; - if (buf) { - writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[numWrites].dstSet = descSet; - writes[numWrites].dstArrayElement = 0; - writes[numWrites].dstBinding = 0; - writes[numWrites].pBufferInfo = &bufferDesc; - writes[numWrites].pImageInfo = nullptr; - writes[numWrites].pTexelBufferView = nullptr; - writes[numWrites].descriptorCount = 1; - writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC; - numWrites++; - } - - for (int i = 0; i < MAX_BOUND_TEXTURES; ++i) { - if (key.imageViews_[i] && key.samplers_[i] && key.samplers_[i]->GetSampler()) { - imageDesc[i].imageView = key.imageViews_[i]; - imageDesc[i].sampler = key.samplers_[i]->GetSampler(); - imageDesc[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - writes[numWrites].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[numWrites].dstSet = descSet; - writes[numWrites].dstArrayElement = 0; - writes[numWrites].dstBinding = i + 1; - writes[numWrites].pBufferInfo = nullptr; - writes[numWrites].pImageInfo = &imageDesc[i]; - writes[numWrites].pTexelBufferView = nullptr; - writes[numWrites].descriptorCount = 1; - writes[numWrites].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - numWrites++; + if (view && sampler) { + descriptors[i + 1].image.view = view; + descriptors[i + 1].image.sampler = sampler; + } else { + descriptors[i + 1].image.view = VK_NULL_HANDLE; + descriptors[i + 1].image.sampler = VK_NULL_HANDLE; } } - - vkUpdateDescriptorSets(device_, numWrites, writes, 0, nullptr); - - frame->descSets_[key] = descSet; - return descSet; } Pipeline *VKContext::CreateGraphicsPipeline(const PipelineDesc &desc, const char *tag) { @@ -1445,7 +1362,7 @@ void VKContext::BindTextures(int start, int count, Texture **textures, TextureBi boundTextures_[i] = static_cast(textures[i - start]); boundTextureFlags_[i] = flags; if (boundTextures_[i]) { - // If a texture is bound, we set these up in GetOrCreateDescriptorSet too. + // If a texture is bound, we set these up in BindDescriptors too. // But we might need to set the view here anyway so it can be queried using GetNativeObject. if (flags & TextureBindFlags::VULKAN_BIND_ARRAY) { boundImageView_[i] = boundTextures_[i]->GetImageArrayView(); @@ -1498,15 +1415,11 @@ void VKContext::Draw(int vertexCount, int offset) { uint32_t ubo_offset = (uint32_t)curPipeline_->PushUBO(push_, vulkan_, &vulkanUBObuf); size_t vbBindOffset = push_->Push(vbuf->GetData(), vbuf->GetSize(), 4, &vulkanVbuf); - VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf); - if (descSet == VK_NULL_HANDLE) { - ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__); - return; - } - BindCurrentPipeline(); ApplyDynamicState(); - renderManager_.Draw(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount, offset); + PackedDescriptor descriptors[4]; + BindDescriptors(vulkanUBObuf, descriptors); + renderManager_.Draw(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount, offset); } void VKContext::DrawIndexed(int vertexCount, int offset) { @@ -1518,15 +1431,11 @@ void VKContext::DrawIndexed(int vertexCount, int offset) { size_t vbBindOffset = push_->Push(vbuf->GetData(), vbuf->GetSize(), 4, &vulkanVbuf); size_t ibBindOffset = push_->Push(ibuf->GetData(), ibuf->GetSize(), 4, &vulkanIbuf); - VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf); - if (descSet == VK_NULL_HANDLE) { - ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__); - return; - } - BindCurrentPipeline(); ApplyDynamicState(); - renderManager_.DrawIndexed(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vulkanIbuf, (int)ibBindOffset + offset * sizeof(uint32_t), vertexCount, 1); + PackedDescriptor descriptors[4]; + BindDescriptors(vulkanUBObuf, descriptors); + renderManager_.DrawIndexed(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vulkanIbuf, (int)ibBindOffset + offset * sizeof(uint32_t), vertexCount, 1); } void VKContext::DrawUP(const void *vdata, int vertexCount) { @@ -1544,15 +1453,11 @@ void VKContext::DrawUP(const void *vdata, int vertexCount) { uint32_t ubo_offset = (uint32_t)curPipeline_->PushUBO(push_, vulkan_, &vulkanUBObuf); - VkDescriptorSet descSet = GetOrCreateDescriptorSet(vulkanUBObuf); - if (descSet == VK_NULL_HANDLE) { - ERROR_LOG(G3D, "GetOrCreateDescriptorSet failed, skipping %s", __FUNCTION__); - return; - } - BindCurrentPipeline(); ApplyDynamicState(); - renderManager_.Draw(descSet, 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount); + PackedDescriptor descriptors[4]; + BindDescriptors(vulkanUBObuf, descriptors); + renderManager_.Draw(descriptors, ARRAY_SIZE(descriptors), 1, &ubo_offset, vulkanVbuf, (int)vbBindOffset + curVBufferOffsets_[0], vertexCount); } void VKContext::BindCurrentPipeline() { diff --git a/Common/UI/Context.cpp b/Common/UI/Context.cpp index 31b59419ed..a660ece765 100644 --- a/Common/UI/Context.cpp +++ b/Common/UI/Context.cpp @@ -83,8 +83,6 @@ void UIContext::BeginPipeline(Draw::Pipeline *pipeline, Draw::SamplerState *samp // Also clear out any other textures bound. Draw::SamplerState *samplers[3]{ samplerState }; draw_->BindSamplerStates(0, 3, samplers); - Draw::Texture *textures[2]{}; - draw_->BindTextures(1, 2, textures); RebindTexture(); UIBegin(pipeline); }