From 5974e0ee6dd25bf048a8a5ecef5d5b5db4fe38da Mon Sep 17 00:00:00 2001
From: Henrik Rydgard <hrydgard@gmail.com>
Date: Sun, 7 May 2017 10:40:11 +0200
Subject: [PATCH] Work towards a clean vulkan shutdown.

---
 Common/Vulkan/VulkanContext.h       | 22 ++++++++++++++++++++++
 Windows/EmuThread.cpp               |  2 ++
 ext/native/thin3d/thin3d_vulkan.cpp | 12 ++++++------
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/Common/Vulkan/VulkanContext.h b/Common/Vulkan/VulkanContext.h
index d9a85f244e..ea93f2a0a9 100644
--- a/Common/Vulkan/VulkanContext.h
+++ b/Common/Vulkan/VulkanContext.h
@@ -66,6 +66,7 @@ class VulkanDeleteList {
 	};
 
 public:
+	void QueueDeleteCommandPool(VkCommandPool &pool) { cmdPools_.push_back(pool); pool = VK_NULL_HANDLE; }
 	void QueueDeleteDescriptorPool(VkDescriptorPool &pool) { descPools_.push_back(pool); pool = VK_NULL_HANDLE; }
 	void QueueDeleteShaderModule(VkShaderModule &module) { modules_.push_back(module); module = VK_NULL_HANDLE; }
 	void QueueDeleteBuffer(VkBuffer &buffer) { buffers_.push_back(buffer); buffer = VK_NULL_HANDLE; }
@@ -78,9 +79,12 @@ public:
 	void QueueDeletePipelineCache(VkPipelineCache &pipelineCache) { pipelineCaches_.push_back(pipelineCache); pipelineCache = VK_NULL_HANDLE; }
 	void QueueDeleteRenderPass(VkRenderPass &renderPass) { renderPasses_.push_back(renderPass); renderPass = VK_NULL_HANDLE; }
 	void QueueDeleteFramebuffer(VkFramebuffer &framebuffer) { framebuffers_.push_back(framebuffer); framebuffer = VK_NULL_HANDLE; }
+	void QueueDeletePipelineLayout(VkPipelineLayout &pipelineLayout) { pipelineLayouts_.push_back(pipelineLayout); pipelineLayout = VK_NULL_HANDLE; }
+	void QueueDeleteDescriptorSetLayout(VkDescriptorSetLayout &descSetLayout) { descSetLayouts_.push_back(descSetLayout); descSetLayout = VK_NULL_HANDLE; }
 	void QueueCallback(void(*func)(void *userdata), void *userdata) { callbacks_.push_back(Callback(func, userdata)); }
 
 	void Take(VulkanDeleteList &del) {
+		assert(cmdPools_.size() == 0);
 		assert(descPools_.size() == 0);
 		assert(modules_.size() == 0);
 		assert(buffers_.size() == 0);
@@ -94,6 +98,7 @@ public:
 		assert(renderPasses_.size() == 0);
 		assert(framebuffers_.size() == 0);
 		assert(callbacks_.size() == 0);
+		cmdPools_ = std::move(del.cmdPools_);
 		descPools_ = std::move(del.descPools_);
 		modules_ = std::move(del.modules_);
 		buffers_ = std::move(del.buffers_);
@@ -106,10 +111,16 @@ public:
 		pipelineCaches_ = std::move(del.pipelineCaches_);
 		renderPasses_ = std::move(del.renderPasses_);
 		framebuffers_ = std::move(del.framebuffers_);
+		pipelineLayouts_ = std::move(del.pipelineLayouts_);
+		descSetLayouts_ = std::move(del.descSetLayouts_);
 		callbacks_ = std::move(del.callbacks_);
 	}
 
 	void PerformDeletes(VkDevice device) {
+		for (auto &cmdPool : cmdPools_) {
+			vkDestroyCommandPool(device, cmdPool, nullptr);
+		}
+		cmdPools_.clear();
 		for (auto &descPool : descPools_) {
 			vkDestroyDescriptorPool(device, descPool, nullptr);
 		}
@@ -158,6 +169,14 @@ public:
 			vkDestroyFramebuffer(device, framebuffer, nullptr);
 		}
 		framebuffers_.clear();
+		for (auto &pipeLayout : pipelineLayouts_) {
+			vkDestroyPipelineLayout(device, pipeLayout, nullptr);
+		}
+		pipelineLayouts_.clear();
+		for (auto &descSetLayout : descSetLayouts_) {
+			vkDestroyDescriptorSetLayout(device, descSetLayout, nullptr);
+		}
+		descSetLayouts_.clear();
 		for (auto &callback : callbacks_) {
 			callback.func(callback.userdata);
 		}
@@ -165,6 +184,7 @@ public:
 	}
 
 private:
+	std::vector<VkCommandPool> cmdPools_;
 	std::vector<VkDescriptorPool> descPools_;
 	std::vector<VkShaderModule> modules_;
 	std::vector<VkBuffer> buffers_;
@@ -177,6 +197,8 @@ private:
 	std::vector<VkPipelineCache> pipelineCaches_;
 	std::vector<VkRenderPass> renderPasses_;
 	std::vector<VkFramebuffer> framebuffers_;
+	std::vector<VkPipelineLayout> pipelineLayouts_;
+	std::vector<VkDescriptorSetLayout> descSetLayouts_;
 	std::vector<Callback> callbacks_;
 };
 
diff --git a/Windows/EmuThread.cpp b/Windows/EmuThread.cpp
index 998f52f7b9..6ff1511caa 100644
--- a/Windows/EmuThread.cpp
+++ b/Windows/EmuThread.cpp
@@ -201,6 +201,8 @@ shutdown:
 	_InterlockedExchange(&emuThreadReady, THREAD_SHUTDOWN);
 
 	NativeShutdownGraphics();
+
+	// NativeShutdown deletes the graphics context through host->ShutdownGraphics().
 	NativeShutdown();
 	
 	_InterlockedExchange(&emuThreadReady, THREAD_END);
diff --git a/ext/native/thin3d/thin3d_vulkan.cpp b/ext/native/thin3d/thin3d_vulkan.cpp
index 237184a8ff..9522411135 100644
--- a/ext/native/thin3d/thin3d_vulkan.cpp
+++ b/ext/native/thin3d/thin3d_vulkan.cpp
@@ -555,7 +555,7 @@ public:
 		assert(VK_SUCCESS == res);
 	}
 	~VKSamplerState() {
-		vkDestroySampler(vulkan_->GetDevice(), sampler_, nullptr);
+		vulkan_->Delete().QueueDeleteSampler(sampler_);
 	}
 
 	VkSampler GetSampler() { return sampler_; }
@@ -716,17 +716,17 @@ VKContext::VKContext(VulkanContext *vulkan)
 }
 
 VKContext::~VKContext() {
-	vkDestroyCommandPool(device_, cmdPool_, nullptr);
+	vulkan_->Delete().QueueDeleteCommandPool(cmdPool_);
 	// This also destroys all descriptor sets.
 	for (int i = 0; i < 2; i++) {
 		frame_[i].descSets_.clear();
-		vkDestroyDescriptorPool(device_, frame_[i].descriptorPool, nullptr);
+		vulkan_->Delete().QueueDeleteDescriptorPool(frame_[i].descriptorPool);
 		frame_[i].pushBuffer->Destroy(vulkan_);
 		delete frame_[i].pushBuffer;
 	}
-	vkDestroyDescriptorSetLayout(device_, descriptorSetLayout_, nullptr);
-	vkDestroyPipelineLayout(device_, pipelineLayout_, nullptr);
-	vkDestroyPipelineCache(device_, pipelineCache_, nullptr);
+	vulkan_->Delete().QueueDeleteDescriptorSetLayout(descriptorSetLayout_);
+	vulkan_->Delete().QueueDeletePipelineLayout(pipelineLayout_);
+	vulkan_->Delete().QueueDeletePipelineCache(pipelineCache_);
 }
 
 void VKContext::Begin(bool clear, uint32_t colorval, float depthVal, int stencilVal) {