Merge branch 'master' into TemporaryHackishBranch

Common/ChunkFile.cpp

@@ -15,6 +15,7 @@
 // Official SVN repository and contact information can be found at
 // http://code.google.com/p/dolphin-emu/
 
+#include <cstdlib>
 #include <cstring>
 #include <snappy-c.h>
 
@@ -271,20 +272,32 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const std::string &filename,
 	INFO_LOG(SAVESTATE, "ChunkReader: Writing %s", filename.c_str());
 
 	File::IOFile pFile(filename, "wb");
-	if (!pFile)
-	{
+	if (!pFile) {
 		ERROR_LOG(SAVESTATE, "ChunkReader: Error opening file for write");
-		delete[] buffer;
+		free(buffer);
 		return ERROR_BAD_FILE;
 	}
 
-	bool compress = true;
+	// Make sure we can allocate a buffer to compress before compressing.
+	size_t write_len = snappy_max_compressed_length(sz);
+	u8 *compressed_buffer = (u8 *)malloc(write_len);
+	u8 *write_buffer = buffer;
+	if (!compressed_buffer) {
+		ERROR_LOG(SAVESTATE, "ChunkReader: Unable to allocate compressed buffer");
+		// We'll save uncompressed.  Better than not saving...
+		write_len = sz;
+	} else {
+		snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &write_len);
+		free(buffer);
+
+		write_buffer = compressed_buffer;
+	}
 
 	// Create header
 	SChunkHeader header;
-	header.Compress = compress ? 1 : 0;
+	header.Compress = compressed_buffer ? 1 : 0;
 	header.Revision = REVISION_CURRENT;
-	header.ExpectedSize = (u32)sz;
+	header.ExpectedSize = (u32)write_len;
 	header.UncompressedSize = (u32)sz;
 	truncate_cpy(header.GitVersion, gitVersion);
 
@@ -292,43 +305,26 @@ CChunkFileReader::Error CChunkFileReader::SaveFile(const std::string &filename,
 	char titleFixed[128];
 	truncate_cpy(titleFixed, title.c_str());
 
-	// Write to file
-	if (compress) {
-		size_t comp_len = snappy_max_compressed_length(sz);
-		u8 *compressed_buffer = new u8[comp_len];
-		snappy_compress((const char *)buffer, sz, (char *)compressed_buffer, &comp_len);
-		delete [] buffer;
-		header.ExpectedSize = (u32)comp_len;
-		if (!pFile.WriteArray(&header, 1)) {
-			ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing header");
-			return ERROR_BAD_FILE;
-		}
-		if (!pFile.WriteArray(titleFixed, sizeof(titleFixed))) {
-			ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing title");
-			return ERROR_BAD_FILE;
-		}
-		if (!pFile.WriteBytes(&compressed_buffer[0], comp_len)) {
-			ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing compressed data");
-			return ERROR_BAD_FILE;
-		}	else {
-			INFO_LOG(SAVESTATE, "Savestate: Compressed %i bytes into %i", (int)sz, (int)comp_len);
-		}
-		delete [] compressed_buffer;
-	} else {
-		if (!pFile.WriteArray(&header, 1))
-		{
-			ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing header");
-			delete[] buffer;
-			return ERROR_BAD_FILE;
-		}
-		if (!pFile.WriteBytes(&buffer[0], sz))
-		{
-			ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing data");
-			delete[] buffer;
-			return ERROR_BAD_FILE;
-		}
-		delete [] buffer;
+	// Now let's start writing out the file...
+	if (!pFile.WriteArray(&header, 1)) {
+		ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing header");
+		free(write_buffer);
+		return ERROR_BAD_FILE;
 	}
+	if (!pFile.WriteArray(titleFixed, sizeof(titleFixed))) {
+		ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing title");
+		free(write_buffer);
+		return ERROR_BAD_FILE;
+	}
+
+	if (!pFile.WriteBytes(write_buffer, write_len)) {
+		ERROR_LOG(SAVESTATE, "ChunkReader: Failed writing compressed data");
+		free(write_buffer);
+		return ERROR_BAD_FILE;
+	} else if (sz != write_len) {
+		INFO_LOG(SAVESTATE, "Savestate: Compressed %i bytes into %i", (int)sz, (int)write_len);
+	}
+	free(write_buffer);
 
 	INFO_LOG(SAVESTATE, "ChunkReader: Done writing %s", filename.c_str());
 	return ERROR_NONE;

Common/ChunkFile.h

@@ -29,6 +29,7 @@
 // + Sections can be versioned for backwards/forwards compatibility
 // - Serialization code for anything complex has to be manually written.
 
+#include <cstdlib>
 #include <map>
 #include <unordered_map>
 #include <deque>
@@ -646,17 +647,9 @@ public:
 	{
 		// Get data
 		size_t const sz = MeasurePtr(_class);
-		u8 *buffer = nullptr;
-#if PPSSPP_PLATFORM(ANDROID)
-		buffer = new u8[sz];
-#else
-		try {
-			buffer = new u8[sz];
-		}
-		catch (std::bad_alloc e) {
+		u8 *buffer = (u8 *)malloc(sz);
+		if (!buffer)
 			return ERROR_BAD_ALLOC;
-		}
-#endif
 		Error error = SavePtr(buffer, _class);
 
 		// SaveFile takes ownership of buffer

Common/MemoryUtil.cpp

@@ -284,9 +284,9 @@ void FreeAlignedMemory(void* ptr) {
 }
 
 bool PlatformIsWXExclusive() {
-	// Only iOS really needs this mode currently. Even without block linking, still should be much faster than IR JIT.
+	// Needed on platforms that disable W^X pages for security. Even without block linking, still should be much faster than IR JIT.
 	// This might also come in useful for UWP (Universal Windows Platform) if I'm understanding things correctly.
-#if defined(IOS) || PPSSPP_PLATFORM(UWP)
+#if defined(IOS) || PPSSPP_PLATFORM(UWP) || defined(__OpenBSD__)
 	return true;
 #else
 	// Returning true here lets you test the W^X path on Windows and other non-W^X platforms.

Common/Vulkan/VulkanContext.cpp

@@ -137,6 +137,7 @@ VkResult VulkanContext::CreateInstance(const char *app_name, int app_ver, uint32
 	if (res != VK_SUCCESS) {
 		init_error_ = "Failed to enumerate physical devices";
 		vkDestroyInstance(instance_, nullptr);
+		instance_ = nullptr;
 		return res;
 	}
 

Common/Vulkan/VulkanContext.h

@@ -378,7 +378,7 @@ private:
 
 	std::vector<VkDebugReportCallbackEXT> msg_callbacks;
 
-	VkSwapchainKHR swapchain_;
+	VkSwapchainKHR swapchain_ = VK_NULL_HANDLE;
 	VkFormat swapchainFormat_;
 
 	uint32_t queue_count = 0;

Common/Vulkan/VulkanImage.h

@@ -35,7 +35,6 @@ public:
 	void UploadMip(VkCommandBuffer cmd, int mip, int mipWidth, int mipHeight, VkBuffer buffer, uint32_t offset, size_t rowLength);  // rowLength is in pixels
 	void EndCreate(VkCommandBuffer cmd);
 
-	int GetNumMips() const { return numMips_; }
 	void Destroy();
 
 	// Used in image copies, etc.
@@ -46,6 +45,8 @@ public:
 
 	int32_t GetWidth() const { return tex_width; }
 	int32_t GetHeight() const { return tex_height; }
+	int32_t GetNumMips() const { return numMips_; }
+	VkFormat GetFormat() const { return format_; }
 
 private:
 	void CreateMappableImage();

GPU/Common/DepalettizeShaderCommon.cpp

@@ -288,4 +288,13 @@ void GenerateDepalShader(char *buffer, GEBufferFormat pixelFormat, ShaderLanguag
 	}
 }
 
+uint32_t DepalShaderCacheCommon::GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) const {
+	return (clutMode & 0xFFFFFF) | (pixelFormat << 24);
+}
+
+uint32_t DepalShaderCacheCommon::GetClutID(GEPaletteFormat clutFormat, uint32_t clutHash) const {
+	// Simplistic.
+	return clutHash ^ (uint32_t)clutFormat;
+}
+
 #undef WRITE

GPU/Common/DepalettizeShaderCommon.h

@@ -17,9 +17,19 @@
 
 #pragma once
 
+#include <cstdint>
 #include "GPU/ge_constants.h"
 #include "GPU/Common/ShaderCommon.h"
 
 static const int DEPAL_TEXTURE_OLD_AGE = 120;
 
 void GenerateDepalShader(char *buffer, GEBufferFormat pixelFormat, ShaderLanguage language);
+
+class DepalShaderCacheCommon {
+public:
+	virtual ~DepalShaderCacheCommon() {}
+
+protected:
+	uint32_t GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) const;
+	uint32_t GetClutID(GEPaletteFormat clutFormat, uint32_t clutHash) const;
+};

GPU/Common/FramebufferCommon.cpp

@@ -17,6 +17,8 @@
 
 #include <algorithm>
 #include <sstream>
+#include <cassert>
+#include <cmath>
 
 #include "ext/native/thin3d/thin3d.h"
 #include "base/timeutil.h"
@@ -199,7 +201,7 @@ bool FramebufferManagerCommon::ShouldDownloadFramebuffer(const VirtualFramebuffe
 
 void FramebufferManagerCommon::SetNumExtraFBOs(int num) {
 	for (size_t i = 0; i < extraFBOs_.size(); i++) {
-		delete extraFBOs_[i];
+		extraFBOs_[i]->Release();
 	}
 	extraFBOs_.clear();
 	for (int i = 0; i < num; i++) {
@@ -216,7 +218,7 @@ void FramebufferManagerCommon::EstimateDrawingSize(u32 fb_address, GEBufferForma
 	static const int MAX_FRAMEBUF_HEIGHT = 512;
 
 	// Games don't always set any of these.  Take the greatest parameter that looks valid based on stride.
-	if (viewport_width > 4 && viewport_width <= fb_stride) {
+	if (viewport_width > 4 && viewport_width <= fb_stride && viewport_height > 0) {
 		drawing_width = viewport_width;
 		drawing_height = viewport_height;
 		// Some games specify a viewport with 0.5, but don't have VRAM for 273.  480x272 is the buffer size.
@@ -303,8 +305,19 @@ void GetFramebufferHeuristicInputs(FramebufferHeuristicParams *params, const GPU
 	params->isModeThrough = gstate.isModeThrough();
 
 	// Viewport-X1 and Y1 are not the upper left corner, but half the width/height. A bit confusing.
-	params->viewportWidth = (int)(fabsf(gstate.getViewportXScale()*2.0f));
-	params->viewportHeight = (int)(fabsf(gstate.getViewportYScale()*2.0f));
+	float vpx = gstate.getViewportXScale();
+	float vpy = gstate.getViewportYScale();
+
+	// Work around problem in F1 Grand Prix, where it draws in through mode with a bogus viewport.
+	// We set bad values to 0 which causes the framebuffer size heuristic to rely on the other parameters instead.
+	if (std::isnan(vpx) || vpx > 10000000.0f) {
+		vpx = 0.f;
+	}
+	if (std::isnan(vpy) || vpy > 10000000.0f) {
+		vpy = 0.f;
+	}
+	params->viewportWidth = (int)(fabsf(vpx) * 2.0f);
+	params->viewportHeight = (int)(fabsf(vpy) * 2.0f);
 	params->regionWidth = gstate.getRegionX2() + 1;
 	params->regionHeight = gstate.getRegionY2() + 1;
 	params->scissorWidth = gstate.getScissorX2() + 1;
@@ -610,7 +623,7 @@ void FramebufferManagerCommon::NotifyRenderFramebufferSwitched(VirtualFramebuffe
 		if (vfb->fbo) {
 			// This should only happen very briefly when toggling useBufferedRendering_.
 			textureCache_->NotifyFramebuffer(vfb->fb_address, vfb, NOTIFY_FB_DESTROYED);
-			delete vfb->fbo;
+			vfb->fbo->Release();
 			vfb->fbo = nullptr;
 		}
 
@@ -828,7 +841,7 @@ void FramebufferManagerCommon::SetViewport2D(int x, int y, int w, int h) {
 }
 
 void FramebufferManagerCommon::CopyDisplayToOutput() {
-	// DownloadFramebufferOnSwitch(currentRenderVfb_);
+	DownloadFramebufferOnSwitch(currentRenderVfb_);
 
 	currentRenderVfb_ = 0;
 
@@ -1063,7 +1076,7 @@ void FramebufferManagerCommon::DecimateFBOs() {
 	currentRenderVfb_ = 0;
 
 	for (auto iter : fbosToDelete_) {
-		delete iter;
+		iter->Release();
 	}
 	fbosToDelete_.clear();
 
@@ -1111,7 +1124,9 @@ void FramebufferManagerCommon::DecimateFBOs() {
 	}
 }
 
-void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, u16 w, u16 h, bool force, bool skipCopy) {
+void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, int w, int h, bool force, bool skipCopy) {
+	assert(w > 0);
+	assert(h > 0);
 	VirtualFramebuffer old = *vfb;
 
 	int oldWidth = vfb->bufferWidth;
@@ -1126,8 +1141,8 @@ void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, u16 w,
 		}
 
 		// In case it gets thin and wide, don't resize down either side.
-		vfb->bufferWidth = std::max(vfb->bufferWidth, w);
-		vfb->bufferHeight = std::max(vfb->bufferHeight, h);
+		vfb->bufferWidth = std::max((int)vfb->bufferWidth, w);
+		vfb->bufferHeight = std::max((int)vfb->bufferHeight, h);
 	}
 
 	SetRenderSize(vfb);
@@ -1161,7 +1176,7 @@ void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, u16 w,
 
 	if (!useBufferedRendering_) {
 		if (vfb->fbo) {
-			delete vfb->fbo;
+			vfb->fbo->Release();
 			vfb->fbo = nullptr;
 		}
 		return;
@@ -1917,6 +1932,9 @@ bool FramebufferManagerCommon::GetFramebuffer(u32 fb_address, int fb_stride, GEB
 	bool flipY = (g_Config.iGPUBackend == GPU_BACKEND_OPENGL && !useBufferedRendering_) ? true : false;
 	buffer.Allocate(w, h, GE_FORMAT_8888, flipY, true);
 	bool retval = draw_->CopyFramebufferToMemorySync(bound, Draw::FB_COLOR_BIT, 0, 0, w, h, Draw::DataFormat::R8G8B8A8_UNORM, buffer.GetData(), w);
+	// Vulkan requires us to re-apply all dynamic state for each command buffer, and the above will cause us to start a new cmdbuf.
+	// So let's dirty the things that are involved in Vulkan dynamic state. Readbacks are not frequent so this won't hurt other backends.
+	gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE);
 	// We may have blitted to a temp FBO.
 	RebindFramebuffer();
 	return retval;
@@ -2019,9 +2037,9 @@ void FramebufferManagerCommon::PackFramebufferSync_(VirtualFramebuffer *vfb, int
 }
 
 void FramebufferManagerCommon::ReadFramebufferToMemory(VirtualFramebuffer *vfb, bool sync, int x, int y, int w, int h) {
-	// Clamp to width. Sometimes block transfers can cause this to hit.
-	if (x + w >= vfb->width) {
-		w = vfb->width - x;
+	// Clamp to bufferWidth. Sometimes block transfers can cause this to hit.
+	if (x + w >= vfb->bufferWidth) {
+		w = vfb->bufferWidth - x;
 	}
 	if (vfb) {
 		// We'll pseudo-blit framebuffers here to get a resized version of vfb.

GPU/Common/FramebufferCommon.h

@@ -338,7 +338,7 @@ protected:
 	virtual void ReformatFramebufferFrom(VirtualFramebuffer *vfb, GEBufferFormat old) = 0;
 	virtual void BlitFramebufferDepth(VirtualFramebuffer *src, VirtualFramebuffer *dst) = 0;
 
-	void ResizeFramebufFBO(VirtualFramebuffer *vfb, u16 w, u16 h, bool force = false, bool skipCopy = false);
+	void ResizeFramebufFBO(VirtualFramebuffer *vfb, int w, int h, bool force = false, bool skipCopy = false);
 	void ShowScreenResolution();
 
 	bool ShouldDownloadFramebuffer(const VirtualFramebuffer *vfb) const;

GPU/Common/TextureCacheCommon.cpp

@@ -637,6 +637,7 @@ bool TextureCacheCommon::AttachFramebuffer(TexCacheEntry *entry, u32 address, Vi
 	const u32 texaddr = ((entry->addr + texaddrOffset) & ~mirrorMask);
 	const bool noOffset = texaddr == addr;
 	const bool exactMatch = noOffset && entry->format < 4;
+	const u32 w = 1 << ((entry->dim >> 0) & 0xf);
 	const u32 h = 1 << ((entry->dim >> 8) & 0xf);
 	// 512 on a 272 framebuffer is sane, so let's be lenient.
 	const u32 minSubareaHeight = h / 4;
@@ -686,6 +687,13 @@ bool TextureCacheCommon::AttachFramebuffer(TexCacheEntry *entry, u32 address, Vi
 			}
 		}
 
+		// Check if it's in bufferWidth (which might be higher than width and may indicate the framebuffer includes the data.)
+		if (fbInfo.xOffset >= framebuffer->bufferWidth && fbInfo.xOffset + w <= (u32)framebuffer->fb_stride) {
+			// This happens in Brave Story, see #10045 - the texture is in the space between strides, with matching stride.
+			DetachFramebuffer(entry, address, framebuffer);
+			return false;
+		}
+
 		if (fbInfo.yOffset + minSubareaHeight >= framebuffer->height) {
 			// Can't be inside the framebuffer then, ram.  Detach to be safe.
 			DetachFramebuffer(entry, address, framebuffer);
@@ -764,7 +772,7 @@ void TextureCacheCommon::SetTextureFramebuffer(TexCacheEntry *entry, VirtualFram
 		nextTexture_ = entry;
 	} else {
 		if (framebuffer->fbo) {
-			delete framebuffer->fbo;
+			framebuffer->fbo->Release();
 			framebuffer->fbo = nullptr;
 		}
 		Unbind();

GPU/D3D11/DepalettizeShaderD3D11.cpp

@@ -70,14 +70,10 @@ DepalShaderCacheD3D11::~DepalShaderCacheD3D11() {
 	inputLayout_->Release();
 }
 
-u32 DepalShaderCacheD3D11::GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) {
-	return (clutMode & 0xFFFFFF) | (pixelFormat << 24);
-}
+ID3D11ShaderResourceView *DepalShaderCacheD3D11::GetClutTexture(GEPaletteFormat clutFormat, const u32 clutHash, u32 *rawClut, bool expandTo32bit) {
+	const u32 clutId = GetClutID(clutFormat, clutHash);
 
-ID3D11ShaderResourceView *DepalShaderCacheD3D11::GetClutTexture(GEPaletteFormat clutFormat, const u32 clutID, u32 *rawClut, bool expandTo32bit) {
-	const u32 realClutID = clutID ^ clutFormat;
-
-	auto oldtex = texCache_.find(realClutID);
+	auto oldtex = texCache_.find(clutId);
 	if (oldtex != texCache_.end()) {
 		oldtex->second->lastFrame = gpuStats.numFlips;
 		return oldtex->second->view;
@@ -128,7 +124,7 @@ ID3D11ShaderResourceView *DepalShaderCacheD3D11::GetClutTexture(GEPaletteFormat
 	ASSERT_SUCCESS(device_->CreateTexture2D(&desc, &data, &tex->texture));
 	ASSERT_SUCCESS(device_->CreateShaderResourceView(tex->texture, nullptr, &tex->view));
 	tex->lastFrame = gpuStats.numFlips;
-	texCache_[realClutID] = tex;
+	texCache_[clutId] = tex;
 
 	if (expandTo32bit) {
 		delete[] expanded;

GPU/D3D11/DepalettizeShaderD3D11.h

@@ -23,6 +23,7 @@
 #include "Common/CommonTypes.h"
 #include "GPU/ge_constants.h"
 #include "thin3d/thin3d.h"
+#include "GPU/Common/DepalettizeShaderCommon.h"
 
 class DepalShaderD3D11 {
 public:
@@ -44,7 +45,7 @@ public:
 };
 
 // Caches both shaders and palette textures.
-class DepalShaderCacheD3D11 {
+class DepalShaderCacheD3D11 : public DepalShaderCacheCommon {
 public:
 	DepalShaderCacheD3D11(Draw::DrawContext *draw);
 	~DepalShaderCacheD3D11();
@@ -60,8 +61,6 @@ public:
 	std::string DebugGetShaderString(std::string id, DebugShaderType type, DebugShaderStringType stringType);
 
 private:
-	u32 GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat);
-
 	ID3D11Device *device_;
 	ID3D11DeviceContext *context_;
 	D3D_FEATURE_LEVEL featureLevel_;

GPU/D3D11/FramebufferManagerD3D11.cpp

@@ -738,7 +738,7 @@ void FramebufferManagerD3D11::DestroyAllFBOs() {
 	bvfbs_.clear();
 
 	for (auto it = tempFBOs_.begin(), end = tempFBOs_.end(); it != end; ++it) {
-		delete it->second.fbo;
+		it->second.fbo->Release();
 	}
 	tempFBOs_.clear();
 

GPU/D3D11/StencilBufferD3D11.cpp

@@ -28,9 +28,6 @@
 #include "GPU/D3D11/TextureCacheD3D11.h"
 #include "GPU/D3D11/D3D11Util.h"
 
-#define STR_HELPER(x) #x
-#define STR(x) STR_HELPER(x)
-
 struct StencilValueUB {
 	uint32_t u_stencilValue[4];
 };

GPU/D3D11/TextureCacheD3D11.cpp

@@ -821,6 +821,20 @@ bool TextureCacheD3D11::GetCurrentTextureDebug(GPUDebugBuffer &buffer, int level
 	if (!nextTexture_)
 		return false;
 
+	// TODO: Centralize.
+	if (nextTexture_->framebuffer) {
+		VirtualFramebuffer *vfb = nextTexture_->framebuffer;
+		bool flipY = g_Config.iGPUBackend == GPU_BACKEND_OPENGL && g_Config.iRenderingMode != FB_NON_BUFFERED_MODE;
+		buffer.Allocate(vfb->bufferWidth, vfb->bufferHeight, GPU_DBG_FORMAT_8888, flipY);
+		bool retval = draw_->CopyFramebufferToMemorySync(vfb->fbo, Draw::FB_COLOR_BIT, 0, 0, vfb->bufferWidth, vfb->bufferHeight, Draw::DataFormat::R8G8B8A8_UNORM, buffer.GetData(), vfb->bufferWidth);
+		// Vulkan requires us to re-apply all dynamic state for each command buffer, and the above will cause us to start a new cmdbuf.
+		// So let's dirty the things that are involved in Vulkan dynamic state. Readbacks are not frequent so this won't hurt other backends.
+		gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE);
+		// We may have blitted to a temp FBO.
+		framebufferManager_->RebindFramebuffer();
+		return retval;
+	}
+
 	ID3D11Texture2D *texture = (ID3D11Texture2D *)nextTexture_->texturePtr;
 	if (!texture)
 		return false;

GPU/Directx9/DepalettizeShaderDX9.cpp

@@ -67,14 +67,10 @@ DepalShaderCacheDX9::~DepalShaderCacheDX9() {
 	}
 }
 
-u32 DepalShaderCacheDX9::GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) {
-	return (clutMode & 0xFFFFFF) | (pixelFormat << 24);
-}
+LPDIRECT3DTEXTURE9 DepalShaderCacheDX9::GetClutTexture(GEPaletteFormat clutFormat, u32 clutHash, u32 *rawClut) {
+	u32 clutId = GetClutID(clutFormat, clutHash);
 
-LPDIRECT3DTEXTURE9 DepalShaderCacheDX9::GetClutTexture(GEPaletteFormat clutFormat, const u32 clutID, u32 *rawClut) {
-	const u32 realClutID = clutID ^ clutFormat;
-
-	auto oldtex = texCache_.find(realClutID);
+	auto oldtex = texCache_.find(clutId);
 	if (oldtex != texCache_.end()) {
 		oldtex->second->lastFrame = gpuStats.numFlips;
 		return oldtex->second->texture;
@@ -117,7 +113,7 @@ LPDIRECT3DTEXTURE9 DepalShaderCacheDX9::GetClutTexture(GEPaletteFormat clutForma
 	device_->SetSamplerState(1, D3DSAMP_MAGFILTER, D3DTEXF_POINT);
 
 	tex->lastFrame = gpuStats.numFlips;
-	texCache_[realClutID] = tex;
+	texCache_[clutId] = tex;
 	return tex->texture;
 }
 

GPU/Directx9/DepalettizeShaderDX9.h

@@ -21,6 +21,7 @@
 #include "Common/CommonTypes.h"
 #include "GPU/ge_constants.h"
 #include "GPU/Common/ShaderCommon.h"
+#include "GPU/Common/DepalettizeShaderCommon.h"
 
 namespace DX9 {
 
@@ -37,7 +38,7 @@ public:
 };
 
 // Caches both shaders and palette textures.
-class DepalShaderCacheDX9 {
+class DepalShaderCacheDX9 : public DepalShaderCacheCommon {
 public:
 	DepalShaderCacheDX9(Draw::DrawContext *draw);
 	~DepalShaderCacheDX9();
@@ -45,15 +46,13 @@ public:
 	// This also uploads the palette and binds the correct texture.
 	LPDIRECT3DPIXELSHADER9 GetDepalettizePixelShader(uint32_t clutMode, GEBufferFormat pixelFormat);
 	LPDIRECT3DVERTEXSHADER9 GetDepalettizeVertexShader() { return vertexShader_; }
-	LPDIRECT3DTEXTURE9 GetClutTexture(GEPaletteFormat clutFormat, const u32 clutHash, u32 *rawClut);
+	LPDIRECT3DTEXTURE9 GetClutTexture(GEPaletteFormat clutFormat, u32 clutHash, u32 *rawClut);
 	void Clear();
 	void Decimate();
 	std::vector<std::string> DebugGetShaderIDs(DebugShaderType type);
 	std::string DebugGetShaderString(std::string id, DebugShaderType type, DebugShaderStringType stringType);
 
 private:
-	u32 GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat);
-
 	LPDIRECT3DDEVICE9 device_;
 	LPDIRECT3DVERTEXSHADER9 vertexShader_;
 	std::map<u32, DepalShaderDX9 *> cache_;

GPU/Directx9/FramebufferDX9.cpp

@@ -131,7 +131,7 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 			drawPixelsTex_->Release();
 		}
 		for (auto it = tempFBOs_.begin(), end = tempFBOs_.end(); it != end; ++it) {
-			delete it->second.fbo;
+			it->second.fbo->Release();
 		}
 		for (auto it = offscreenSurfaces_.begin(), end = offscreenSurfaces_.end(); it != end; ++it) {
 			it->second.surface->Release();
@@ -726,7 +726,7 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 		bvfbs_.clear();
 
 		for (auto it = tempFBOs_.begin(), end = tempFBOs_.end(); it != end; ++it) {
-			delete it->second.fbo;
+			it->second.fbo->Release();
 		}
 		tempFBOs_.clear();
 
@@ -777,7 +777,7 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 				success = GetRenderTargetFramebuffer(renderTarget, offscreen, w, h, buffer);
 			}
 			if (tempFBO) {
-				delete tempFBO;
+				tempFBO->Release();
 			}
 		}
 

GPU/GLES/DepalettizeShaderGLES.cpp

@@ -120,14 +120,10 @@ bool DepalShaderCacheGLES::CreateVertexShader() {
 	return !vertexShaderFailed_;
 }
 
-u32 DepalShaderCacheGLES::GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) {
-	return (clutMode & 0xFFFFFF) | (pixelFormat << 24);
-}
+GLuint DepalShaderCacheGLES::GetClutTexture(GEPaletteFormat clutFormat, const u32 clutHash, u32 *rawClut) {
+	u32 clutId = GetClutID(clutFormat, clutHash);
 
-GLuint DepalShaderCacheGLES::GetClutTexture(GEPaletteFormat clutFormat, const u32 clutID, u32 *rawClut) {
-	const u32 realClutID = clutID ^ clutFormat;
-
-	auto oldtex = texCache_.find(realClutID);
+	auto oldtex = texCache_.find(clutId);
 	if (oldtex != texCache_.end()) {
 		oldtex->second->lastFrame = gpuStats.numFlips;
 		return oldtex->second->texture;
@@ -156,7 +152,7 @@ GLuint DepalShaderCacheGLES::GetClutTexture(GEPaletteFormat clutFormat, const u3
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 
 	tex->lastFrame = gpuStats.numFlips;
-	texCache_[realClutID] = tex;
+	texCache_[clutId] = tex;
 	return tex->texture;
 }
 

GPU/GLES/DepalettizeShaderGLES.h

@@ -21,6 +21,7 @@
 #include "gfx/gl_common.h"
 #include "GPU/ge_constants.h"
 #include "GPU/Common/ShaderCommon.h"
+#include "GPU/Common/DepalettizeShaderCommon.h"
 
 class DepalShader {
 public:
@@ -38,7 +39,7 @@ public:
 };
 
 // Caches both shaders and palette textures.
-class DepalShaderCacheGLES {
+class DepalShaderCacheGLES : public DepalShaderCacheCommon {
 public:
 	DepalShaderCacheGLES();
 	~DepalShaderCacheGLES();
@@ -52,7 +53,6 @@ public:
 	std::string DebugGetShaderString(std::string id, DebugShaderType type, DebugShaderStringType stringType);
 
 private:
-	u32 GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat);
 	bool CreateVertexShader();
 
 	bool useGL3_;

GPU/GLES/FramebufferManagerGLES.cpp

@@ -254,7 +254,7 @@ FramebufferManagerGLES::~FramebufferManagerGLES() {
 	SetNumExtraFBOs(0);
 
 	for (auto it = tempFBOs_.begin(), end = tempFBOs_.end(); it != end; ++it) {
-		delete it->second.fbo;
+		it->second.fbo->Release();
 	}
 
 	delete [] pixelBufObj_;
@@ -1075,7 +1075,7 @@ void FramebufferManagerGLES::DestroyAllFBOs() {
 	bvfbs_.clear();
 
 	for (auto it = tempFBOs_.begin(), end = tempFBOs_.end(); it != end; ++it) {
-		delete it->second.fbo;
+		it->second.fbo->Release();
 	}
 	tempFBOs_.clear();
 

GPU/GPU.vcxproj

@@ -360,7 +360,6 @@
     <ClCompile Include="Software\SoftGpu.cpp" />
     <ClCompile Include="Software\TransformUnit.cpp" />
     <ClCompile Include="Common\TextureDecoder.cpp" />
-    <ClCompile Include="Source.cpp" />
     <ClCompile Include="Vulkan\DepalettizeShaderVulkan.cpp" />
     <ClCompile Include="Vulkan\DrawEngineVulkan.cpp" />
     <ClCompile Include="Vulkan\FragmentShaderGeneratorVulkan.cpp" />
@@ -386,4 +385,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

GPU/GPU.vcxproj.filters

@@ -255,7 +255,6 @@
     <ClInclude Include="D3D11\D3D11Util.h">
       <Filter>D3D11</Filter>
     </ClInclude>
-    <ClInclude Include="D3D11\StateMappingD3D11.h" />
     <ClInclude Include="Common\ShaderTranslation.h">
       <Filter>Common</Filter>
     </ClInclude>
@@ -268,6 +267,9 @@
     <ClInclude Include="Common\StencilCommon.h">
       <Filter>Common</Filter>
     </ClInclude>
+    <ClInclude Include="D3D11\StateMappingD3D11.h">
+      <Filter>D3D11</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="Math3D.cpp">
@@ -531,11 +533,8 @@
     <ClCompile Include="Vulkan\StencilBufferVulkan.cpp">
       <Filter>Vulkan</Filter>
     </ClCompile>
-    <ClCompile Include="Source.cpp">
-      <Filter>Common</Filter>
-    </ClCompile>
     <ClCompile Include="Common\StencilCommon.cpp">
       <Filter>Common</Filter>
     </ClCompile>
   </ItemGroup>
-</Project>
+</Project>

GPU/Vulkan/DepalettizeShaderVulkan.cpp

@@ -103,11 +103,9 @@ DepalShaderVulkan *DepalShaderCacheVulkan::GetDepalettizeShader(uint32_t clutMod
 	return depal;
 }
 
-VulkanTexture *DepalShaderCacheVulkan::GetClutTexture(GEPaletteFormat clutFormat, u32 clutID, u32 *rawClut) {
-	VkCommandBuffer cmd = (VkCommandBuffer)draw_->GetNativeObject(Draw::NativeObject::INIT_COMMANDBUFFER);
-	const u32 realClutID = clutID ^ clutFormat;
-
-	auto oldtex = texCache_.find(realClutID);
+VulkanTexture *DepalShaderCacheVulkan::GetClutTexture(GEPaletteFormat clutFormat, u32 clutHash, u32 *rawClut) {
+	u32 clutId = GetClutID(clutFormat, clutHash);
+	auto oldtex = texCache_.find(clutId);
 	if (oldtex != texCache_.end()) {
 		oldtex->second->lastFrame = gpuStats.numFlips;
 		return oldtex->second->texture;
@@ -121,6 +119,7 @@ VulkanTexture *DepalShaderCacheVulkan::GetClutTexture(GEPaletteFormat clutFormat
 	uint32_t pushOffset = push_->PushAligned(rawClut, 1024, 4, &pushBuffer);
 
 	VulkanTexture *vktex = new VulkanTexture(vulkan_, alloc_);
+	VkCommandBuffer cmd = (VkCommandBuffer)draw_->GetNativeObject(Draw::NativeObject::INIT_COMMANDBUFFER);
 	if (!vktex->CreateDirect(cmd, texturePixels, 1, 1, destFormat,
 		VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, &componentMapping)) {
 		Crash();
@@ -131,7 +130,7 @@ VulkanTexture *DepalShaderCacheVulkan::GetClutTexture(GEPaletteFormat clutFormat
 	DepalTextureVulkan *tex = new DepalTextureVulkan();
 	tex->texture = vktex;
 	tex->lastFrame = gpuStats.numFlips;
-	texCache_[realClutID] = tex;
+	texCache_[clutId] = tex;
 	return tex->texture;
 }
 
@@ -161,7 +160,3 @@ void DepalShaderCacheVulkan::Decimate() {
 		}
 	}
 }
-
-u32 DepalShaderCacheVulkan::GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat) {
-	return (clutMode & 0xFFFFFF) | (pixelFormat << 24);
-}

GPU/Vulkan/DepalettizeShaderVulkan.h

@@ -25,6 +25,7 @@
 #include "Common/Vulkan/VulkanMemory.h"
 #include "GPU/ge_constants.h"
 #include "thin3d/thin3d.h"
+#include "GPU/Common/DepalettizeShaderCommon.h"
 
 class DepalShaderVulkan {
 public:
@@ -33,7 +34,7 @@ public:
 	}
 	// A Vulkan2D pipeline. Set texture to slot 0 and palette texture to slot 1.
 	VkPipeline pipeline = VK_NULL_HANDLE;
-	const char *code = nullptr;;
+	const char *code = nullptr;
 };
 
 class DepalTextureVulkan {
@@ -47,7 +48,7 @@ class Vulkan2D;
 
 // Caches both shaders and palette textures.
 // Could even avoid bothering with palette texture and just use uniform data...
-class DepalShaderCacheVulkan {
+class DepalShaderCacheVulkan : public DepalShaderCacheCommon {
 public:
 	DepalShaderCacheVulkan(Draw::DrawContext *draw, VulkanContext *vulkan);
 	~DepalShaderCacheVulkan();
@@ -64,8 +65,6 @@ public:
 	void SetVShader(VkShaderModule vshader) { vshader_ = vshader; }
 
 private:
-	u32 GenerateShaderID(uint32_t clutMode, GEBufferFormat pixelFormat);
-
 	Draw::DrawContext *draw_ = nullptr;
 	VulkanContext *vulkan_ = nullptr;
 	VulkanPushBuffer *push_ = nullptr;

GPU/Vulkan/GPU_Vulkan.cpp

@@ -173,7 +173,6 @@ GPU_Vulkan::~GPU_Vulkan() {
 
 void GPU_Vulkan::CheckGPUFeatures() {
 	uint32_t features = 0;
-	features |= GPU_SUPPORTS_16BIT_FORMATS;
 	if (vulkan_->GetFeaturesEnabled().wideLines) {
 		features |= GPU_SUPPORTS_WIDE_LINES;
 	}

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -19,8 +19,6 @@
 //#define SHADERLOG
 #endif
 
-#include <map>
-
 #include "base/logging.h"
 #include "base/stringutil.h"
 #include "math/lin/matrix4x4.h"

GPU/Vulkan/ShaderManagerVulkan.h

@@ -17,8 +17,6 @@
 
 #pragma once
 
-#include <map>
-
 #include "base/basictypes.h"
 #include "Common/Hashmaps.h"
 #include "Common/Vulkan/VulkanMemory.h"

GPU/Vulkan/StateMappingVulkan.cpp

@@ -152,8 +152,13 @@ void DrawEngineVulkan::ConvertStateToVulkanKey(FramebufferManagerVulkan &fbManag
 			bool alphaMask = gstate.isClearModeAlphaMask();
 			key.colorWriteMask = (colorMask ? (VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT) : 0) | (alphaMask ? VK_COLOR_COMPONENT_A_BIT : 0);
 		} else {
-			key.logicOpEnable = false;
-			key.logicOp = VK_LOGIC_OP_CLEAR;
+			if (gstate_c.Supports(GPU_SUPPORTS_LOGIC_OP) && gstate.isLogicOpEnabled() && gstate.getLogicOp() != GE_LOGIC_COPY) {
+				key.logicOpEnable = true;
+				key.logicOp = logicOps[gstate.getLogicOp()];
+			} else {
+				key.logicOpEnable = false;
+				key.logicOp = VK_LOGIC_OP_CLEAR;
+			}
 
 			// Set blend - unless we need to do it in the shader.
 			GenericBlendState blendState;
@@ -275,16 +280,6 @@ void DrawEngineVulkan::ConvertStateToVulkanKey(FramebufferManagerVulkan &fbManag
 				dynState.useStencil = false;
 			}
 		} else {
-			if (gstate_c.Supports(GPU_SUPPORTS_LOGIC_OP)) {
-				// Logic Ops
-				if (gstate.isLogicOpEnabled() && gstate.getLogicOp() != GE_LOGIC_COPY) {
-					key.logicOpEnable = true;
-					key.logicOp = logicOps[gstate.getLogicOp()];
-				} else {
-					key.logicOpEnable = false;
-				}
-			}
-
 			// Depth Test
 			if (gstate.isDepthTestEnabled()) {
 				key.depthTestEnable = true;

GPU/Vulkan/StencilBufferVulkan.cpp

@@ -27,29 +27,22 @@
 #include "GPU/Vulkan/TextureCacheVulkan.h"
 #include "GPU/Vulkan/VulkanUtil.h"
 
-#define STR_HELPER(x) #x
-#define STR(x) STR_HELPER(x)
-
-struct StencilValueUB {
-	uint32_t u_stencilValue[4];
-};
-
 static const char *stencil_fs = R"(#version 400
 #extension GL_ARB_separate_shader_objects : enable
 #extension GL_ARB_shading_language_420pack : enable
 layout (binding = 0) uniform sampler2D tex;
 layout(push_constant) uniform params {
-  int u_stencilValue;
+	int u_stencilValue;
 };
 layout (location = 0) in vec2 v_texcoord0;
 layout (location = 0) out vec4 fragColor0;
 
 void main() {
-  vec4 index = texture(tex, v_texcoord0);
+	vec4 index = texture(tex, v_texcoord0);
 	int indexBits = int(floor(index.a * 255.99)) & 0xFF;
 	if ((indexBits & u_stencilValue) == 0)
 		discard;
-  fragColor0 = index.aaaa;
+	fragColor0 = index.aaaa;
 }
 )";
 
@@ -60,9 +53,9 @@ layout (location = 0) out vec2 v_texcoord0;
 out gl_PerVertex { vec4 gl_Position; };
 void main() {
 	int id = gl_VertexIndex;
-  v_texcoord0.x = (id == 2) ? 2.0 : 0.0;
-  v_texcoord0.y = (id == 1) ? 2.0 : 0.0;
-  gl_Position = vec4(v_texcoord0 * vec2(2.0, 2.0) + vec2(-1.0, -1.0), 0.0, 1.0);
+	v_texcoord0.x = (id == 2) ? 2.0 : 0.0;
+	v_texcoord0.y = (id == 1) ? 2.0 : 0.0;
+	gl_Position = vec4(v_texcoord0 * vec2(2.0, 2.0) + vec2(-1.0, -1.0), 0.0, 1.0);
 }
 )";
 

GPU/Vulkan/TextureCacheVulkan.cpp

@@ -367,12 +367,13 @@ void TextureCacheVulkan::Unbind() {
 
 void TextureCacheVulkan::ApplyTextureFramebuffer(TexCacheEntry *entry, VirtualFramebuffer *framebuffer) {
 	DepalShaderVulkan *depalShader = nullptr;
-	const GEPaletteFormat clutFormat = gstate.getClutPaletteFormat();
+	uint32_t clutMode = gstate.clutformat & 0xFFFFFF;
 	if ((entry->status & TexCacheEntry::STATUS_DEPALETTIZE) && !g_Config.bDisableSlowFramebufEffects) {
-		depalShader = depalShaderCache_->GetDepalettizeShader(clutFormat, framebuffer->drawnFormat);
+		depalShader = depalShaderCache_->GetDepalettizeShader(clutMode, framebuffer->drawnFormat);
 	}
 	if (depalShader) {
 		depalShaderCache_->SetPushBuffer(drawEngine_->GetPushBufferForTextureData());
+		const GEPaletteFormat clutFormat = gstate.getClutPaletteFormat();
 		VulkanTexture *clutTexture = depalShaderCache_->GetClutTexture(clutFormat, clutHash_, clutBuf_);
 
 		Draw::Framebuffer *depalFBO = framebufferManager_->GetTempFBO(
@@ -455,6 +456,7 @@ void TextureCacheVulkan::ApplyTextureFramebuffer(TexCacheEntry *entry, VirtualFr
 
 		// Need to rebind the pipeline since we switched it.
 		drawEngine_->DirtyPipeline();
+		gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE);
 	} else {
 		entry->status &= ~TexCacheEntry::STATUS_DEPALETTIZE;
 
@@ -812,6 +814,62 @@ void TextureCacheVulkan::LoadTextureLevel(TexCacheEntry &entry, uint8_t *writePt
 }
 
 bool TextureCacheVulkan::GetCurrentTextureDebug(GPUDebugBuffer &buffer, int level) {
-	// TODO
-	return false;
+	ApplyTexture();
+	SetTexture(false);
+	if (!nextTexture_)
+		return false;
+
+	// TODO: Centralize?
+	if (nextTexture_->framebuffer) {
+		VirtualFramebuffer *vfb = nextTexture_->framebuffer;
+		bool flipY = g_Config.iGPUBackend == GPU_BACKEND_OPENGL && g_Config.iRenderingMode != FB_NON_BUFFERED_MODE;
+		buffer.Allocate(vfb->bufferWidth, vfb->bufferHeight, GPU_DBG_FORMAT_8888, flipY);
+		bool retval = draw_->CopyFramebufferToMemorySync(vfb->fbo, Draw::FB_COLOR_BIT, 0, 0, vfb->bufferWidth, vfb->bufferHeight, Draw::DataFormat::R8G8B8A8_UNORM, buffer.GetData(), vfb->bufferWidth);
+		// Vulkan requires us to re-apply all dynamic state for each command buffer, and the above will cause us to start a new cmdbuf.
+		// So let's dirty the things that are involved in Vulkan dynamic state. Readbacks are not frequent so this won't hurt other backends.
+		gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE);
+		// We may have blitted to a temp FBO.
+		framebufferManager_->RebindFramebuffer();
+		return retval;
+	}
+
+	if (!nextTexture_->vkTex || !nextTexture_->vkTex->texture_)
+		return false;
+	VulkanTexture *texture = nextTexture_->vkTex->texture_;
+	VulkanRenderManager *renderManager = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
+
+	GPUDebugBufferFormat bufferFormat;
+	Draw::DataFormat drawFormat;
+	switch (texture->GetFormat()) {
+	case VULKAN_565_FORMAT:
+		bufferFormat = GPU_DBG_FORMAT_565;
+		drawFormat = Draw::DataFormat::B5G6R5_UNORM_PACK16;
+		break;
+	case VULKAN_1555_FORMAT:
+		bufferFormat = GPU_DBG_FORMAT_5551;
+		drawFormat = Draw::DataFormat::B5G5R5A1_UNORM_PACK16;
+		break;
+	case VULKAN_4444_FORMAT:
+		bufferFormat = GPU_DBG_FORMAT_4444;
+		drawFormat = Draw::DataFormat::B4G4R4A4_UNORM_PACK16;
+		break;
+	case VULKAN_8888_FORMAT:
+	default:
+		bufferFormat = GPU_DBG_FORMAT_8888;
+		drawFormat = Draw::DataFormat::R8G8B8A8_UNORM;
+		break;
+	}
+
+	int w = texture->GetWidth();
+	int h = texture->GetHeight();
+	buffer.Allocate(w, h, bufferFormat);
+
+	renderManager->CopyImageToMemorySync(texture->GetImage(), level, 0, 0, w, h, drawFormat, (uint8_t *)buffer.GetData(), w);
+
+	// Vulkan requires us to re-apply all dynamic state for each command buffer, and the above will cause us to start a new cmdbuf.
+	// So let's dirty the things that are involved in Vulkan dynamic state. Readbacks are not frequent so this won't hurt other backends.
+	gstate_c.Dirty(DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE);
+	framebufferManager_->RebindFramebuffer();
+
+	return true;
 }

GPU/Vulkan/VulkanUtil.cpp

@@ -210,7 +210,7 @@ VkPipeline Vulkan2D::GetPipeline(VkRenderPass rp, VkShaderModule vs, VkShaderMod
 
 	VkPipelineColorBlendAttachmentState blend0 = {};
 	blend0.blendEnable = false;
-	blend0.colorWriteMask = depthStencilMode == VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS ? 0 : 0xF;
+	blend0.colorWriteMask = depthStencilMode == VK2DDepthStencilMode::STENCIL_REPLACE_ALWAYS ? VK_COLOR_COMPONENT_A_BIT : 0xF;
 
 	VkPipelineColorBlendStateCreateInfo cbs = { VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO };
 	cbs.pAttachments = &blend0;

UI/EmuScreen.cpp

@@ -988,7 +988,6 @@ void EmuScreen::preRender() {
 	bool useBufferedRendering = g_Config.iRenderingMode != FB_NON_BUFFERED_MODE;
 	if ((!useBufferedRendering && !g_Config.bSoftwareRendering) || Core_IsStepping()) {
 		// We need to clear here already so that drawing during the frame is done on a clean slate.
-		DrawContext *draw = screenManager()->getDrawContext();
 		draw->BindFramebufferAsRenderTarget(nullptr, { RPAction::CLEAR, RPAction::CLEAR, 0xFF000000 });
 
 		Viewport viewport;
@@ -1061,6 +1060,9 @@ void EmuScreen::render() {
 	if (!osm.IsEmpty() || g_Config.bShowDebugStats || g_Config.iShowFPSCounter || g_Config.bShowTouchControls || g_Config.bShowDeveloperMenu || g_Config.bShowAudioDebug || saveStatePreview_->GetVisibility() != UI::V_GONE || g_Config.bShowFrameProfiler) {
 		DrawContext *thin3d = screenManager()->getDrawContext();
 
+		// It's possible we never ended up outputted anything - make sure we have the backbuffer.
+		thin3d->BindFramebufferAsRenderTarget(nullptr, { RPAction::KEEP, RPAction::KEEP });
+
 		// This sets up some important states but not the viewport.
 		screenManager()->getUIContext()->Begin();
 

Windows/GPU/WindowsVulkanContext.cpp

@@ -208,7 +208,8 @@ bool WindowsVulkanContext::Init(HINSTANCE hInst, HWND hWnd, std::string *error_m
 }
 
 void WindowsVulkanContext::Shutdown() {
-	draw_->HandleEvent(Draw::Event::LOST_BACKBUFFER, g_Vulkan->GetBackbufferWidth(), g_Vulkan->GetBackbufferHeight());
+	if (draw_)
+		draw_->HandleEvent(Draw::Event::LOST_BACKBUFFER, g_Vulkan->GetBackbufferWidth(), g_Vulkan->GetBackbufferHeight());
 
 	delete draw_;
 	draw_ = nullptr;
@@ -227,7 +228,6 @@ void WindowsVulkanContext::SwapBuffers() {
 }
 
 void WindowsVulkanContext::Resize() {
-	g_Vulkan->WaitUntilQueueIdle();
 	draw_->HandleEvent(Draw::Event::LOST_BACKBUFFER, g_Vulkan->GetBackbufferWidth(), g_Vulkan->GetBackbufferHeight());
 	g_Vulkan->DestroyObjects();
 

android/jni/app-android.cpp

@@ -341,7 +341,6 @@ void AndroidVulkanContext::SwapBuffers() {
 }
 
 void AndroidVulkanContext::Resize() {
-	g_Vulkan->WaitUntilQueueIdle();
 	draw_->HandleEvent(Draw::Event::LOST_BACKBUFFER, g_Vulkan->GetBackbufferWidth(), g_Vulkan->GetBackbufferHeight());
 	g_Vulkan->DestroyObjects();
 

build.gradle

@@ -1,7 +1,8 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 buildscript {
     repositories {
-       jcenter()
+        google()
+        jcenter()
     }
     dependencies {
         classpath 'com.android.tools.build:gradle:3.0.0'
@@ -10,6 +11,7 @@ buildscript {
 
 allprojects {
     repositories {
+        google()
         jcenter()
     }
 }

ext/native/thin3d/DataFormat.h

@@ -66,6 +66,13 @@ enum class DataFormat : uint8_t {
 	D32F_S8,
 };
 
+size_t DataFormatSizeInBytes(DataFormat fmt);
+bool DataFormatIsDepthStencil(DataFormat fmt);
+inline bool DataFormatIsColor(DataFormat fmt) {
+	return !DataFormatIsDepthStencil(fmt);
+}
+
 void ConvertFromRGBA8888(uint8_t *dst, const uint8_t *src, uint32_t dstStride, uint32_t srcStride, uint32_t width, uint32_t height, DataFormat format);
+void ConvertToD32F(uint8_t *dst, const uint8_t *src, uint32_t dstStride, uint32_t srcStride, uint32_t width, uint32_t height, DataFormat format);
 
 }  // namespace

ext/native/thin3d/VulkanQueueRunner.cpp

@@ -2,6 +2,8 @@
 #include "VulkanQueueRunner.h"
 #include "VulkanRenderManager.h"
 
+// TODO: This is only enough for 4x render resolution / 4x texture upscale for debugger.
+// Maybe we should use a dynamically allocated one for larger?
 const uint32_t readbackBufferSize = 2048 * 2048 * 4;
 
 void VulkanQueueRunner::CreateDeviceObjects() {
@@ -187,6 +189,7 @@ void VulkanQueueRunner::InitRenderpasses() {
 		}
 	}
 }
+
 void VulkanQueueRunner::RunSteps(VkCommandBuffer cmd, const std::vector<VKRStep *> &steps) {
 	// Optimizes renderpasses, then sequences them.
 	for (int i = 0; i < steps.size(); i++) {
@@ -204,6 +207,9 @@ void VulkanQueueRunner::RunSteps(VkCommandBuffer cmd, const std::vector<VKRStep
 		case VKRStepType::READBACK:
 			PerformReadback(step, cmd);
 			break;
+		case VKRStepType::READBACK_IMAGE:
+			PerformReadbackImage(step, cmd);
+			break;
 		}
 		delete steps[i];
 	}
@@ -226,6 +232,9 @@ void VulkanQueueRunner::LogSteps(const std::vector<VKRStep *> &steps) {
 		case VKRStepType::READBACK:
 			LogReadback(step);
 			break;
+		case VKRStepType::READBACK_IMAGE:
+			LogReadbackImage(step);
+			break;
 		}
 	}
 }
@@ -279,6 +288,10 @@ void VulkanQueueRunner::LogReadback(const VKRStep &pass) {
 	ILOG("Readback");
 }
 
+void VulkanQueueRunner::LogReadbackImage(const VKRStep &pass) {
+	ILOG("ReadbackImage");
+}
+
 void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer cmd) {
 	// TODO: If there are multiple, we can transition them together.
 	for (const auto &iter : step.preTransitions) {
@@ -326,6 +339,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 			vkCmdPipelineBarrier(cmd, srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &barrier);
 			iter.fb->color.layout = barrier.newLayout;
 		}
+		iter.fb->Release();
 	}
 
 	// This is supposed to bind a vulkan render pass to the command buffer.
@@ -378,7 +392,7 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::DRAW_INDEXED:
 			vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, c.drawIndexed.pipelineLayout, 0, 1, &c.drawIndexed.ds, c.drawIndexed.numUboOffsets, c.drawIndexed.uboOffsets);
-			vkCmdBindIndexBuffer(cmd, c.drawIndexed.ibuffer, c.drawIndexed.ioffset, VK_INDEX_TYPE_UINT16);
+			vkCmdBindIndexBuffer(cmd, c.drawIndexed.ibuffer, c.drawIndexed.ioffset, c.drawIndexed.indexType);
 			vkCmdBindVertexBuffers(cmd, 0, 1, &c.drawIndexed.vbuffer, &c.drawIndexed.voffset);
 			vkCmdDrawIndexed(cmd, c.drawIndexed.count, c.drawIndexed.instances, 0, 0, 0);
 			break;
@@ -557,6 +571,9 @@ void VulkanQueueRunner::PerformBindFramebufferAsRenderTarget(const VKRStep &step
 		}
 	} else {
 		renderPass = GetBackbufferRenderPass();
+		assert(step.render.color == VKRRenderPassAction::CLEAR || step.render.color == VKRRenderPassAction::DONT_CARE);
+		assert(step.render.depthStencil == VKRRenderPassAction::CLEAR || step.render.depthStencil == VKRRenderPassAction::DONT_CARE);
+		Uint8x4ToFloat4(clearVal[0].color.float32, step.render.clearColor);
 		numClearVals = 2;  // We don't bother with a depth buffer here.
 		clearVal[1].depthStencil.depth = 0.0f;
 		clearVal[1].depthStencil.stencil = 0;
@@ -572,6 +589,10 @@ void VulkanQueueRunner::PerformBindFramebufferAsRenderTarget(const VKRStep &step
 	rp_begin.clearValueCount = numClearVals;
 	rp_begin.pClearValues = numClearVals ? clearVal : nullptr;
 	vkCmdBeginRenderPass(cmd, &rp_begin, VK_SUBPASS_CONTENTS_INLINE);
+
+	if (step.render.framebuffer) {
+		step.render.framebuffer->Release();
+	}
 }
 
 void VulkanQueueRunner::PerformCopy(const VKRStep &step, VkCommandBuffer cmd) {
@@ -645,6 +666,9 @@ void VulkanQueueRunner::PerformCopy(const VKRStep &step, VkCommandBuffer cmd) {
 		}
 		vkCmdCopyImage(cmd, src->depth.image, src->depth.layout, dst->depth.image, dst->depth.layout, 1, &copy);
 	}
+
+	src->Release();
+	dst->Release();
 }
 
 void VulkanQueueRunner::PerformBlit(const VKRStep &step, VkCommandBuffer cmd) {
@@ -728,6 +752,9 @@ void VulkanQueueRunner::PerformBlit(const VKRStep &step, VkCommandBuffer cmd) {
 		}
 		vkCmdBlitImage(cmd, src->depth.image, src->depth.layout, dst->depth.image, dst->depth.layout, 1, &blit, step.blit.filter);
 	}
+
+	src->Release();
+	dst->Release();
 }
 
 void VulkanQueueRunner::SetupTransitionToTransferSrc(VKRImage &img, VkImageMemoryBarrier &barrier, VkPipelineStageFlags &stage, VkImageAspectFlags aspect) {
@@ -810,12 +837,12 @@ void VulkanQueueRunner::PerformReadback(const VKRStep &step, VkCommandBuffer cmd
 		assert(false);
 	}
 
-	assert(srcImage->format == VK_FORMAT_R8G8B8A8_UNORM);
-
 	VkImageMemoryBarrier barrier{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER };
 	VkPipelineStageFlags stage = 0;
-	SetupTransitionToTransferSrc(*srcImage, barrier, stage, step.readback.aspectMask);
-	vkCmdPipelineBarrier(cmd, stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+	if (srcImage->layout != VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) {
+		SetupTransitionToTransferSrc(*srcImage, barrier, stage, step.readback.aspectMask);
+		vkCmdPipelineBarrier(cmd, stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+	}
 
 	VkBufferImageCopy region{};
 	region.imageOffset = { step.readback.srcRect.offset.x, step.readback.srcRect.offset.y, 0 };
@@ -828,15 +855,64 @@ void VulkanQueueRunner::PerformReadback(const VKRStep &step, VkCommandBuffer cmd
 	vkCmdCopyImageToBuffer(cmd, srcImage->image, srcImage->layout, readbackBuffer_, 1, &region);
 
 	// NOTE: Can't read the buffer using the CPU here - need to sync first.
+
+	step.readback.src->Release();
 }
 
-void VulkanQueueRunner::CopyReadbackBuffer(int width, int height, Draw::DataFormat destFormat, int pixelStride, uint8_t *pixels) {
+void VulkanQueueRunner::PerformReadbackImage(const VKRStep &step, VkCommandBuffer cmd) {
+	// TODO: Clean this up - just reusing `SetupTransitionToTransferSrc`.
+	VKRImage srcImage;
+	srcImage.image = step.readback_image.image;
+	srcImage.layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+
+	VkImageMemoryBarrier barrier{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER };
+	VkPipelineStageFlags stage = 0;
+	SetupTransitionToTransferSrc(srcImage, barrier, stage, VK_IMAGE_ASPECT_COLOR_BIT);
+	vkCmdPipelineBarrier(cmd, stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
+
+	VkBufferImageCopy region{};
+	region.imageOffset = { step.readback_image.srcRect.offset.x, step.readback_image.srcRect.offset.y, 0 };
+	region.imageExtent = { step.readback_image.srcRect.extent.width, step.readback_image.srcRect.extent.height, 1 };
+	region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+	region.imageSubresource.layerCount = 1;
+	region.imageSubresource.mipLevel = step.readback_image.mipLevel;
+	region.bufferOffset = 0;
+	region.bufferRowLength = step.readback_image.srcRect.extent.width;
+	region.bufferImageHeight = step.readback_image.srcRect.extent.height;
+	vkCmdCopyImageToBuffer(cmd, step.readback_image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, readbackBuffer_, 1, &region);
+
+	// Now transfer it back to a texture.
+	TransitionImageLayout2(cmd, step.readback_image.image,
+		VK_IMAGE_ASPECT_COLOR_BIT,
+		VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
+		VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+		VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_SHADER_READ_BIT);
+
+	// NOTE: Can't read the buffer using the CPU here - need to sync first.
+}
+
+void VulkanQueueRunner::CopyReadbackBuffer(int width, int height, Draw::DataFormat srcFormat, Draw::DataFormat destFormat, int pixelStride, uint8_t *pixels) {
 	// Read back to the requested address in ram from buffer.
 	void *mappedData;
-	const int srcPixelSize = 4;  // TODO: Fix.
+	const size_t srcPixelSize = DataFormatSizeInBytes(srcFormat);
 
 	VkResult res = vkMapMemory(vulkan_->GetDevice(), readbackMemory_, 0, width * height * srcPixelSize, 0, &mappedData);
 	assert(res == VK_SUCCESS);
-	ConvertFromRGBA8888(pixels, (const uint8_t *)mappedData, pixelStride, width, width, height, destFormat);
+	if (srcFormat == Draw::DataFormat::R8G8B8A8_UNORM) {
+		ConvertFromRGBA8888(pixels, (const uint8_t *)mappedData, pixelStride, width, width, height, destFormat);
+	} else if (srcFormat == destFormat) {
+		uint8_t *dst = pixels;
+		const uint8_t *src = (const uint8_t *)mappedData;
+		for (int y = 0; y < height; ++y) {
+			memcpy(dst, src, width * srcPixelSize);
+			src += width * srcPixelSize;
+			dst += pixelStride * srcPixelSize;
+		}
+	} else if (destFormat == Draw::DataFormat::D32F) {
+		ConvertToD32F(pixels, (const uint8_t *)mappedData, pixelStride, width, width, height, srcFormat);
+	} else {
+		// TODO: Maybe a depth conversion or something?
+		assert(false);
+	}
 	vkUnmapMemory(vulkan_->GetDevice(), readbackMemory_);
 }

ext/native/thin3d/VulkanQueueRunner.h

@@ -84,6 +84,7 @@ enum class VKRStepType : uint8_t {
 	COPY,
 	BLIT,
 	READBACK,
+	READBACK_IMAGE,
 };
 
 enum class VKRRenderPassAction {
@@ -133,6 +134,11 @@ struct VKRStep {
 			VKRFramebuffer *src;
 			VkRect2D srcRect;
 		} readback;
+		struct {
+			VkImage image;
+			VkRect2D srcRect;
+			int mipLevel;
+		} readback_image;
 	};
 };
 
@@ -159,7 +165,7 @@ public:
 		return (int)depth * 3 + (int)color;
 	}
 
-	void CopyReadbackBuffer(int width, int height, Draw::DataFormat destFormat, int pixelStride, uint8_t *pixels);
+	void CopyReadbackBuffer(int width, int height, Draw::DataFormat srcFormat, Draw::DataFormat destFormat, int pixelStride, uint8_t *pixels);
 
 private:
 	void InitBackbufferRenderPass();
@@ -170,11 +176,13 @@ private:
 	void PerformCopy(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformBlit(const VKRStep &pass, VkCommandBuffer cmd);
 	void PerformReadback(const VKRStep &pass, VkCommandBuffer cmd);
+	void PerformReadbackImage(const VKRStep &pass, VkCommandBuffer cmd);
 
 	void LogRenderPass(const VKRStep &pass);
 	void LogCopy(const VKRStep &pass);
 	void LogBlit(const VKRStep &pass);
 	void LogReadback(const VKRStep &pass);
+	void LogReadbackImage(const VKRStep &pass);
 
 	static void SetupTransitionToTransferSrc(VKRImage &img, VkImageMemoryBarrier &barrier, VkPipelineStageFlags &stage, VkImageAspectFlags aspect);
 	static void SetupTransitionToTransferDst(VKRImage &img, VkImageMemoryBarrier &barrier, VkPipelineStageFlags &stage, VkImageAspectFlags aspect);

ext/native/thin3d/VulkanRenderManager.cpp

@@ -96,6 +96,16 @@ void CreateImage(VulkanContext *vulkan, VkCommandBuffer cmd, VKRImage &img, int
 	img.format = format;
 }
 
+bool VKRFramebuffer::Release() {
+	if (--refcount_ == 0) {
+		delete this;
+		return true;
+	} else if (refcount_ >= 10000 || refcount_ < 0) {
+		ELOG("Refcount (%d) invalid for object %p - corrupt?", refcount_.load(), this);
+	}
+	return false;
+}
+
 VulkanRenderManager::VulkanRenderManager(VulkanContext *vulkan) : vulkan_(vulkan), queueRunner_(vulkan) {
 	VkSemaphoreCreateInfo semaphoreCreateInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO };
 	semaphoreCreateInfo.flags = 0;
@@ -177,29 +187,52 @@ void VulkanRenderManager::CreateBackbuffers() {
 	curWidth_ = -1;
 	curHeight_ = -1;
 
+	VLOG("Backbuffers Created");
+
 	// Start the thread.
 	if (useThread) {
 		run_ = true;
+		// Won't necessarily be 0.
+		threadInitFrame_ = vulkan_->GetCurFrame();
+		VLOG("starting thread");
 		thread_ = std::thread(&VulkanRenderManager::ThreadFunc, this);
 	}
 }
 
-void VulkanRenderManager::DestroyBackbuffers() {
-	if (useThread) {
+void VulkanRenderManager::StopThread(bool shutdown) {
+	if (useThread && run_) {
 		run_ = false;
 		// Stop the thread.
 		for (int i = 0; i < vulkan_->GetInflightFrames(); i++) {
+			auto &frameData = frameData_[i];
 			{
-				std::unique_lock<std::mutex> lock(frameData_[i].push_mutex);
-				frameData_[i].push_condVar.notify_all();
+				std::unique_lock<std::mutex> lock(frameData.push_mutex);
+				frameData.push_condVar.notify_all();
 			}
 			{
-				std::unique_lock<std::mutex> lock(frameData_[i].pull_mutex);
-				frameData_[i].pull_condVar.notify_all();
+				std::unique_lock<std::mutex> lock(frameData.pull_mutex);
+				frameData.pull_condVar.notify_all();
 			}
 		}
 		thread_.join();
+		VLOG("thread joined.");
+
+		// Resignal fences for next time around - must be done after join.
+		for (int i = 0; i < vulkan_->GetInflightFrames(); i++) {
+			auto &frameData = frameData_[i];
+			frameData.readyForRun = false;
+			if (!shutdown && !frameData.readyForFence) {
+				vkDestroyFence(vulkan_->GetDevice(), frameData.fence, nullptr);
+				frameData.fence = vulkan_->CreateFence(true);
+			}
+		}
 	}
+}
+
+void VulkanRenderManager::DestroyBackbuffers() {
+	StopThread(false);
+	vulkan_->WaitUntilQueueIdle();
+
 	VkDevice device = vulkan_->GetDevice();
 	for (uint32_t i = 0; i < swapchainImageCount_; i++) {
 		vulkan_->Delete().QueueDeleteImageView(swapchainImages_[i].view);
@@ -214,12 +247,14 @@ void VulkanRenderManager::DestroyBackbuffers() {
 	framebuffers_.clear();
 
 	swapchainImages_.clear();
+	VLOG("Backbuffers Destroyed");
 }
 
 VulkanRenderManager::~VulkanRenderManager() {
-	run_ = false;
-	VkDevice device = vulkan_->GetDevice();
+	StopThread(true);
 	vulkan_->WaitUntilQueueIdle();
+
+	VkDevice device = vulkan_->GetDevice();
 	vkDestroySemaphore(device, acquireSemaphore_, nullptr);
 	vkDestroySemaphore(device, renderingCompleteSemaphore_, nullptr);
 	for (int i = 0; i < vulkan_->GetInflightFrames(); i++) {
@@ -236,12 +271,15 @@ VulkanRenderManager::~VulkanRenderManager() {
 // TODO: Activate this code.
 void VulkanRenderManager::ThreadFunc() {
 	setCurrentThreadName("RenderMan");
-	int threadFrame = -1;  // Increment first, start at 0.
+	int threadFrame = threadInitFrame_;
+	bool nextFrame = false;
 	while (run_) {
 		{
-			threadFrame++;
-			if (threadFrame >= vulkan_->GetInflightFrames())
-				threadFrame = 0;
+			if (nextFrame) {
+				threadFrame++;
+				if (threadFrame >= vulkan_->GetInflightFrames())
+					threadFrame = 0;
+			}
 			FrameData &frameData = frameData_[threadFrame];
 			std::unique_lock<std::mutex> lock(frameData.pull_mutex);
 			while (!frameData.readyForRun && run_) {
@@ -250,15 +288,22 @@ void VulkanRenderManager::ThreadFunc() {
 			}
 			VLOG("PULL: frame[%d].readyForRun = false", threadFrame);
 			frameData.readyForRun = false;
-			if (!run_)  // quick exit if bailing.
-				return;
+			// Previously we had a quick exit here that avoided calling Run() if run_ was suddenly false,
+			// but that created a race condition where frames could end up not finished properly on resize etc.
+
+			// Only increment next time if we're done.
+			nextFrame = frameData.type == VKRRunType::END;
+			assert(frameData.type == VKRRunType::END || frameData.type == VKRRunType::SYNC);
 		}
 		VLOG("PULL: Running frame %d", threadFrame);
 		Run(threadFrame);
+		VLOG("PULL: Finished frame %d", threadFrame);
 	}
+	VLOG("PULL: Quitting");
 }
 
 void VulkanRenderManager::BeginFrame() {
+	VLOG("BeginFrame");
 	VkDevice device = vulkan_->GetDevice();
 
 	int curFrame = vulkan_->GetCurFrame();
@@ -318,7 +363,9 @@ void VulkanRenderManager::BindFramebufferAsRenderTarget(VKRFramebuffer *fb, VKRR
 		curRenderStep_ = nullptr;
 	}
 	if (curRenderStep_ && curRenderStep_->commands.size() == 0) {
+#ifdef _DEBUG
 		ILOG("Empty render step. Usually happens after uploading pixels..");
+#endif
 	}
 
 	VKRStep *step = new VKRStep{ VKRStepType::RENDER };
@@ -332,6 +379,9 @@ void VulkanRenderManager::BindFramebufferAsRenderTarget(VKRFramebuffer *fb, VKRR
 	step->render.numDraws = 0;
 	step->render.finalColorLayout = !fb ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_UNDEFINED;
 	steps_.push_back(step);
+	if (fb) {
+		fb->AddRef();
+	}
 
 	curRenderStep_ = step;
 	curWidth_ = fb ? fb->width : vulkan_->GetBackbufferWidth();
@@ -345,13 +395,49 @@ void VulkanRenderManager::CopyFramebufferToMemorySync(VKRFramebuffer *src, int a
 	step->readback.srcRect.offset = { x, y };
 	step->readback.srcRect.extent = { (uint32_t)w, (uint32_t)h };
 	steps_.push_back(step);
+	src->AddRef();
 
 	curRenderStep_ = nullptr;
 
 	FlushSync();
 
-	// Need to call this after FlushSyfnc so the pixels are guaranteed to be ready in CPU-accessible VRAM.
-	queueRunner_.CopyReadbackBuffer(w, h, destFormat, pixelStride, pixels);
+	Draw::DataFormat srcFormat;
+	if (aspectBits & VK_IMAGE_ASPECT_COLOR_BIT) {
+		switch (src->color.format) {
+		case VK_FORMAT_R8G8B8A8_UNORM: srcFormat = Draw::DataFormat::R8G8B8A8_UNORM; break;
+		default: assert(false);
+		}
+	} else if (aspectBits & VK_IMAGE_ASPECT_STENCIL_BIT) {
+		// Copies from stencil are always S8.
+		srcFormat = Draw::DataFormat::S8;
+	} else if (aspectBits & VK_IMAGE_ASPECT_DEPTH_BIT) {
+		switch (src->depth.format) {
+		case VK_FORMAT_D24_UNORM_S8_UINT: srcFormat = Draw::DataFormat::D24_S8; break;
+		case VK_FORMAT_D32_SFLOAT_S8_UINT: srcFormat = Draw::DataFormat::D32F; break;
+		case VK_FORMAT_D16_UNORM_S8_UINT: srcFormat = Draw::DataFormat::D16; break;
+		default: assert(false);
+		}
+	} else {
+		assert(false);
+	}
+	// Need to call this after FlushSync so the pixels are guaranteed to be ready in CPU-accessible VRAM.
+	queueRunner_.CopyReadbackBuffer(w, h, srcFormat, destFormat, pixelStride, pixels);
+}
+
+void VulkanRenderManager::CopyImageToMemorySync(VkImage image, int mipLevel, int x, int y, int w, int h, Draw::DataFormat destFormat, uint8_t *pixels, int pixelStride) {
+	VKRStep *step = new VKRStep{ VKRStepType::READBACK_IMAGE };
+	step->readback_image.image = image;
+	step->readback_image.srcRect.offset = { x, y };
+	step->readback_image.srcRect.extent = { (uint32_t)w, (uint32_t)h };
+	step->readback_image.mipLevel = mipLevel;
+	steps_.push_back(step);
+
+	curRenderStep_ = nullptr;
+
+	FlushSync();
+
+	// Need to call this after FlushSync so the pixels are guaranteed to be ready in CPU-accessible VRAM.
+	queueRunner_.CopyReadbackBuffer(w, h, destFormat, destFormat, pixelStride, pixels);
 }
 
 void VulkanRenderManager::InitBackbufferFramebuffers(int width, int height) {
@@ -486,6 +572,8 @@ void VulkanRenderManager::CopyFramebuffer(VKRFramebuffer *src, VkRect2D srcRect,
 	step->copy.srcRect = srcRect;
 	step->copy.dst = dst;
 	step->copy.dstPos = dstPos;
+	src->AddRef();
+	dst->AddRef();
 
 	std::unique_lock<std::mutex> lock(mutex_);
 	steps_.push_back(step);
@@ -511,6 +599,8 @@ void VulkanRenderManager::BlitFramebuffer(VKRFramebuffer *src, VkRect2D srcRect,
 	step->blit.dst = dst;
 	step->blit.dstRect = dstRect;
 	step->blit.filter = filter;
+	src->AddRef();
+	dst->AddRef();
 
 	std::unique_lock<std::mutex> lock(mutex_);
 	steps_.push_back(step);
@@ -534,6 +624,7 @@ VkImageView VulkanRenderManager::BindFramebufferAsTexture(VKRFramebuffer *fb, in
 	}
 
 	curRenderStep_->preTransitions.push_back({ fb, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL });
+	fb->AddRef();
 	return fb->color.imageView;
 }
 
@@ -543,21 +634,48 @@ void VulkanRenderManager::Finish() {
 	FrameData &frameData = frameData_[curFrame];
 	if (!useThread) {
 		frameData.steps = std::move(steps_);
+		frameData.type = VKRRunType::END;
 		Run(curFrame);
 	} else {
 		std::unique_lock<std::mutex> lock(frameData.pull_mutex);
 		VLOG("PUSH: Frame[%d].readyForRun = true", curFrame);
 		frameData.steps = std::move(steps_);
 		frameData.readyForRun = true;
+		frameData.type = VKRRunType::END;
 		frameData.pull_condVar.notify_all();
 	}
 	vulkan_->EndFrame();
+
+	insideFrame_ = false;
 }
 
 void VulkanRenderManager::Wipe() {
 	int curFrame = vulkan_->GetCurFrame();
-	for (auto iter : steps_) {
-		delete iter;
+	for (auto step : steps_) {
+		// Need to release held framebuffers.
+		switch (step->stepType) {
+		case VKRStepType::RENDER:
+			for (const auto &iter : step->preTransitions) {
+				iter.fb->Release();
+			}
+			break;
+		case VKRStepType::COPY:
+			step->copy.src->Release();
+			step->copy.dst->Release();
+			break;
+		case VKRStepType::BLIT:
+			step->blit.src->Release();
+			step->blit.dst->Release();
+			break;
+		case VKRStepType::READBACK:
+			step->readback.src->Release();
+			break;
+		case VKRStepType::READBACK_IMAGE:
+			break;
+		default:
+			assert(false);
+		}
+		delete step;
 	}
 	steps_.clear();
 }
@@ -638,7 +756,8 @@ void VulkanRenderManager::Submit(int frame, bool triggerFence) {
 		assert(res == VK_SUCCESS);
 	}
 
-	if (useThread) {
+	// When !triggerFence, we notify after syncing with Vulkan.
+	if (useThread && triggerFence) {
 		VLOG("PULL: Frame %d.readyForFence = true", frame);
 		std::unique_lock<std::mutex> lock(frameData.push_mutex);
 		frameData.readyForFence = true;
@@ -649,7 +768,6 @@ void VulkanRenderManager::Submit(int frame, bool triggerFence) {
 void VulkanRenderManager::EndSubmitFrame(int frame) {
 	FrameData &frameData = frameData_[frame];
 	frameData.hasBegun = false;
-	insideFrame_ = false;
 
 	Submit(frame, true);
 
@@ -672,8 +790,6 @@ void VulkanRenderManager::EndSubmitFrame(int frame) {
 }
 
 void VulkanRenderManager::Run(int frame) {
-	VkDevice device = vulkan_->GetDevice();
-
 	BeginSubmitFrame(frame);
 
 	FrameData &frameData = frameData_[frame];
@@ -683,23 +799,24 @@ void VulkanRenderManager::Run(int frame) {
 	queueRunner_.RunSteps(cmd, stepsOnThread);
 	stepsOnThread.clear();
 
-	EndSubmitFrame(frame);
+	switch (frameData.type) {
+	case VKRRunType::END:
+		EndSubmitFrame(frame);
+		break;
+
+	case VKRRunType::SYNC:
+		EndSyncFrame(frame);
+		break;
+
+	default:
+		assert(false);
+	}
 
 	VLOG("PULL: Finished running frame %d", frame);
 }
 
-void VulkanRenderManager::FlushSync() {
-	int frame = vulkan_->GetCurFrame();
-	BeginSubmitFrame(frame);
-
+void VulkanRenderManager::EndSyncFrame(int frame) {
 	FrameData &frameData = frameData_[frame];
-	frameData.steps = std::move(steps_);
-
-	auto &stepsOnThread = frameData_[frame].steps;
-	VkCommandBuffer cmd = frameData.mainCmd;
-	queueRunner_.RunSteps(cmd, stepsOnThread);
-	stepsOnThread.clear();
-
 	Submit(frame, false);
 
 	// This is brutal! Should probably wait for a fence instead, not that it'll matter much since we'll
@@ -716,4 +833,39 @@ void VulkanRenderManager::FlushSync() {
 	};
 	VkResult res = vkBeginCommandBuffer(frameData.mainCmd, &begin);
 	assert(res == VK_SUCCESS);
+
+	if (useThread) {
+		std::unique_lock<std::mutex> lock(frameData.push_mutex);
+		frameData.readyForFence = true;
+		frameData.push_condVar.notify_all();
+	}
+}
+
+void VulkanRenderManager::FlushSync() {
+	// TODO: Reset curRenderStep_?
+	int curFrame = vulkan_->GetCurFrame();
+	FrameData &frameData = frameData_[curFrame];
+	if (!useThread) {
+		frameData.steps = std::move(steps_);
+		frameData.type = VKRRunType::SYNC;
+		Run(curFrame);
+	} else {
+		std::unique_lock<std::mutex> lock(frameData.pull_mutex);
+		VLOG("PUSH: Frame[%d].readyForRun = true (sync)", curFrame);
+		frameData.steps = std::move(steps_);
+		frameData.readyForRun = true;
+		assert(frameData.readyForFence == false);
+		frameData.type = VKRRunType::SYNC;
+		frameData.pull_condVar.notify_all();
+	}
+
+	if (useThread) {
+		std::unique_lock<std::mutex> lock(frameData.push_mutex);
+		// Wait for the flush to be hit, since we're syncing.
+		while (!frameData.readyForFence) {
+			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (sync)", curFrame);
+			frameData.push_condVar.wait(lock);
+		}
+		frameData.readyForFence = false;
+	}
 }

ext/native/thin3d/VulkanRenderManager.h

@@ -4,10 +4,11 @@
 // Only draws and binds are handled here, resource creation and allocations are handled as normal -
 // that's the nice thing with Vulkan.
 
-#include <cstdint>
-#include <thread>
-#include <mutex>
+#include <atomic>
 #include <condition_variable>
+#include <cstdint>
+#include <mutex>
+#include <thread>
 
 #include "Common/Vulkan/VulkanContext.h"
 #include "math/dataconv.h"
@@ -28,6 +29,7 @@ void CreateImage(VulkanContext *vulkan, VkCommandBuffer cmd, VKRImage &img, int
 class VKRFramebuffer {
 public:
 	VKRFramebuffer(VulkanContext *vk, VkCommandBuffer initCmd, VkRenderPass renderPass, int _width, int _height) : vulkan_(vk) {
+		refcount_ = 1;
 		width = _width;
 		height = _height;
 
@@ -58,6 +60,11 @@ public:
 		vulkan_->Delete().QueueDeleteFramebuffer(framebuf);
 	}
 
+	void AddRef() {
+		refcount_++;
+	}
+	bool Release();
+
 	int numShadows = 1;  // TODO: Support this.
 
 	VkFramebuffer framebuf = VK_NULL_HANDLE;
@@ -68,6 +75,12 @@ public:
 
 private:
 	VulkanContext *vulkan_;
+	std::atomic<int> refcount_;
+};
+
+enum class VKRRunType {
+	END,
+	SYNC,
 };
 
 class VulkanRenderManager {
@@ -89,6 +102,7 @@ public:
 	void BindFramebufferAsRenderTarget(VKRFramebuffer *fb, VKRRenderPassAction color, VKRRenderPassAction depth, uint32_t clearColor, float clearDepth, uint8_t clearStencil);
 	VkImageView BindFramebufferAsTexture(VKRFramebuffer *fb, int binding, int aspectBit, int attachment);
 	void CopyFramebufferToMemorySync(VKRFramebuffer *src, int aspectBits, int x, int y, int w, int h, Draw::DataFormat destFormat, uint8_t *pixels, int pixelStride);
+	void CopyImageToMemorySync(VkImage image, int mipLevel, int x, int y, int w, int h, Draw::DataFormat destFormat, uint8_t *pixels, int pixelStride);
 
 	void CopyFramebuffer(VKRFramebuffer *src, VkRect2D srcRect, VKRFramebuffer *dst, VkOffset2D dstPos, int aspectMask);
 	void BlitFramebuffer(VKRFramebuffer *src, VkRect2D srcRect, VKRFramebuffer *dst, VkRect2D dstRect, int aspectMask, VkFilter filter);
@@ -206,6 +220,9 @@ private:
 
 	// Bad for performance but sometimes necessary for synchronous CPU readbacks (screenshots and whatnot).
 	void FlushSync();
+	void EndSyncFrame(int frame);
+
+	void StopThread(bool shutdown);
 
 	// Permanent objects
 	VkSemaphore acquireSemaphore_;
@@ -220,8 +237,9 @@ private:
 		std::condition_variable pull_condVar;
 
 		bool readyForFence = true;
-
 		bool readyForRun = false;
+		VKRRunType type = VKRRunType::END;
+
 		VkFence fence;
 		// These are on different threads so need separate pools.
 		VkCommandPool cmdPoolInit;
@@ -233,7 +251,7 @@ private:
 
 		// Swapchain.
 		bool hasBegun = false;
-		uint32_t curSwapchainImage;
+		uint32_t curSwapchainImage = -1;
 	};
 	FrameData frameData_[VulkanContext::MAX_INFLIGHT_FRAMES];
 
@@ -249,6 +267,7 @@ private:
 	VulkanContext *vulkan_;
 	std::thread thread_;
 	std::mutex mutex_;
+	int threadInitFrame_ = 0;
 	VulkanQueueRunner queueRunner_;
 
 	// Swap chain management

ext/native/thin3d/thin3d.cpp

@@ -1,3 +1,4 @@
+#include <cassert>
 #include <cstring>
 #include <cstdint>
 
@@ -13,6 +14,7 @@ size_t DataFormatSizeInBytes(DataFormat fmt) {
 	case DataFormat::R8G8_UNORM: return 2;
 	case DataFormat::R8G8B8_UNORM: return 3;
 
+	case DataFormat::R4G4_UNORM_PACK8: return 1;
 	case DataFormat::R4G4B4A4_UNORM_PACK16: return 2;
 	case DataFormat::B4G4R4A4_UNORM_PACK16: return 2;
 	case DataFormat::A4R4G4B4_UNORM_PACK16: return 2;
@@ -38,6 +40,13 @@ size_t DataFormatSizeInBytes(DataFormat fmt) {
 	case DataFormat::R32G32B32_FLOAT: return 12;
 	case DataFormat::R32G32B32A32_FLOAT: return 16;
 
+	case DataFormat::S8: return 1;
+	case DataFormat::D16: return 2;
+	case DataFormat::D24_S8: return 4;
+	case DataFormat::D32F: return 4;
+	// Or maybe 8...
+	case DataFormat::D32F_S8: return 5;
+
 	default:
 		return 0;
 	}
@@ -372,5 +381,43 @@ void ConvertFromRGBA8888(uint8_t *dst, const uint8_t *src, uint32_t dstStride, u
 	}
 }
 
+void ConvertToD32F(uint8_t *dst, const uint8_t *src, uint32_t dstStride, uint32_t srcStride, uint32_t width, uint32_t height, DataFormat format) {
+	if (format == Draw::DataFormat::D32F) {
+		const float *src32 = (const float *)src;
+		float *dst32 = (float *)dst;
+		if (src == dst) {
+			return;
+		} else {
+			for (uint32_t y = 0; y < height; ++y) {
+				memcpy(dst32, src32, width * 4);
+				src32 += srcStride;
+				dst32 += dstStride;
+			}
+		}
+	} else if (format == Draw::DataFormat::D16) {
+		const uint16_t *src16 = (const uint16_t *)src;
+		float *dst32 = (float *)dst;
+		for (uint32_t y = 0; y < height; ++y) {
+			for (uint32_t x = 0; x < width; ++x) {
+				dst32[x] = (float)(int)src16[x] / 65535.0f;
+			}
+			src16 += srcStride;
+			dst32 += dstStride;
+		}
+	} else if (format == Draw::DataFormat::D24_S8) {
+		const uint32_t *src32 = (const uint32_t *)src;
+		float *dst32 = (float *)dst;
+		for (uint32_t y = 0; y < height; ++y) {
+			for (uint32_t x = 0; x < width; ++x) {
+				dst32[x] = (src32[x] & 0x00FFFFFF) / 16777215.0f;
+			}
+			src32 += srcStride;
+			dst32 += dstStride;
+		}
+	} else {
+		assert(false);
+	}
+}
+
 
 }  // namespace Draw

ext/native/thin3d/thin3d.h

@@ -644,12 +644,6 @@ protected:
 	int targetHeight_;
 };
 
-size_t DataFormatSizeInBytes(DataFormat fmt);
-bool DataFormatIsDepthStencil(DataFormat fmt);
-inline bool DataFormatIsColor(DataFormat fmt) {
-	return !DataFormatIsDepthStencil(fmt);
-}
-
 DrawContext *T3DCreateGLContext();
 
 extern const UniformBufferDesc UBPresetDesc;

ext/native/thin3d/thin3d_vulkan.cpp

@@ -1118,8 +1118,8 @@ void VKContext::Draw(int vertexCount, int offset) {
 }
 
 void VKContext::DrawIndexed(int vertexCount, int offset) {
-	VKBuffer *ibuf = static_cast<VKBuffer *>(curIBuffer_);
-	VKBuffer *vbuf = static_cast<VKBuffer *>(curVBuffers_[0]);
+	VKBuffer *ibuf = curIBuffer_;
+	VKBuffer *vbuf = curVBuffers_[0];
 
 	VkBuffer vulkanVbuf, vulkanIbuf, vulkanUBObuf;
 	uint32_t ubo_offset = (uint32_t)curPipeline_->PushUBO(push_, vulkan_, &vulkanUBObuf);
@@ -1248,9 +1248,10 @@ uint32_t VKContext::GetDataFormatSupport(DataFormat fmt) const {
 // use this frame's init command buffer.
 class VKFramebuffer : public Framebuffer {
 public:
+	// Inherits ownership so no AddRef.
 	VKFramebuffer(VKRFramebuffer *fb) : buf_(fb) {}
 	~VKFramebuffer() {
-		delete buf_;
+		buf_->Release();
 	}
 	VKRFramebuffer *GetFB() const { return buf_; }
 private: