Merge pull request #22435 from hrydgard/softgpu-depth-swizzle

SoftGPU: Implement depth swizzle (address translation), also for CPU access

Core/MIPS/ARM/ArmCompLoadStore.cpp

@@ -284,6 +284,10 @@ namespace MIPSComp
 		MIPSGPReg rt = _RT;
 		MIPSGPReg rs = _RS;
 		int o = op>>26;
+		// 16-bit accesses can go through the depth layout (Memory::DepthMirrored16).
+		if (Memory::DepthMirrorsActive() && (o == 33 || o == 37 || o == 41)) {
+			DISABLE;
+		}
 		if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) {
 			// Don't load anything into $zr
 			return;

Core/MIPS/ARM64/Arm64CompLoadStore.cpp

@@ -279,6 +279,10 @@ namespace MIPSComp {
 		MIPSGPReg rt = _RT;
 		MIPSGPReg rs = _RS;
 		int o = op >> 26;
+		// 16-bit accesses can go through the depth layout (Memory::DepthMirrored16).
+		if (Memory::DepthMirrorsActive() && (o == 33 || o == 37 || o == 41)) {
+			DISABLE;
+		}
 		if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) {
 			// Don't load anything into $zr
 			return;

Core/MIPS/IR/IRNativeCommon.cpp

@@ -280,10 +280,17 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
 		CompIR_Div(inst);
 		break;
 
-	case IROp::Load8:
-	case IROp::Load8Ext:
 	case IROp::Load16:
 	case IROp::Load16Ext:
+		// 16-bit accesses can go through the depth layout (Memory::DepthMirrored16).
+		if (Memory::DepthMirrorsActive())
+			CompIR_Generic(inst);
+		else
+			CompIR_Load(inst);
+		break;
+
+	case IROp::Load8:
+	case IROp::Load8Ext:
 	case IROp::Load32:
 	case IROp::Load32Linked:
 		CompIR_Load(inst);
@@ -302,8 +309,14 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
 		CompIR_VecLoad(inst);
 		break;
 
-	case IROp::Store8:
 	case IROp::Store16:
+		if (Memory::DepthMirrorsActive())
+			CompIR_Generic(inst);
+		else
+			CompIR_Store(inst);
+		break;
+
+	case IROp::Store8:
 	case IROp::Store32:
 		CompIR_Store(inst);
 		break;

Core/MIPS/IR/IRPassSimplify.cpp

@@ -6,6 +6,7 @@
 #include "Common/Data/Convert/SmallDataConvert.h"
 #include "Common/Log.h"
 #include "Core/Config.h"
+#include "Core/MemMap.h"
 #include "Core/MIPS/MIPSVFPUUtils.h"
 #include "Core/MIPS/IR/IRAnalysis.h"
 #include "Core/MIPS/IR/IRInterpreter.h"
@@ -1357,7 +1358,10 @@ bool ReduceLoads(const IRWriter &in, IRWriter &out, const IROptions &opts) {
 	for (int i = 0, n = (int)in.GetInstructions().size(); i < n; i++) {
 		IRInst inst = in.GetInstructions()[i];
 
-		if (inst.op == IROp::Load32 || inst.op == IROp::Load16 || inst.op == IROp::Load16Ext) {
+		// With the depth mirrors, a load's width decides whether it goes through the depth layout
+		// (Memory::DepthMirrored16), so 16-bit loads keep theirs, and no load becomes one.
+		const bool depthMirrors = Memory::DepthMirrorsActive();
+		if (inst.op == IROp::Load32 || ((inst.op == IROp::Load16 || inst.op == IROp::Load16Ext) && !depthMirrors)) {
 			int dest = IRDestGPR(GetIRMeta(inst));
 			for (int j = i + 1; j < n; j++) {
 				const IRInstMeta laterInst = GetIRMeta(in.GetInstructions()[j]);
@@ -1375,7 +1379,7 @@ bool ReduceLoads(const IRWriter &in, IRWriter &out, const IROptions &opts) {
 							if (mask == 0xff) {
 								nextSkip = j;
 							}
-						} else if ((mask & 0xffff0000) == 0 && inst.op == IROp::Load32) {
+						} else if ((mask & 0xffff0000) == 0 && inst.op == IROp::Load32 && !depthMirrors) {
 							inst.op = IROp::Load16;
 							if (mask == 0xffff) {
 								nextSkip = j;

Core/MIPS/x86/CompLoadStore.cpp

@@ -302,6 +302,10 @@ namespace MIPSComp {
 		MIPSGPReg rs = _RS;
 		MIPSGPReg rt = _RT;
 		int o = op>>26;
+		// 16-bit accesses can go through the depth layout (Memory::DepthMirrored16).
+		if (Memory::DepthMirrorsActive() && (o == 33 || o == 37 || o == 41)) {
+			DISABLE;
+		}
 		if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) {
 			// Don't load anything into $zr
 			return;

Core/MemMap.cpp

@@ -82,6 +82,18 @@ static u8 *m_pUncachedVRAM[4];
 // Required for HD Remasters to work properly.
 // This replaces RAM_NORMAL_SIZE at runtime.
 u32 g_MemorySize;
+int g_depthTranslationIndex = DepthTranslationIndex(0x400);
+
+bool DepthMirrorsActive() {
+	return PSP_CoreParameter().gpuCore == GPUCORE_SOFTWARE;
+}
+
+u32 DepthMirroredSlow(u32 address) {
+	if (!DepthMirrorsActive())
+		return address;
+	const DepthLayout &layout = DEPTH_LAYOUTS[(address >> 22) & 1][g_depthTranslationIndex];
+	return (address & 0x40000000) | 0x04000000 | layout.Stored(address & 0x001FFFFF);
+}
 // Used to store the PSP model on game startup.
 u32 g_PSPModel;
 u32 g_UserPartitionSize;

Core/MemMap.h

@@ -29,6 +29,7 @@
 #include "Common/CommonTypes.h"
 #include "Common/Swap.h"
 #include "Core/Opcode.h"
+#include "GPU/Common/DepthSwizzle.h"
 
 // PPSSPP is very aggressive about trying to do memory accesses directly, for speed.
 // This can be a problem when debugging though, as stray memory reads and writes will
@@ -192,7 +193,25 @@ inline float ReadUnchecked_Float(const u32 address) {
 #endif
 }
 
-inline u16 ReadUnchecked_U16(const u32 address) {
+// With the software renderer, which stores depth as the GE does, VRAM's 0x04200000 and 0x04600000 mirrors show it
+// through the GE's 16-bit and 32-bit color layouts (GPU/Common/DepthSwizzle.h, ppsspp-re geprobe exp88). Only 16-bit
+// accesses go through them: games read and write depth with nothing else. The layouts are fixed but for the EDRAM
+// address translation, which the software renderer keeps here as DepthTranslationIndex().
+extern int g_depthTranslationIndex;
+
+// Whether the depth mirrors apply the layouts: only with the software renderer. Hardware renderers keep depth
+// plain and handle the games that need the layout their own way.
+bool DepthMirrorsActive();
+u32 DepthMirroredSlow(u32 address);
+
+inline u32 DepthMirrored16(u32 address) {
+	if ((address & 0xBFA00000) == 0x04200000)
+		return DepthMirroredSlow(address);
+	return address;
+}
+
+inline u16 ReadUnchecked_U16(u32 address) {
+	address = DepthMirrored16(address);
 #ifdef MASKED_PSP_MEMORY
 	return *(u16_le *)(base + (address & MEMVIEW32_MASK));
 #else
@@ -233,6 +252,7 @@ inline void WriteUnchecked_Float(float data, u32 address) {
 }
 
 inline void WriteUnchecked_U16(u16 data, u32 address) {
+	address = DepthMirrored16(address);
 #ifdef MASKED_PSP_MEMORY
 	*(u16_le *)(base + (address & MEMVIEW32_MASK)) = data;
 #else

Core/MemMapFunctions.cpp

@@ -134,7 +134,7 @@ u8 ReadOrException_U8(const u32 address) {
 
 u16 ReadOrException_U16(const u32 address) {
 	u16_le value = 0;
-	ReadMemoryOrException<u16_le>(value, address);
+	ReadMemoryOrException<u16_le>(value, DepthMirrored16(address));
 	return (u16)value;
 }
 
@@ -155,7 +155,7 @@ void WriteOrException_U8(const u8 _Data, const u32 address) {
 }
 
 void WriteOrException_U16(const u16 _Data, const u32 address) {
-	WriteMemoryOrException<u16_le>(address, _Data);
+	WriteMemoryOrException<u16_le>(DepthMirrored16(address), _Data);
 }
 
 void WriteOrException_U32(const u32 _Data, const u32 address) {

GPU/CMakeLists.txt

@@ -80,6 +80,7 @@ set(GPU_SOURCES
 	Common/DepthBufferCommon.cpp
 	Common/DepthRaster.cpp
 	Common/DepthRaster.h
+	Common/DepthSwizzle.h
 	Common/TextureShaderCommon.cpp
 	Common/TextureShaderCommon.h
 	Common/DepalettizeShaderCommon.cpp

GPU/Common/DepthSwizzle.h

@@ -0,0 +1,86 @@
+// Copyright (c) 2012- PPSSPP Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0 or later versions.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official git repository and contact information can be found at
+// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
+
+#pragma once
+
+#include <cstdint>
+
+// Where the GE stores a depth buffer in VRAM (ppsspp-re geprobe exp8-exp10, docs/ge-precision-and-depth-layout.md).
+// The depth pixel at linear VRAM offset L (zbuf + (y * stride + x) * 2) is stored at Stored(L): with a 32-bit
+// color buffer, address bits 5 to log2(T) - 1 rotate left by one (bits 9 and 10 swap at T 0), then
+// (T << 3) | 0x40 is XORed in (0x600 at T 0). T is sceGeEdramSetAddrTranslation's value: 0, 0x200, 0x400,
+// 0x800 or 0x1000. The offsets are absolute, independent of the buffer's address and stride.
+//
+// Bits 0-4 never change, so each 32-byte aligned run of 16 pixels is stored together. The CPU (and texturing)
+// sees the same mapping through VRAM's mirrors: an access at 0x04200000 + L goes to Stored(L) of the 16-bit
+// layout, at 0x04600000 + L to the 32-bit one's (exp88).
+struct DepthLayout {
+	uint32_t rotMask = 0;
+	int rotShift = 0;
+	uint32_t xorBits = 0;
+
+	constexpr uint32_t Stored(uint32_t offset) const {
+		const uint32_t f = offset & rotMask;
+		return ((offset & ~rotMask) | (((f << 1) | (f >> rotShift)) & rotMask)) ^ xorBits;
+	}
+};
+
+constexpr DepthLayout GetDepthLayout(uint32_t translation, bool color32) {
+	DepthLayout layout;
+	if (translation == 0) {
+		layout.xorBits = 0x600;
+		if (color32) {
+			layout.rotMask = 0x600;
+			layout.rotShift = 1;
+		}
+		return layout;
+	}
+	layout.xorBits = (translation << 3) | 0x40;
+	int top = 0;
+	while ((2u << top) <= translation)
+		top++;
+	// Bits 5 to top - 1.
+	if (color32 && top - 5 > 1) {
+		layout.rotMask = ((1u << (top - 5)) - 1) << 5;
+		layout.rotShift = top - 5 - 1;
+	}
+	return layout;
+}
+
+// The translations as an index 0 to 4: 0, 0x200, 0x400, 0x800, 0x1000.
+constexpr int DepthTranslationIndex(uint32_t translation) {
+	int index = 0;
+	while (translation >= 0x200 && index < 4) {
+		translation >>= 1;
+		index++;
+	}
+	return index;
+}
+
+constexpr uint32_t DepthTranslationFromIndex(int index) {
+	return index == 0 ? 0 : 0x100 << index;
+}
+
+// Every layout there is, by [color32][translation index].
+inline constexpr DepthLayout DEPTH_LAYOUTS[2][5] = {
+	{ GetDepthLayout(0, false), GetDepthLayout(0x200, false), GetDepthLayout(0x400, false), GetDepthLayout(0x800, false), GetDepthLayout(0x1000, false) },
+	{ GetDepthLayout(0, true), GetDepthLayout(0x200, true), GetDepthLayout(0x400, true), GetDepthLayout(0x800, true), GetDepthLayout(0x1000, true) },
+};
+
+// A couple of exp8-exp10's measurements (TestDepthSwizzle has more).
+static_assert(DEPTH_LAYOUTS[0][2].Stored(0x088000) == 0x08a040, "16-bit layout at the default translation");
+static_assert(DEPTH_LAYOUTS[1][0].Stored(0x03171e) == 0x03111e, "32-bit layout at translation 0");

GPU/Debugger/Playback.cpp

@@ -140,8 +140,8 @@ public:
 	BufMapping(const std::vector<u8> &pushbuf) : pushbuf_(pushbuf) {
 	}
 
-	// Returns a pointer to contiguous memory for this access, or else 0 (failure).
-	u32 Map(u32 bufpos, u32 sz, const std::function<void()> &flush);
+	// Returns a pointer to contiguous memory for this access, aligned to align bytes, or else 0 (failure).
+	u32 Map(u32 bufpos, u32 sz, const std::function<void()> &flush, u32 align = 1);
 
 	// Clear and reset allocations made.
 	void Reset() {
@@ -229,10 +229,15 @@ protected:
 	const std::vector<u8> &pushbuf_;
 };
 
-u32 BufMapping::Map(u32 bufpos, u32 sz, const std::function<void()> &flush) {
+u32 BufMapping::Map(u32 bufpos, u32 sz, const std::function<void()> &flush, u32 align) {
 	int slab1 = bufpos / SLAB_SIZE;
 	int slab2 = (bufpos + sz - 1) / SLAB_SIZE;
 
+	// Older recorders could reuse data at any offset (a CLUT found one byte into another), but the GE
+	// ignores the low bits of a CLUT or texture address. The extra mappings are separate, aligned copies.
+	if ((bufpos & (align - 1)) != 0)
+		return MapExtra(bufpos, sz, flush);
+
 	if (slab1 == slab2) {
 		// Shortcut in case it's simply the most recent slab.
 		if (slabs_[lastSlab_].Matches(slab1 * SLAB_SIZE))
@@ -425,6 +430,10 @@ private:
 	bool zTest_ = false;
 	bool zWriteDisable_ = false;
 	u32 clearMode_ = 0;
+	// Right after INIT, where a CLUT command is the CLUT the GE had loaded.
+	bool initialClut_ = false;
+	// Whether the last DISPLAY showed something (address 0 turns the display off).
+	bool haveDisplay_ = false;
 	// The vertices recorded for the next draw.
 	u32 lastVertsPtr_ = 0;
 	u32 lastVertsSize_ = 0;
@@ -463,6 +472,7 @@ void DumpExecute::SyncStall() {
 }
 
 void DumpExecute::Registers(u32 ptr, u32 sz) {
+	initialClut_ = false;
 	if (execListBuf == 0) {
 		u32 allocSize = LIST_BUF_SIZE;
 		execListBuf = userMemory.Alloc(allocSize, true, "List buf");
@@ -488,22 +498,21 @@ void DumpExecute::Registers(u32 ptr, u32 sz) {
 	// Validate space for jump.
 	u32 allocSize = pendingSize + sz + 8;
 	if (execListPos + allocSize >= execListBuf + LIST_BUF_SIZE) {
-		Memory::WriteUnchecked_U32((GE_CMD_BASE << 24) | ((execListBuf >> 8) & 0x00FF0000), execListPos);
-		Memory::WriteUnchecked_U32((GE_CMD_JUMP << 24) | (execListBuf & 0x00FFFFFF), execListPos + 4);
+		// Finish this list, wait for the GE to run it, and start a new one at the buffer's start. Jumping
+		// back instead let the next lap overwrite commands the GE hadn't reached yet: Warriors (14660) lost
+		// half its draws. The PSP replayer does the same. GE state, BASE included, carries over.
+		Memory::WriteUnchecked_U32(GE_CMD_FINISH << 24, execListPos);
+		Memory::WriteUnchecked_U32(GE_CMD_END << 24, execListPos + 4);
+		execListPos += 8;
+		SyncStall();
+		ExecuteOnMain(Operation{ OpType::ListSync, execListID });
 
 		execListPos = execListBuf;
-		// The queued vertex, index and texture addresses were made for the previous base, which the jump
-		// just replaced (Tiger Woods ULUS10420 then read draw 21026's indices from 16 MB away).
-		if (lastBase_ != 0xFFFFFFFF && lastBase_ != (execListBuf & 0xFF000000)) {
-			Memory::WriteUnchecked_U32((GE_CMD_BASE << 24) | ((lastBase_ >> 8) & 0x00FF0000), execListPos);
-			execListPos += 4;
-		} else {
-			lastBase_ = execListBuf & 0xFF000000;
-		}
-
-		// Don't continue until we've stalled.
-		// TODO: Is this really needed? It seems fine without it.
-		SyncStall();
+		Memory::WriteUnchecked_U32(GE_CMD_NOP << 24, execListPos);
+		execListPos += 4;
+		gpu->EnableInterrupts(false);
+		execListID = ExecuteOnMain(Operation{ OpType::EnqueueList, execListBuf, execListPos });
+		gpu->EnableInterrupts(true);
 	}
 
 	Memory::MemcpyUnchecked(execListPos, execListQueue.data(), pendingSize);
@@ -600,7 +609,18 @@ void DumpExecute::SubmitListEnd() {
 }
 
 void DumpExecute::Init(u32 ptr, u32 sz) {
-	gstate.Restore((u32_le *)(pushbuf_.data() + ptr));
+	// Dumps from before savedContextVersion 1 hold PPSSPP's old context layout, with the matrices as raw
+	// floats; the new one ends its commands with an END. Restoring an old one as new turned its matrices
+	// into garbage and the scene disappeared (4140). The PSP replayer tells them apart the same way.
+	const u32_le *context = (const u32_le *)(pushbuf_.data() + ptr);
+	bool oldLayout = true;
+	for (int i = 17; i < 512; ++i) {
+		if (context[i] == GE_CMD_END << 24) {
+			oldLayout = false;
+			break;
+		}
+	}
+	gstate.Restore(context, oldLayout);
 	ExecuteOnMain(Operation{ OpType::ReapplyGfxState });
 	fbPtr_ = gstate.fbptr & 0x00FFFFFF;
 	fbWidth_ = gstate.fbwidth & 0x00FFFFFF;
@@ -620,6 +640,7 @@ void DumpExecute::Init(u32 ptr, u32 sz) {
 		lastTex_[i] = 0;
 	}
 	lastBase_ = 0xFFFFFFFF;
+	initialClut_ = true;
 }
 
 // The drawing-space bounds of a through mode draw's vertices, or false if they aren't simple to read
@@ -774,7 +795,7 @@ void DumpExecute::Clut(u32 ptr, u32 sz) {
 
 		execClutAddr = 0;
 	} else {
-		u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this));
+		u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this), 16);
 		if (psp == 0) {
 			ERROR_LOG(Log::GeDebugger, "Unable to allocate for clut");
 			return;
@@ -782,11 +803,21 @@ void DumpExecute::Clut(u32 ptr, u32 sz) {
 
 		execListQueue.push_back((GE_CMD_CLUTADDRUPPER << 24) | ((psp >> 8) & 0x00FF0000));
 		execListQueue.push_back((GE_CMD_CLUTADDR << 24) | (psp & 0x00FFFFFF));
+		if (initialClut_) {
+			// The CLUT the GE had loaded when the recording started (the recorder saves it right after
+			// INIT), with no LOADCLUT to follow, unlike the CLUTs recorded at a LOADCLUT: load it here,
+			// then put the game's CLUT address back. Without this, draws used whatever CLUT was loaded
+			// before (HotBrain 16131, ULUS10268, drew its save screen background black).
+			execListQueue.push_back((GE_CMD_LOADCLUT << 24) | ((sz / 32) & 0x3F));
+			execListQueue.push_back(gstate.clutaddrupper);
+			execListQueue.push_back(gstate.clutaddr);
+			initialClut_ = false;
+		}
 	}
 }
 
 void DumpExecute::TransferSrc(u32 ptr, u32 sz) {
-	u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this));
+	u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this), 16);
 	if (psp == 0) {
 		ERROR_LOG(Log::GeDebugger, "Unable to allocate for transfer");
 		return;
@@ -799,6 +830,36 @@ void DumpExecute::TransferSrc(u32 ptr, u32 sz) {
 	execListQueue.push_back(((GE_CMD_TRANSFERSRC) << 24) | (psp & 0x00FFFFFF));
 }
 
+// The PSP replayer writes recorded memory with the CPU, and the PSP sends every CPU access to VRAM's 0x04200000 and
+// 0x04600000 mirrors through the depth layout. With the software renderer PPSSPP's mirrors do that for 16-bit
+// accesses only, so a write into one goes a run of 32 aligned bytes at a time, which the layout keeps together.
+// IL-2's depth texture at 0x04710000 is recorded as the game sampled it, and only comes back so.
+static void CopyToMemory(u32 addr, const u8 *data, u32 size) {
+	if (!Memory::DepthMirrorsActive() || !Memory::IsDepthTexVRAMAddress(addr)) {
+		Memory::MemcpyUnchecked(addr, data, size);
+		return;
+	}
+	for (u32 pos = 0; pos < size; ) {
+		const u32 a = addr + pos;
+		const u32 n = std::min(32 - (a & 31), size - pos);
+		Memory::MemcpyUnchecked(Memory::DepthMirrored16(a), data + pos, n);
+		pos += n;
+	}
+}
+
+static void SetMemory(u32 addr, u8 value, u32 size) {
+	if (!Memory::DepthMirrorsActive() || !Memory::IsDepthTexVRAMAddress(addr)) {
+		Memory::Memset(addr, value, size);
+		return;
+	}
+	for (u32 pos = 0; pos < size; ) {
+		const u32 a = addr + pos;
+		const u32 n = std::min(32 - (a & 31), size - pos);
+		Memory::Memset(Memory::DepthMirrored16(a), value, n);
+		pos += n;
+	}
+}
+
 void DumpExecute::Memset(u32 ptr, u32 sz) {
 	PROFILE_THIS_SCOPE("ReplayMemset");
 	struct MemsetCommand {
@@ -816,7 +877,7 @@ void DumpExecute::Memset(u32 ptr, u32 sz) {
 		// TODO: should probably do this as an operation.
 		// The software renderer leaves the memset to the caller, like sceKernelMemset does.
 		if (!gpu->PerformMemorySet(data->dest, (u8)data->value, data->sz))
-			Memory::Memset(data->dest, (u8)data->value, data->sz);
+			SetMemory(data->dest, (u8)data->value, data->sz);
 	}
 }
 
@@ -829,14 +890,14 @@ void DumpExecute::Memcpy(u32 ptr, u32 sz) {
 	if (Memory::IsVRAMAddress(execMemcpyDest)) {
 		SyncStall();
 		gpu->Flush();
-		Memory::MemcpyUnchecked(execMemcpyDest, pushbuf_.data() + ptr, sz);
+		CopyToMemory(execMemcpyDest, pushbuf_.data() + ptr, sz);
 		NotifyMemInfo(MemBlockFlags::WRITE, execMemcpyDest, sz, "ReplayMemcpy");
 		gpu->PerformWriteColorFromMemory(execMemcpyDest, sz);
 	}
 }
 
 void DumpExecute::Texture(int level, u32 ptr, u32 sz) {
-	u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this));
+	u32 psp = mapping_.Map(ptr, sz, std::bind(&DumpExecute::SyncStall, this), 16);
 	if (psp == 0) {
 		ERROR_LOG(Log::GeDebugger, "Unable to allocate for texture");
 		return;
@@ -921,7 +982,7 @@ void DumpExecute::CopyAroundDrawn(u32 addr, const u8 *data, u32 size) {
 	int64_t pos = start;
 	auto copyTo = [&](int64_t until) {
 		if (until > pos) {
-			Memory::MemcpyUnchecked(addr + (u32)(pos - start), data + (pos - start), (u32)(until - pos));
+			CopyToMemory(addr + (u32)(pos - start), data + (pos - start), (u32)(until - pos));
 			NotifyMemInfo(MemBlockFlags::WRITE, addr + (u32)(pos - start), (u32)(until - pos), "ReplayTex");
 		}
 	};
@@ -943,6 +1004,11 @@ void DumpExecute::Display(u32 ptr, u32 sz, bool allowFlip) {
 	// Sync up drawing.
 	SyncStall();
 
+	// A display turned off shows nothing to compare, so the result falls back to the last framebuffer
+	// drawn to, as for a dump without a DISPLAY (Auditorium 9213). The PSP replayer does the same.
+	haveDisplay_ = disp->topaddr.ptr != 0;
+	if (!haveDisplay_)
+		return;
 	__DisplaySetFramebuf(disp->topaddr.ptr, disp->linesize, disp->pixelFormat, 1);
 	if (allowFlip) {
 		__DisplaySetFramebuf(disp->topaddr.ptr, disp->linesize, disp->pixelFormat, 0);
@@ -986,6 +1052,15 @@ ReplayResult DumpExecute::Run() {
 		}
 
 		const Command &cmd = commands_[i];
+		// Only the first INIT is real. Before 71210f3fa2 a second dump request during a recording started
+		// it again, writing another INIT (and initial CLUT) into the dump: the GE state of that moment,
+		// which already includes register writes the dump only has after it. Applied, it ran the last
+		// draws with the wrong state (GOD EATER BURST 13950 came out black, Street Riders 14746 garbled).
+		if (cmd.type == CommandType::INIT && i != 0) {
+			if (i + 1 < commands_.size() && commands_[i + 1].type == CommandType::CLUT)
+				++i;
+			continue;
+		}
 		switch (cmd.type) {
 		case CommandType::INIT:
 			Init(cmd.ptr, cmd.sz);
@@ -1064,6 +1139,12 @@ ReplayResult DumpExecute::Run() {
 	}
 
 	SubmitListEnd();
+	if (!haveDisplay_ && fbWidth_ != 0) {
+		SyncStall();
+		const u32 addr = 0x04000000 | (fbPtr_ & 0x001FFFF0);
+		__DisplaySetFramebuf(addr, fbWidth_ & 0x07FC, fbFormat_, 1);
+		__DisplaySetFramebuf(addr, fbWidth_ & 0x07FC, fbFormat_, 0);
+	}
 	return ReplayResult::Done;
 }
 

GPU/GPU.vcxproj

@@ -256,6 +256,7 @@
   <ItemGroup>
     <ClInclude Include="..\ext\xbrz\xbrz.h" />
     <ClInclude Include="Common\DepthRaster.h" />
+    <ClInclude Include="Common\DepthSwizzle.h" />
     <ClInclude Include="Common\ImageCommon.h" />
     <ClInclude Include="Common\ReplacedTexture.h" />
     <ClInclude Include="Common\TextureReplacer.h" />

GPU/GPU.vcxproj.filters

@@ -264,6 +264,9 @@
     <ClInclude Include="Common\DepthRaster.h">
       <Filter>Common</Filter>
     </ClInclude>
+    <ClInclude Include="Common\DepthSwizzle.h">
+      <Filter>Common</Filter>
+    </ClInclude>
     <ClInclude Include="GPUStateSIMDUtil.h">
       <Filter>Common</Filter>
     </ClInclude>

GPU/GPUCommon.cpp

@@ -2015,8 +2015,10 @@ void GPUCommon::DoBlockTransfer(u32 skipDrawReason) {
 		// Can use GetPointerUnchecked because we checked the addresses above. We could also avoid them
 		// entirely by walking a couple of pointers...
 
-		// Simple case: just a straight copy, no overlap or wrapping.
-		if (srcStride == dstStride && (u32)width == srcStride && !srcDstOverlap && srcValid && dstValid) {
+		if (CopyBlockTransfer(srcBasePtr, srcStride, srcX, srcY, dstBasePtr, dstStride, dstX, dstY, width, height, bpp)) {
+			// The backend copied it.
+		} else if (srcStride == dstStride && (u32)width == srcStride && !srcDstOverlap && srcValid && dstValid) {
+			// Simple case: just a straight copy, no overlap or wrapping.
 			u32 srcLineStartAddr = srcBasePtr + (srcY * srcStride + srcX) * bpp;
 			u32 dstLineStartAddr = dstBasePtr + (dstY * dstStride + dstX) * bpp;
 			u32 bytesToCopy = width * height * bpp;

GPU/GPUCommon.h

@@ -144,7 +144,7 @@ public:
 	// For example, a debugger is active.
 	bool ShouldSplitOverGe() const;
 
-	uint32_t SetAddrTranslation(uint32_t value);
+	virtual uint32_t SetAddrTranslation(uint32_t value);
 	uint32_t GetAddrTranslation();
 
 	virtual void SetDisplayFramebuffer(u32 framebuf, u32 stride, GEBufferFormat format) = 0;
@@ -344,6 +344,10 @@ protected:
 	void FastLoadBoneMatrix(u32 target);
 	void FlushImm();
 	void DoBlockTransfer(u32 skipDrawReason);
+	// Does a block transfer's copy itself and returns true, when the backend needs to.
+	virtual bool CopyBlockTransfer(u32 srcBasePtr, u32 srcStride, int srcX, int srcY, u32 dstBasePtr, u32 dstStride, int dstX, int dstY, int width, int height, int bpp) {
+		return false;
+	}
 
 	// TODO: Unify this. Vulkan and OpenGL are different due to how they buffer data.
 	virtual void FinishDeferred() {}

GPU/GPUState.cpp

@@ -205,6 +205,10 @@ void GEState::FastLoadBoneMatrix(u32 addr) {
 }
 
 void GEState::Restore(const u32_le *ptr) {
+	Restore(ptr, savedContextVersion == 0);
+}
+
+void GEState::Restore(const u32_le *ptr, bool oldLayout) {
 	// Not sure what the first 10 values are, exactly, but these seem right.
 	gstate_c.vertexAddr = ptr[5];
 	gstate_c.indexAddr = ptr[6];
@@ -218,7 +222,7 @@ void GEState::Restore(const u32_le *ptr) {
 		}
 	}
 
-	if (savedContextVersion == 0) {
+	if (oldLayout) {
 		if (Memory::IsValidAddress(getClutAddress()))
 			loadclut = *cmds++;
 		boneMatrixNumber = *cmds++;

GPU/GPUState.h

@@ -439,6 +439,8 @@ struct GEState {
 	static void Reset();
 	void Save(u32_le *ptr);
 	void Restore(const u32_le *ptr);
+	// oldLayout: PPSSPP's layout before savedContextVersion 1, with the matrices as raw floats.
+	void Restore(const u32_le *ptr, bool oldLayout);
 };
 
 bool vertTypeIsSkinningEnabled(u32 vertType);

GPU/Software/BinManager.cpp

@@ -26,11 +26,13 @@
 #include "Common/Thread/ThreadManager.h"
 #include "Common/Data/Text/StringWriter.h"
 #include "Common/TimeUtil.h"
+#include "Core/MemMap.h"
 #include "Core/System.h"
 #include "GPU/Common/TextureDecoder.h"
 #include "GPU/Software/BinManager.h"
 #include "GPU/Software/Rasterizer.h"
 #include "GPU/Software/RasterizerRectangle.h"
+#include "GPU/Software/SoftGpu.h"
 
 // Sometimes useful for debugging.
 static constexpr bool FORCE_SINGLE_THREAD = false;
@@ -216,9 +218,42 @@ void BinManager::PushState() {
 		}
 	}
 	states_[stateIndex_].samplerID.cached.clut = cluts_[clutIndex_].readable;
+	if (states_[stateIndex_].enableTextures)
+		DeswizzleMirrorTextures(states_[stateIndex_]);
 	creatingState_ = false;
 }
 
+// Texturing from VRAM's 0x04200000 and 0x04600000 mirrors reads it as the CPU does there: depth stored in the
+// GE's layout comes out linear (16-bit and 32-bit color layouts). Such levels are drawn from a linear copy,
+// made after everything queued is drawn. Since each such state flushes, one set of copies is enough.
+void BinManager::DeswizzleMirrorTextures(RasterizerState &state) {
+	bool flushed = false;
+	for (int i = 0; i <= state.maxTexLevel; ++i) {
+		const uint32_t addr = state.texaddr[i];
+		if (!state.texptr[i] || !Memory::IsDepthTexVRAMAddress(addr))
+			continue;
+		if (!flushed) {
+			// Queued drawing may still write what this reads. This state is the last in the ring, which
+			// survives the flush.
+			Flush("depth view");
+			flushed = true;
+		}
+		const DepthLayout layout = GetDepthLayout(depthbuf.translation, (addr & 0x00400000) != 0);
+		const uint32_t bytes = state.texbufw[i] * textureBitsPerPixel[state.samplerID.texfmt] / 8 * state.samplerID.cached.sizes[i].h;
+		std::vector<u8> &view = depthViews_[i];
+		view.resize(bytes);
+		const uint32_t start = addr & 0x001FFFFF;
+		for (uint32_t pos = 0; pos < bytes; ) {
+			// Stored in runs of 32 aligned bytes.
+			const uint32_t offset = start + pos;
+			const uint32_t n = std::min(32 - (offset & 31), bytes - pos);
+			memcpy(view.data() + pos, depthbuf.vram + layout.Stored(offset), n);
+			pos += n;
+		}
+		state.texptr[i] = view.data();
+	}
+}
+
 void BinManager::UpdateState() {
 	PROFILE_THIS_SCOPE("bin_state");
 	auto jitGen = JitGeneration;

GPU/Software/BinManager.h

@@ -338,6 +338,8 @@ private:
 	bool pastStride_ = false;
 	Rasterizer::RasterizerState selfTexState_;
 	std::vector<u8> selfTexBuf_[8];
+	// Linear copies of texture levels read through a swizzled VRAM mirror (DeswizzleMirrorTextures).
+	std::vector<u8> depthViews_[8];
 	uint32_t selfTexAddr_[8]{};
 	bool selfTexValid_ = false;
 	uint32_t texFlushGen_ = 0;
@@ -370,6 +372,7 @@ private:
 	const Rasterizer::RasterizerState &SelfTextureSnapshot(const BinItem &item, const Rasterizer::RasterizerState &state);
 	void OptimizePendingStates(uint16_t first, uint16_t last);
 	void PushState();
+	void DeswizzleMirrorTextures(Rasterizer::RasterizerState &state);
 	template <typename F>
 	void AddFlags(F calculate);
 	BinCoords Scissor(BinCoords range);

GPU/Software/DrawPixel.h

@@ -84,6 +84,7 @@ private:
 	// Note: these may require a temporary reg.
 	RegCache::Reg GetColorOff(const PixelFuncID &id);
 	RegCache::Reg GetDepthOff(const PixelFuncID &id);
+	void EmitDepthPointer(const PixelFuncID &id, RegCache::Reg index, RegCache::Reg temp);
 	RegCache::Reg GetDestStencil(const PixelFuncID &id);
 
 	void WriteConstantPool(const PixelFuncID &id);

GPU/Software/DrawPixelX86.cpp

@@ -20,6 +20,7 @@
 #if PPSSPP_ARCH(AMD64)
 
 #include "Common/x64Emitter.h"
+#include "Common/BitSet.h"
 #include "Common/CPUDetect.h"
 #include "Common/LogReporting.h"
 #include "Common/Math/SIMDHeaders.h"
@@ -148,22 +149,13 @@ RegCache::Reg PixelJitCache::GetColorOff(const PixelFuncID &id) {
 			ADD(32, R(argXReg), R(argYReg));
 
 			// Now add the pointer for the color buffer.
-			if (loadDepthOff) {
-				_assert_msg_(Accessible(&fb.data, &depthbuf.data), "fb.data and depthbuf.data too far apart: %p %p (fb=%08x d=%08x)", fb.data, depthbuf.data, gstate.getFrameBufAddress(), gstate.getDepthBufAddress());
+			if (loadDepthOff)
 				depthTemp = regCache_.Alloc(RegCache::GEN_DEPTH_OFF);
-				if (RipAccessible(&fb.data) && RipAccessible(&depthbuf.data)) {
-					MOV(PTRBITS, R(argYReg), M(&fb.data));
-				} else {
-					MOV(PTRBITS, R(depthTemp), ImmPtr(&fb.data));
-					MOV(PTRBITS, R(argYReg), MatR(depthTemp));
-				}
+			if (RipAccessible(&fb.data)) {
+				MOV(PTRBITS, R(argYReg), M(&fb.data));
 			} else {
-				if (RipAccessible(&fb.data)) {
-					MOV(PTRBITS, R(argYReg), M(&fb.data));
-				} else {
-					MOV(PTRBITS, R(argYReg), ImmPtr(&fb.data));
-					MOV(PTRBITS, R(argYReg), MatR(argYReg));
-				}
+				MOV(PTRBITS, R(argYReg), ImmPtr(&fb.data));
+				MOV(PTRBITS, R(argYReg), MatR(argYReg));
 			}
 			LEA(PTRBITS, argYReg, MComplex(argYReg, argXReg, id.FBFormat() == GE_FORMAT_8888 ? 4 : 2, 0));
 			// With that, argYOff is now GEN_COLOR_OFF.
@@ -174,12 +166,7 @@ RegCache::Reg PixelJitCache::GetColorOff(const PixelFuncID &id) {
 
 			// Next, also calculate the depth offset, unless we won't need it at all.
 			if (loadDepthOff) {
-				if (RipAccessible(&fb.data) && RipAccessible(&depthbuf.data)) {
-					MOV(PTRBITS, R(depthTemp), M(&depthbuf.data));
-				} else {
-					MOV(PTRBITS, R(depthTemp), MAccessibleDisp(depthTemp, &fb.data, &depthbuf.data));
-				}
-				LEA(PTRBITS, argXReg, MComplex(depthTemp, argXReg, 2, 0));
+				EmitDepthPointer(id, argXReg, depthTemp);
 				regCache_.Release(depthTemp, RegCache::GEN_DEPTH_OFF);
 
 				// Okay, same deal - release as GEN_DEPTH_OFF and force retain it.
@@ -269,13 +256,7 @@ RegCache::Reg PixelJitCache::GetDepthOff(const PixelFuncID &id) {
 		regCache_.Unlock(argXReg, RegCache::GEN_ARG_X);
 
 		X64Reg temp = regCache_.Alloc(RegCache::GEN_TEMP_HELPER);
-		if (RipAccessible(&depthbuf.data)) {
-			MOV(PTRBITS, R(temp), M(&depthbuf.data));
-		} else {
-			MOV(PTRBITS, R(temp), ImmPtr(&depthbuf.data));
-			MOV(PTRBITS, R(temp), MatR(temp));
-		}
-		LEA(PTRBITS, r, MComplex(temp, r, 2, 0));
+		EmitDepthPointer(id, r, temp);
 		regCache_.Release(temp, RegCache::GEN_TEMP_HELPER);
 
 		return r;
@@ -283,6 +264,41 @@ RegCache::Reg PixelJitCache::GetDepthOff(const PixelFuncID &id) {
 	return regCache_.Find(RegCache::GEN_DEPTH_OFF);
 }
 
+// Turns a pixel index (y * stride + x) into a pointer to its depth, stored as the GE stores it
+// (GPU/Common/DepthSwizzle.h). The layout's constants come from the ID.
+void PixelJitCache::EmitDepthPointer(const PixelFuncID &id, X64Reg index, X64Reg temp) {
+	const DepthLayout layout = GetDepthLayout(DepthTranslationFromIndex(id.depthLayout), id.FBFormat() == GE_FORMAT_8888);
+	SHL(32, R(index), Imm8(1));
+	if (RipAccessible(&depthbuf.base)) {
+		ADD(32, R(index), M(&depthbuf.base));
+	} else {
+		MOV(PTRBITS, R(temp), ImmPtr(&depthbuf.base));
+		ADD(32, R(index), MatR(temp));
+	}
+	if (layout.rotMask != 0) {
+		// Rotate the field left by one: shifted to the bottom, its top bit carried around.
+		const int low = LeastSignificantSetBit(layout.rotMask);
+		const int width = layout.rotShift + 1;
+		MOV(32, R(temp), R(index));
+		AND(32, R(index), Imm32(~layout.rotMask));
+		SHR(32, R(temp), Imm8(low));
+		AND(32, R(temp), Imm32((1 << width) - 1));
+		SHL(32, R(temp), Imm8(1));
+		BTR(32, R(temp), Imm8(width));
+		ADC(32, R(temp), Imm8(0));
+		SHL(32, R(temp), Imm8(low));
+		OR(32, R(index), R(temp));
+	}
+	XOR(32, R(index), Imm32(layout.xorBits));
+	if (RipAccessible(&depthbuf.vram)) {
+		MOV(PTRBITS, R(temp), M(&depthbuf.vram));
+	} else {
+		MOV(PTRBITS, R(temp), ImmPtr(&depthbuf.vram));
+		MOV(PTRBITS, R(temp), MatR(temp));
+	}
+	LEA(PTRBITS, index, MRegSum(temp, index));
+}
+
 
 RegCache::Reg PixelJitCache::GetDestStencil(const PixelFuncID &id) {
 	// Skip if 565, since stencil is fixed zero.

GPU/Software/FuncId.cpp

@@ -22,6 +22,7 @@
 #include "GPU/Common/TextureDecoder.h"
 #include "GPU/GPUState.h"
 #include "GPU/Software/FuncId.h"
+#include "GPU/Software/SoftGpu.h"
 
 static_assert(sizeof(SamplerID) == sizeof(SamplerID::fullKey) + sizeof(SamplerID::cached) + sizeof(SamplerID::pad), "Bad sampler ID size");
 static_assert(sizeof(PixelFuncID) == sizeof(PixelFuncID::fullKey) + sizeof(PixelFuncID::cached), "Bad pixel func ID size");
@@ -182,17 +183,19 @@ void ComputePixelFuncID(PixelFuncID *id) {
 				id->alphaBlendDst = (uint8_t)OptimizeAlphaFactor(gstate.getFixB());
 		}
 
+		id->applyLogicOp = gstate.isLogicOpEnabled() && gstate.getLogicOp() != GE_LOGIC_COPY;
+		id->applyFog = gstate.isFogEnabled() && !gstate.isModeThrough();
+
 		if (id->colorTest && gstate.getColorTestFunction() == GE_COMP_NOTEQUAL && gstate.getColorTestRef() == 0 && gstate.getColorTestMask() == 0xFFFFFF) {
 			if (!id->depthWrite && !id->stencilTest && id->alphaBlend && id->AlphaBlendEq() == GE_BLENDMODE_MUL_AND_ADD) {
-				// Might be a pointless color test (seen in Ridge Racer, for example.)
-				if (id->AlphaBlendDst() == PixelBlendFactor::ONE)
+				// Rejecting black then only skips adding nothing to the destination, unless dithering, fog
+				// or a logic op would still change it: Ridge Racer 2 (UCES00422) adds a glow with dithering,
+				// and its black pixels must not get the dither.
+				if (id->AlphaBlendDst() == PixelBlendFactor::ONE && !id->dithering && !id->applyFog && !id->applyLogicOp)
 					id->colorTest = false;
 			}
 		}
 
-		id->applyLogicOp = gstate.isLogicOpEnabled() && gstate.getLogicOp() != GE_LOGIC_COPY;
-		id->applyFog = gstate.isFogEnabled() && !gstate.isModeThrough();
-
 		id->earlyZChecks = id->DepthTestFunc() != GE_COMP_ALWAYS;
 		if (id->stencilTest && id->earlyZChecks) {
 			// Can't do them early if stencil might need to write.
@@ -201,6 +204,8 @@ void ComputePixelFuncID(PixelFuncID *id) {
 		}
 	}
 
+	if (id->depthTestFunc != GE_COMP_ALWAYS || id->depthWrite)
+		id->depthLayout = DepthTranslationIndex(depthbuf.translation);
 	if (id->useStandardStride && (id->depthTestFunc != GE_COMP_ALWAYS || id->depthWrite))
 		id->useStandardStride = gstate.DepthBufStride() == 512;
 
@@ -446,6 +451,12 @@ std::string DescribePixelFuncID(const PixelFuncID &id) {
 	else if (id.clearMode)
 		desc = "INVALID:" + desc;
 
+	// DepthTranslationIndex() only gives 0 to 4.
+	if (id.depthLayout > 4)
+		desc = "INVALID:" + desc;
+	else if (id.depthLayout != 0)
+		desc += StringFromFormat("ZLayout%d:", (int)id.depthLayout);
+
 	if (desc.empty())
 		return "INVALID";
 	desc.resize(desc.size() - 1);

GPU/Software/FuncId.h

@@ -98,17 +98,20 @@ struct PixelFuncID {
 			bool hasStencilTestMask : 1;
 			bool dithering : 1;
 			bool applyLogicOp : 1;
-			// 48 bits before applyFog.
+			// 48 bits. MSVC doesn't let a bit field straddle a byte, so each byte below is filled exactly.
+			uint8_t sFail : 3;
+			uint8_t zFail : 3;
 			bool applyFog : 1;
 			// Meaning: fb_stride == 512 && z_stride == 512
 			bool useStandardStride : 1;
+			uint8_t zPass : 3;
 			// Meaning: maskRGB != 0 || maskA != 0
 			bool applyColorWriteMask : 1;
-			uint8_t sFail : 3;
-			uint8_t zFail : 3;
-			uint8_t zPass : 3;
 			bool earlyZChecks : 1;
-			// 61 bits, 3 free.
+			// The EDRAM address translation, as DepthTranslationIndex(), which with fbFormat picks how depth is
+			// stored (GPU/Common/DepthSwizzle.h).
+			uint8_t depthLayout : 3;
+			// 64 bits.
 		};
 	};
 

GPU/Software/Rasterizer.cpp

@@ -718,6 +718,14 @@ static inline void ApplyTexturing(const RasterizerState &state, Vec4<int> *prim_
 	}
 }
 
+// Depth at (x, y) and (x + 1, y). They're stored together unless x is last in a 16-pixel run.
+static inline u32 ReadDepthPair(int x, int y, int stride) {
+	const uint32_t offset = depthbuf.base + (uint32_t)(x + y * stride) * 2;
+	if ((offset & 31) != 30)
+		return *(const u32 *)depthbuf.Get16Ptr(x, y, stride);
+	return depthbuf.Get16(x, y, stride) | ((u32)depthbuf.Get16(x + 1, y, stride) << 16);
+}
+
 static inline Vec4<int> SOFTRAST_CALL CheckDepthTestPassed4(const Vec4<int> &mask, GEComparison func, int x, int y, int stride, Vec4<int> z) {
 	// Skip the depth buffer read if we're masked already.
 #if defined(_M_SSE)
@@ -734,8 +742,8 @@ static inline Vec4<int> SOFTRAST_CALL CheckDepthTestPassed4(const Vec4<int> &mas
 	// Read in the existing depth values.
 #if defined(_M_SSE)
 	// Tried using flags from maskbits to skip dwords... seemed neutral.
-	__m128i refz = _mm_cvtsi32_si128(*(u32 *)depthbuf.Get16Ptr(x, y, stride));
-	refz = _mm_unpacklo_epi32(refz, _mm_cvtsi32_si128(*(u32 *)depthbuf.Get16Ptr(x, y + 1, stride)));
+	__m128i refz = _mm_cvtsi32_si128(ReadDepthPair(x, y, stride));
+	refz = _mm_unpacklo_epi32(refz, _mm_cvtsi32_si128(ReadDepthPair(x, y + 1, stride)));
 	refz = _mm_unpacklo_epi16(refz, _mm_setzero_si128());
 #else
 	Vec4<int> refz(depthbuf.Get16(x, y, stride), depthbuf.Get16(x + 1, y, stride), depthbuf.Get16(x, y + 1, stride), depthbuf.Get16(x + 1, y + 1, stride));
@@ -1879,12 +1887,17 @@ void ClearRectangle(const VertexData &v0, const VertexData &v1, const BinCoords
 		const u16 z = v1.screenpos.z;
 		const int stride = pixelID.cached.depthbufStride;
 
-		// If both bytes of Z equal, we can just use memset directly which is faster.
+		// If both bytes of Z equal, we can just use memset directly which is faster. Depth is stored in runs of
+		// 16 pixels, each 32 bytes aligned.
 		if ((z & 0xFF) == (z >> 8)) {
 			DrawingCoords p = pprime;
 			for (p.y = pprime.y; p.y <= pend.y; ++p.y) {
-				u16 *row = depthbuf.Get16Ptr(p.x, p.y, stride);
-				memset(row, z, w * 2);
+				for (int x = 0; x < w; ) {
+					const uint32_t offset = depthbuf.base + (uint32_t)(pprime.x + x + p.y * stride) * 2;
+					const int n = std::min(w - x, 16 - (int)((offset >> 1) & 15));
+					memset(depthbuf.Get16Ptr(pprime.x + x, p.y, stride), z, n * 2);
+					x += n;
+				}
 			}
 		} else {
 			DrawingCoords p = pprime;

GPU/Software/SoftGpu.cpp

@@ -18,6 +18,7 @@
 #include <set>
 
 #include "Common/System/Display.h"
+#include "Common/Serialize/Serializer.h"
 #include "Common/GPU/OpenGL/GLFeatures.h"
 
 #include "GPU/GPUState.h"
@@ -54,7 +55,7 @@ constexpr int FB_HEIGHT = 272;
 
 uint8_t clut[1024];
 FormatBuffer fb;
-FormatBuffer depthbuf;
+DepthBuffer depthbuf;
 
 struct CommandInfo {
 	uint64_t flags;
@@ -397,7 +398,8 @@ SoftGPU::SoftGPU(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	: GPUCommon(gfxCtx, draw)
 {
 	fb.data = Memory::GetPointerWriteOrException(0x44000000); // TODO: correct default address?
-	depthbuf.data = Memory::GetPointerWriteOrException(0x44000000); // TODO: correct default address?
+	depthbuf.vram = Memory::GetPointerWriteOrException(0x04000000);
+	UpdateDepthBuffer();
 
 	memset(softgpuCmdInfo, 0, sizeof(softgpuCmdInfo));
 
@@ -473,6 +475,8 @@ void SoftGPU::DeviceRestore(Draw::DrawContext *draw) {
 }
 
 SoftGPU::~SoftGPU() {
+	// The firmware's default, until the next software renderer sets it.
+	Memory::g_depthTranslationIndex = DepthTranslationIndex(0x400);
 	if (fbTex) {
 		fbTex->Release();
 		fbTex = nullptr;
@@ -875,7 +879,10 @@ void SoftGPU::Execute_BlockTransferStart(u32 op, u32 diff) {
 	const uint32_t dstSize = (height - 1) * (dstStride + width) * bpp;
 
 	// Need to flush both source and target, so we overwrite properly.
-	if (Memory::IsValidRange(src, srcSize) && Memory::IsValidRange(dst, dstSize)) {
+	if (Memory::IsDepthTexVRAMAddress(srcBasePtr) || Memory::IsDepthTexVRAMAddress(dstBasePtr)) {
+		// Through the depth layout (CopyBlockTransfer), which the ranges don't follow.
+		drawEngine_->transformUnit.Flush(this, "blockxfer_depth");
+	} else if (Memory::IsValidRange(src, srcSize) && Memory::IsValidRange(dst, dstSize)) {
 		drawEngine_->transformUnit.FlushIfOverlap(this, "blockxfer", false, src, srcStride * bpp, width * bpp, height);
 		drawEngine_->transformUnit.FlushIfOverlap(this, "blockxfer", true, dst, dstStride * bpp, width * bpp, height);
 	} else {
@@ -888,6 +895,47 @@ void SoftGPU::Execute_BlockTransferStart(u32 op, u32 diff) {
 	MarkDirty(dst, dstSize, SoftGPUVRAMDirty::DIRTY | SoftGPUVRAMDirty::REALLY_DIRTY);
 }
 
+// A block transfer from or to VRAM's 0x04200000 or 0x04600000 mirror goes through the depth layout there, as the
+// CPU's accesses do: Hayate no Gotoku restores its depth buffer through 0x04600000 (#17878).
+bool SoftGPU::CopyBlockTransfer(u32 srcBasePtr, u32 srcStride, int srcX, int srcY, u32 dstBasePtr, u32 dstStride, int dstX, int dstY, int width, int height, int bpp) {
+	if (!Memory::IsDepthTexVRAMAddress(srcBasePtr) && !Memory::IsDepthTexVRAMAddress(dstBasePtr))
+		return false;
+	const DepthLayout layout16 = GetDepthLayout(GetAddrTranslation(), false);
+	const DepthLayout layout32 = GetDepthLayout(GetAddrTranslation(), true);
+	// The memory of each run of a row, which the layout keeps together within 32 aligned bytes. Past VRAM's end, a
+	// transfer wraps to its start, which isn't a swizzled mirror.
+	auto run = [&](u32 addr) -> u8 * {
+		if ((addr & 0x04800000) == 0x04800000)
+			addr &= ~0x00800000;
+		if (Memory::IsDepthTexVRAMAddress(addr))
+			return depthbuf.vram + ((addr & 0x00400000) ? layout32 : layout16).Stored(addr & 0x001FFFFF);
+		return Memory::IsValidAddress(addr) ? Memory::GetPointerWriteUnchecked(addr) : nullptr;
+	};
+	const u32 bytes = width * bpp;
+	std::vector<u8> row(bytes);
+	for (int y = 0; y < height; ++y) {
+		const u32 srcRow = srcBasePtr + ((y + srcY) * srcStride + srcX) * bpp;
+		const u32 dstRow = dstBasePtr + ((y + dstY) * dstStride + dstX) * bpp;
+		for (u32 pos = 0; pos < bytes; ) {
+			const u32 n = std::min(32 - ((srcRow + pos) & 31), bytes - pos);
+			const u8 *p = run(srcRow + pos);
+			if (p)
+				memcpy(row.data() + pos, p, n);
+			else
+				memset(row.data() + pos, 0, n);
+			pos += n;
+		}
+		for (u32 pos = 0; pos < bytes; ) {
+			const u32 n = std::min(32 - ((dstRow + pos) & 31), bytes - pos);
+			u8 *p = run(dstRow + pos);
+			if (p)
+				memcpy(p, row.data() + pos, n);
+			pos += n;
+		}
+	}
+	return true;
+}
+
 void SoftGPU::Execute_Prim(u32 op, u32 diff) {
 	u32 count = op & 0xFFFF;
 	// Upper bits are ignored.
@@ -1082,8 +1130,11 @@ void SoftGPU::Execute_FramebufPtr(u32 op, u32 diff) {
 
 void SoftGPU::Execute_FramebufFormat(u32 op, u32 diff) {
 	// We should flush, because ranges within bins may change.
-	if (diff)
+	if (diff) {
 		drawEngine_->transformUnit.Flush(this, "framebuf");
+		// The depth layout depends on the color format.
+		UpdateDepthBuffer();
+	}
 }
 
 void SoftGPU::Execute_TexFlush(u32 op, u32 diff) {
@@ -1091,15 +1142,43 @@ void SoftGPU::Execute_TexFlush(u32 op, u32 diff) {
 }
 
 void SoftGPU::Execute_ZbufPtr(u32 op, u32 diff) {
-	// We assume depthbuf.data won't change while we're drawing.
+	// We assume depthbuf won't change while we're drawing.
 	if (diff) {
 		drawEngine_->transformUnit.Flush(this, "depthbuf");
-		// For the pointer, ignore memory mirrors.  This also gives some buffer for draws that go outside.
-		// TODO: Confirm how wrapping is handled in drawing.  Adjust if we ever handle VRAM mirrors more accurately.
-		depthbuf.data = Memory::GetPointerWriteOrException(gstate.getDepthBufAddress() & 0x041FFFF0);
+		UpdateDepthBuffer();
 	}
 }
 
+// Call with nothing queued: the threads draw with depthbuf.
+void SoftGPU::UpdateDepthBuffer() {
+	// The GE ignores the mirror bits of its own pointers (exp89). Draws that go outside reach the mirrors,
+	// which map back to VRAM.
+	depthbuf.base = gstate.getDepthBufAddress() & 0x001FFFF0;
+	depthbuf.translation = GetAddrTranslation();
+	depthbuf.layout = GetDepthLayout(depthbuf.translation, gstate.FrameBufFormat() == GE_FORMAT_8888);
+	Memory::g_depthTranslationIndex = DepthTranslationIndex(depthbuf.translation);
+}
+
+void SoftGPU::DoState(PointerWrap &p) {
+	GPUCommon::DoState(p);
+	// The translation comes back without SetAddrTranslation.
+	if (p.mode == PointerWrap::MODE_READ)
+		UpdateDepthBuffer();
+}
+
+uint32_t SoftGPU::SetAddrTranslation(uint32_t value) {
+	if (value != GetAddrTranslation()) {
+		// Queued drawing uses the old layout.
+		drawEngine_->transformUnit.Flush(this, "translation");
+		const uint32_t previous = GPUCommon::SetAddrTranslation(value);
+		UpdateDepthBuffer();
+		dirtyFlags_ |= SoftDirty::PIXEL_BASIC;
+		drawEngine_->transformUnit.SetDirty(SoftDirty::PIXEL_BASIC);
+		return previous;
+	}
+	return GPUCommon::SetAddrTranslation(value);
+}
+
 void SoftGPU::Execute_VertexType(u32 op, u32 diff) {
 	if ((diff & GE_VTYPE_THROUGH_MASK) != 0) {
 		// This affects a lot of things, but some don't matter if it's off - so defer to when it's back on.
@@ -1464,13 +1543,10 @@ bool SoftGPU::GetCurrentDepthbuffer(GPUDebugBuffer &buffer) {
 	DrawingCoords size = GetTargetSize(gstate.DepthBufStride());
 	buffer.Allocate(size.x, size.y, GPU_DBG_FORMAT_16BIT);
 
-	const int depth = 2;
-	const u8 *src = depthbuf.data;
-	u8 *dst = buffer.GetData();
+	u16 *dst = (u16 *)buffer.GetData();
 	for (int16_t y = 0; y < size.y; ++y) {
-		memcpy(dst, src, size.x * depth);
-		dst += size.x * depth;
-		src += gstate.DepthBufStride() * depth;
+		for (int16_t x = 0; x < size.x; ++x)
+			*dst++ = depthbuf.Get16(x, y, gstate.DepthBufStride());
 	}
 	return true;
 }

GPU/Software/SoftGpu.h

@@ -20,7 +20,7 @@
 #include <cstdint>
 
 #include "GPU/GPUCommon.h"
-#include "GPU/GPUCommon.h"
+#include "GPU/Common/DepthSwizzle.h"
 #include "Common/GPU/thin3d.h"
 
 struct FormatBuffer {
@@ -56,6 +56,26 @@ struct FormatBuffer {
 	}
 };
 
+// The depth buffer, stored in VRAM as the GE stores it (GPU/Common/DepthSwizzle.h).
+struct DepthBuffer {
+	// VRAM's start, and the buffer's offset in it.
+	u8 *vram = nullptr;
+	uint32_t base = 0;
+	DepthLayout layout;
+	// sceGeEdramSetAddrTranslation's value, which with the color format picks the layout.
+	uint32_t translation = 0x400;
+
+	inline u16 *Get16Ptr(int x, int y, int stride) const {
+		return (u16 *)(vram + layout.Stored(base + (uint32_t)(x + y * stride) * 2));
+	}
+	inline u16 Get16(int x, int y, int stride) const {
+		return *Get16Ptr(x, y, stride);
+	}
+	inline void Set16(int x, int y, int stride, u16 v) const {
+		*Get16Ptr(x, y, stride) = v;
+	}
+};
+
 enum class SoftDirty : uint64_t {
 	NONE = 0,
 
@@ -137,6 +157,9 @@ public:
 	void UpdateCmdInfo() override {}
 
 	void SetDisplayFramebuffer(u32 framebuf, u32 stride, GEBufferFormat format) override;
+	uint32_t SetAddrTranslation(uint32_t value) override;
+	bool CopyBlockTransfer(u32 srcBasePtr, u32 srcStride, int srcX, int srcY, u32 dstBasePtr, u32 dstStride, int dstX, int dstY, int width, int height, int bpp) override;
+	void DoState(PointerWrap &p) override;
 	void SetCurFramebufferDirty(bool dirty) override {}
 	void PrepareCopyDisplayToOutput(const DisplayLayoutConfig &config) override;
 	void CopyDisplayToOutput(const DisplayLayoutConfig &config) override;
@@ -184,6 +207,7 @@ public:
 	void Execute_FramebufPtr(u32 op, u32 diff);
 	void Execute_FramebufFormat(u32 op, u32 diff);
 	void Execute_ZbufPtr(u32 op, u32 diff);
+	void UpdateDepthBuffer();
 	void Execute_TexFlush(u32 op, u32 diff);
 	void Execute_VertexType(u32 op, u32 diff);
 
@@ -247,7 +271,7 @@ private:
 // TODO: These shouldn't be global.
 extern uint8_t clut[1024];
 extern FormatBuffer fb;
-extern FormatBuffer depthbuf;
+extern DepthBuffer depthbuf;
 
 // Type for the DarkStalkers stretch replacement.
 enum class DSStretch {

UWP/GPU_UWP/GPU_UWP.vcxproj

@@ -86,6 +86,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="..\..\GPU\Common\DepthRaster.h" />
+    <ClInclude Include="..\..\GPU\Common\DepthSwizzle.h" />
     <ClInclude Include="..\..\GPU\Common\ReplacedTexture.h" />
     <ClInclude Include="..\..\GPU\Common\TextureReplacer.h" />
     <ClInclude Include="..\..\GPU\Common\TextureShaderCommon.h" />

UWP/GPU_UWP/GPU_UWP.vcxproj.filters

@@ -165,6 +165,7 @@
       <Filter>Debugger</Filter>
     </ClInclude>
     <ClInclude Include="..\..\GPU\Common\DepthRaster.h" />
+    <ClInclude Include="..\..\GPU\Common\DepthSwizzle.h" />
   </ItemGroup>
   <ItemGroup>
     <Filter Include="Debugger">

unittest/TestSoftwareGPUJit.cpp

@@ -129,11 +129,14 @@ static bool TestPixelJit() {
 	bool header = false;
 
 	u32 *fb_data = new u32[512 * 2];
-	u16 *zb_data = new u16[512 * 2];
+	// Random IDs pick random depth layouts, which reach up to 0x8040 bytes past the pixel's offset.
+	const size_t zbSize = 0x10000;
+	u16 *zb_data = new u16[zbSize];
 	fb.as32 = fb_data;
-	depthbuf.as16 = zb_data;
+	depthbuf.vram = (u8 *)zb_data;
+	depthbuf.base = 0;
 	memset(fb_data, 0, sizeof(u32) * 512 * 2);
-	memset(zb_data, 0, sizeof(u16) * 512 * 2);
+	memset(zb_data, 0, sizeof(u16) * zbSize);
 
 	for (int i = 0; i < count; ) {
 		PixelFuncID id;

unittest/UnitTest.cpp

@@ -115,6 +115,7 @@
 #include "Core/HLE/sceCtrl.h"
 #include "Core/Util/PathUtil.h"
 #include "Core/MIPS/MIPSVFPUUtils.h"
+#include "GPU/Common/DepthSwizzle.h"
 #include "GPU/Common/TextureDecoder.h"
 #include "GPU/Common/GPUStateUtils.h"
 #include "GPU/Math3D.h"
@@ -2437,6 +2438,71 @@ static bool TestWrapText() {
 	return true;
 }
 
+// Against ppsspp-re geprobe's swz.py, the layout that fits exp8-exp10.
+static bool TestDepthSwizzle() {
+	static const struct {
+		uint32_t translation;
+		bool color32;
+		uint32_t linear;
+		uint32_t stored;
+	} cases[] = {
+		{ 0x0, false, 0x088000, 0x088600 },
+		{ 0x0, false, 0x08a0e2, 0x08a6e2 },
+		{ 0x0, false, 0x1ffffe, 0x1ff9fe },
+		{ 0x0, false, 0x14b9ac, 0x14bfac },
+		{ 0x0, false, 0x09a782, 0x09a182 },
+		{ 0x0, true, 0x088000, 0x088600 },
+		{ 0x0, true, 0x08a0e2, 0x08a6e2 },
+		{ 0x0, true, 0x1ffffe, 0x1ff9fe },
+		{ 0x0, true, 0x1944c8, 0x1944c8 },
+		{ 0x0, true, 0x03171e, 0x03111e },
+		{ 0x200, false, 0x088000, 0x089040 },
+		{ 0x200, false, 0x08a0e2, 0x08b0a2 },
+		{ 0x200, false, 0x1ffffe, 0x1fefbe },
+		{ 0x200, false, 0x04a2ca, 0x04b28a },
+		{ 0x200, false, 0x06063a, 0x06167a },
+		{ 0x200, true, 0x088000, 0x089040 },
+		{ 0x200, true, 0x08a0e2, 0x08b182 },
+		{ 0x200, true, 0x1ffffe, 0x1fefbe },
+		{ 0x200, true, 0x176772, 0x1776b2 },
+		{ 0x200, true, 0x03b640, 0x03a6c0 },
+		{ 0x400, false, 0x088000, 0x08a040 },
+		{ 0x400, false, 0x08a0e2, 0x0880a2 },
+		{ 0x400, false, 0x1ffffe, 0x1fdfbe },
+		{ 0x400, false, 0x0dbd9c, 0x0d9ddc },
+		{ 0x400, false, 0x026654, 0x024614 },
+		{ 0x400, true, 0x088000, 0x08a040 },
+		{ 0x400, true, 0x08a0e2, 0x088182 },
+		{ 0x400, true, 0x1ffffe, 0x1fdfbe },
+		{ 0x400, true, 0x058028, 0x05a008 },
+		{ 0x400, true, 0x1bc0d8, 0x1be1d8 },
+		{ 0x800, false, 0x088000, 0x08c040 },
+		{ 0x800, false, 0x08a0e2, 0x08e0a2 },
+		{ 0x800, false, 0x1ffffe, 0x1fbfbe },
+		{ 0x800, false, 0x1ac354, 0x1a8314 },
+		{ 0x800, false, 0x047882, 0x0438c2 },
+		{ 0x800, true, 0x088000, 0x08c040 },
+		{ 0x800, true, 0x08a0e2, 0x08e182 },
+		{ 0x800, true, 0x1ffffe, 0x1fbfbe },
+		{ 0x800, true, 0x0f6704, 0x0f2664 },
+		{ 0x800, true, 0x05ce3c, 0x058c3c },
+		{ 0x1000, false, 0x088000, 0x080040 },
+		{ 0x1000, false, 0x08a0e2, 0x0820a2 },
+		{ 0x1000, false, 0x1ffffe, 0x1f7fbe },
+		{ 0x1000, false, 0x1b2b52, 0x1bab12 },
+		{ 0x1000, false, 0x03c876, 0x034836 },
+		{ 0x1000, true, 0x088000, 0x080040 },
+		{ 0x1000, true, 0x08a0e2, 0x082182 },
+		{ 0x1000, true, 0x1ffffe, 0x1f7fbe },
+		{ 0x1000, true, 0x07ec5e, 0x0768fe },
+		{ 0x1000, true, 0x0e498c, 0x0ec36c },
+	};
+	for (const auto &c : cases) {
+		EXPECT_EQ_HEX(GetDepthLayout(c.translation, c.color32).Stored(c.linear), c.stored);
+	}
+	return true;
+}
+
 static bool TestSmallDataConvert() {
 	float f[4] = { 1.0f / 255.0f, 2.0f / 255.0f, 3.0f / 255.0f, 4.0f / 255.f };
 	uint32_t result = Float4ToUint8x4_NoClamp(f);
@@ -3191,6 +3257,7 @@ TestItem availableTests[] = {
 	TEST_ITEM(MpegCsc),
 	TEST_ITEM(SplineTessellation),
 	TEST_ITEM(GEMath),
+	TEST_ITEM(DepthSwizzle),
 	TEST_ITEM(Demangle),
 	TEST_ITEM(TextureReplacer),
 	TEST_ITEM(UITabOrder),