GPU: Rasterize software depth on a worker thread

The depth draws were queued and rasterized on the emu thread at each render target change, in
Wipeout about 40 small batches a frame, and around 40% of that thread's time. Now a render target
change hands the batch to a worker thread and carries on. Waiting for it is left to whatever reads
the depth buffers: DrawSync and ListSync, block transfers, framebuffer copies, showing a frame,
savestates and shutdown, and the texture cache when the texture or CLUT it reads overlaps a depth
buffer still being written.

Batches grow as needed instead of being preallocated at full size. A full batch is now handed over
and a new one started, where the draw used to be dropped.

Wipeout headless, output unchanged: 16% less wall time on a Galaxy S8 (8.2 s to 6.9 s for 10
emulated seconds). On an M1, where headless waits on the GPU, the emu thread uses 38% less CPU time.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Core/System.cpp

@@ -652,6 +652,11 @@ void CPU_Shutdown(bool success) {
 	// Core_LockAgainstShutdown(); it's recursive, so the nested acquire in Memory::Shutdown() is fine.
 	CoreShutdownLock coreLock = Core_LockAgainstShutdown();
 
+	// The GPU outlives memory (GPU_Shutdown comes later), so it must stop writing to it now.
+	if (gpu) {
+		gpu->FlushPendingDrawing();
+	}
+
 	UninstallExceptionHandler();
 
 	GPURecord::Replay_Unload();

GPU/Common/DepthRaster.cpp

@@ -7,6 +7,10 @@
 #include "GPU/Math3D.h"
 #include "Common/Math/math_util.h"
 #include "GPU/Common/VertexDecoderCommon.h"
+#include "Common/MemoryUtil.h"
+#include "Common/Thread/ThreadUtil.h"
+#include "Common/TimeUtil.h"
+#include "Core/MemMap.h"
 
 DepthScissor DepthScissor::Tile(int tile, int numTiles) const {
 	if (numTiles == 1) {
@@ -634,3 +638,209 @@ void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, con
 		_dbg_assert_(false);
 	}
 }
+
+// The depth buffers are all in VRAM, so busy ranges are kept as offsets into it, which also covers
+// the mirrors.
+static constexpr u32 VRAM_OFFSET_MASK = 0x001FFFFF;
+
+DepthRasterWorker::DepthRasterWorker(int maxVerts, int maxIndices) : maxVerts_(maxVerts), maxIndices_(maxIndices) {
+	// x, y and z, each with room for a group of four triangles past the end.
+	screenVerts_ = (int *)AllocateMemoryPages((maxVerts_ * sizeof(int) + 384) * 3, MEM_PROT_READ | MEM_PROT_WRITE);
+	current_ = NewBatch();
+	thread_ = std::thread([this] { Run(); });
+}
+
+DepthRasterWorker::~DepthRasterWorker() {
+	Wait();
+	{
+		std::lock_guard<std::mutex> guard(mutex_);
+		quit_ = true;
+	}
+	work_.notify_one();
+	thread_.join();
+	for (DepthBatch *batch : all_) {
+		FreeBatch(batch);
+	}
+	FreeMemoryPages(screenVerts_, (maxVerts_ * sizeof(int) + 384) * 3);
+}
+
+DepthBatch *DepthRasterWorker::NewBatch() {
+	DepthBatch *batch = new DepthBatch();
+	batch->draws.reserve(16);
+	all_.push_back(batch);
+	return batch;
+}
+
+void DepthRasterWorker::FreeBatch(DepthBatch *batch) {
+	FreeAlignedMemory(batch->transformed);
+	FreeAlignedMemory(batch->indices);
+	delete batch;
+}
+
+void DepthRasterWorker::Reserve(int vertices, int indices) {
+	DepthBatch &batch = *current_;
+	const int needVerts = batch.vertexCount + vertices;
+	if (needVerts > batch.vertexCapacity) {
+		const int capacity = std::min(std::max(needVerts, batch.vertexCapacity * 2), maxVerts_);
+		float *grown = (float *)AllocateAlignedMemory(capacity * 4 * sizeof(float), 16);
+		if (batch.transformed) {
+			memcpy(grown, batch.transformed, batch.vertexCount * 4 * sizeof(float));
+			FreeAlignedMemory(batch.transformed);
+		}
+		batch.transformed = grown;
+		batch.vertexCapacity = capacity;
+	}
+	const int needIndices = batch.indexCount + indices;
+	if (needIndices > batch.indexCapacity) {
+		const int capacity = std::min(std::max(needIndices, batch.indexCapacity * 2), maxIndices_);
+		uint16_t *grown = (uint16_t *)AllocateAlignedMemory(capacity * sizeof(uint16_t), 16);
+		if (batch.indices) {
+			memcpy(grown, batch.indices, batch.indexCount * sizeof(uint16_t));
+			FreeAlignedMemory(batch.indices);
+		}
+		batch.indices = grown;
+		batch.indexCapacity = capacity;
+	}
+}
+
+void DepthRasterWorker::Submit() {
+	if (current_->draws.empty()) {
+		return;
+	}
+	// Wipeout submits about 40 a frame. Past this many, wait for one to come back.
+	constexpr size_t MAX_BATCHES = 128;
+	std::unique_lock<std::mutex> lock(mutex_);
+	for (const DepthDraw &draw : current_->draws) {
+		const u32 low = (draw.depthAddr & VRAM_OFFSET_MASK) + draw.scissor.y1 * draw.depthStride * 2;
+		const u32 high = (draw.depthAddr & VRAM_OFFSET_MASK) + (draw.scissor.y2 + 1) * draw.depthStride * 2;
+		if (!busy_) {
+			busyRanges_.clear();
+			busyLow_ = low;
+			busyHigh_ = high;
+			busy_ = true;
+		} else {
+			busyLow_ = std::min((u32)busyLow_, low);
+			busyHigh_ = std::max((u32)busyHigh_, high);
+		}
+		bool merged = false;
+		for (auto &range : busyRanges_) {
+			if (low <= range.second && high >= range.first) {
+				range.first = std::min(range.first, low);
+				range.second = std::max(range.second, high);
+				merged = true;
+				break;
+			}
+		}
+		if (!merged) {
+			busyRanges_.emplace_back(low, high);
+		}
+	}
+	queue_.push_back(current_);
+	work_.notify_one();
+	if (free_.empty() && all_.size() < MAX_BATCHES) {
+		lock.unlock();
+		current_ = NewBatch();
+		return;
+	}
+	done_.wait(lock, [this] { return !free_.empty(); });
+	current_ = free_.back();
+	free_.pop_back();
+	current_->draws.clear();
+	current_->vertexCount = 0;
+	current_->indexCount = 0;
+}
+
+void DepthRasterWorker::Wait() {
+	Submit();
+	if (!busy_) {
+		return;
+	}
+	std::unique_lock<std::mutex> lock(mutex_);
+	done_.wait(lock, [this] { return queue_.empty() && running_ == 0; });
+}
+
+void DepthRasterWorker::WaitIfWriting(u32 addr, u32 size) {
+	if (!busy_) {
+		return;
+	}
+	const u32 low = addr & VRAM_OFFSET_MASK;
+	if (low >= busyHigh_ || low + size <= busyLow_) {
+		return;
+	}
+	std::unique_lock<std::mutex> lock(mutex_);
+	bool overlaps = false;
+	for (const auto &range : busyRanges_) {
+		overlaps = overlaps || (low < range.second && low + size > range.first);
+	}
+	if (busy_ && overlaps) {
+		done_.wait(lock, [this] { return queue_.empty() && running_ == 0; });
+	}
+}
+
+void DepthRasterWorker::TakeTimes(double *cull, double *raster) {
+	std::lock_guard<std::mutex> guard(mutex_);
+	*cull += cullTime_;
+	*raster += rasterTime_;
+	cullTime_ = 0.0;
+	rasterTime_ = 0.0;
+}
+
+void DepthRasterWorker::Run() {
+	SetCurrentThreadName("DepthRaster");
+	std::unique_lock<std::mutex> lock(mutex_);
+	while (true) {
+		work_.wait(lock, [this] { return quit_ || !queue_.empty(); });
+		if (quit_) {
+			break;
+		}
+		DepthBatch *batch = queue_.front();
+		queue_.erase(queue_.begin());
+		running_++;
+		lock.unlock();
+
+		double cull = 0.0, raster = 0.0;
+		Rasterize(*batch, &cull, &raster);
+
+		lock.lock();
+		running_--;
+		cullTime_ += cull;
+		rasterTime_ += raster;
+		free_.push_back(batch);
+		if (queue_.empty() && running_ == 0) {
+			busy_ = false;
+		}
+		done_.notify_all();
+	}
+}
+
+void DepthRasterWorker::Rasterize(const DepthBatch &batch, double *cull, double *raster) {
+	int *tx = screenVerts_;
+	int *ty = screenVerts_ + maxVerts_;
+	float *tz = (float *)(screenVerts_ + maxVerts_ * 2);
+	for (const DepthDraw &draw : batch.draws) {
+		const float *vertices = batch.transformed + 4 * draw.vertexOffset;
+		const uint16_t *indices = batch.indices + draw.indexOffset;
+		const DepthScissor tileScissor = draw.scissor.Tile(0, 1);
+
+		double start = time_now_d();
+		int outVertCount = 0;
+		switch (draw.prim) {
+		case GE_PRIM_RECTANGLES:
+			outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, vertices, indices, draw, tileScissor);
+			break;
+		case GE_PRIM_TRIANGLES:
+			outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, vertices, indices, draw, tileScissor, maxVerts_);
+			break;
+		default:
+			_dbg_assert_(false);
+			break;
+		}
+		double mid = time_now_d();
+		*cull += mid - start;
+		if (outVertCount > 0 && Memory::IsValid4AlignedAddress(draw.depthAddr)) {
+			u16 *depthPtr = (uint16_t *)Memory::GetPointerWriteUnchecked(draw.depthAddr);
+			DepthRasterScreenVerts(depthPtr, draw.depthStride, tx, ty, tz, outVertCount, draw, tileScissor, batch.lowQ);
+			*raster += time_now_d() - mid;
+		}
+	}
+}

GPU/Common/DepthRaster.h

@@ -1,5 +1,11 @@
 #pragma once
 
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <thread>
+#include <vector>
+
 #include "Common/CommonTypes.h"
 #include "GPU/ge_constants.h"
 
@@ -58,3 +64,71 @@ void DecodeAndTransformForDepthRaster(float *dest, const float *worldviewproj, c
 void TransformPredecodedForDepthRaster(float *dest, const float *worldviewproj, const void *decodedVertexData, const VertexDecoder *dec, int count);
 void ConvertPredecodedThroughForDepthRaster(float *dest, const void *decodedVertexData, const VertexDecoder *dec, int count);
 void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, const int *ty, const float *tz, int count, const DepthDraw &draw, const DepthScissor scissor, bool lowQ);
+
+// A batch of queued depth draws, with their transformed vertices and indices.
+struct DepthBatch {
+	float *transformed = nullptr;
+	uint16_t *indices = nullptr;
+	std::vector<DepthDraw> draws;
+	int vertexCount = 0;
+	int indexCount = 0;
+	int vertexCapacity = 0;
+	int indexCapacity = 0;
+	bool lowQ = false;
+};
+
+// Rasterizes batches of depth draws on a thread of its own, in the order they're submitted, so
+// that it overlaps with emulating the CPU and the rest of the GE. The PSP memory it writes is only
+// the depth buffers of the draws, so anything reading those has to Wait() first (or WaitIfWriting).
+class DepthRasterWorker {
+public:
+	DepthRasterWorker(int maxVerts, int maxIndices);
+	~DepthRasterWorker();
+
+	// The batch to add draws to.
+	DepthBatch &Current() { return *current_; }
+	// Makes room in the current batch for this many more vertices and indices.
+	void Reserve(int vertices, int indices);
+	// Starts rasterizing the current batch (if it has draws) and makes an empty one current.
+	void Submit();
+	// Submits the current batch and waits until all are rasterized.
+	void Wait();
+	// Waits only if a batch being rasterized may write to [addr, addr + size).
+	void WaitIfWriting(u32 addr, u32 size);
+
+	// Time spent culling and rasterizing since the last call, for the stats.
+	void TakeTimes(double *cull, double *raster);
+
+private:
+	DepthBatch *NewBatch();
+	static void FreeBatch(DepthBatch *batch);
+	void Run();
+	void Rasterize(const DepthBatch &batch, double *cull, double *raster);
+
+	const int maxVerts_;
+	const int maxIndices_;
+	int *screenVerts_ = nullptr;
+
+	std::vector<DepthBatch *> all_;
+	std::vector<DepthBatch *> free_;
+	std::vector<DepthBatch *> queue_;  // Oldest first.
+	DepthBatch *current_ = nullptr;
+	int running_ = 0;  // Batches taken off the queue and not yet done.
+
+	// The depth buffer bytes the submitted, unfinished batches can write.
+	// The depth buffers being written (VRAM offsets, end exclusive), and their span, which only grows
+	// while busy_, so it can be checked without the lock.
+	std::vector<std::pair<u32, u32>> busyRanges_;
+	std::atomic<u32> busyLow_{};
+	std::atomic<u32> busyHigh_{};
+	std::atomic<bool> busy_{};
+
+	double cullTime_ = 0.0;
+	double rasterTime_ = 0.0;
+
+	bool quit_ = false;
+	std::mutex mutex_;
+	std::condition_variable work_;
+	std::condition_variable done_;
+	std::thread thread_;
+};

GPU/Common/DrawEngineCommon.cpp

@@ -1126,20 +1126,15 @@ bool DrawEngineCommon::DescribeCodePtr(const u8 *ptr, std::string &name) const {
 
 enum {
 	DEPTH_TRANSFORMED_MAX_VERTS = VERTEX_BUFFER_MAX,
-	DEPTH_TRANSFORMED_BYTES = DEPTH_TRANSFORMED_MAX_VERTS * 4 * sizeof(float),
-	DEPTH_SCREENVERTS_COMPONENT_COUNT = VERTEX_BUFFER_MAX,
-	DEPTH_SCREENVERTS_COMPONENT_BYTES = DEPTH_SCREENVERTS_COMPONENT_COUNT * sizeof(int) + 384,
-	DEPTH_SCREENVERTS_TOTAL_BYTES = DEPTH_SCREENVERTS_COMPONENT_BYTES * 3,
 	DEPTH_INDEXBUFFER_MAX_INDICES = DEPTH_TRANSFORMED_MAX_VERTS * 3,
-	DEPTH_INDEXBUFFER_BYTES = DEPTH_INDEXBUFFER_MAX_INDICES * sizeof(uint16_t),
 };
 
 // We process vertices for depth rendering in several stages:
-// First, we transform and collect vertices into depthTransformed_ (4-vectors, xyzw).
+// First, we transform and collect vertices into a DepthBatch (4-vectors, xyzw).
 // Then, we group and cull the vertices into four-triangle groups, which are placed in
-// depthScreenVerts_, with x, y and z separated into different part of the array.
+// the worker's screen vertices, with x, y and z separated into different part of the array.
 // (Alternatively, if drawing rectangles, they're just added linearly).
-// After that, we send these groups out for SIMD setup and rasterization.
+// After that, the worker sends these groups out for SIMD setup and rasterization.
 void DrawEngineCommon::InitDepthRaster() {
 	switch ((DepthRasterMode)g_Config.iDepthRasterMode) {
 	case DepthRasterMode::DEFAULT:
@@ -1154,23 +1149,13 @@ void DrawEngineCommon::InitDepthRaster() {
 	}
 
 	if (useDepthRaster_) {
-		depthDraws_.reserve(256);
-		depthTransformed_ = (float *)AllocateMemoryPages(DEPTH_TRANSFORMED_BYTES, MEM_PROT_READ | MEM_PROT_WRITE);
-		depthScreenVerts_ = (int *)AllocateMemoryPages(DEPTH_SCREENVERTS_TOTAL_BYTES, MEM_PROT_READ | MEM_PROT_WRITE);
-		depthIndices_ = (uint16_t *)AllocateMemoryPages(DEPTH_INDEXBUFFER_BYTES, MEM_PROT_READ | MEM_PROT_WRITE);
+		depthWorker_ = new DepthRasterWorker(DEPTH_TRANSFORMED_MAX_VERTS, DEPTH_INDEXBUFFER_MAX_INDICES);
 	}
 }
 
 void DrawEngineCommon::ShutdownDepthRaster() {
-	if (depthTransformed_) {
-		FreeMemoryPages(depthTransformed_, DEPTH_TRANSFORMED_BYTES);
-	}
-	if (depthScreenVerts_) {
-		FreeMemoryPages(depthScreenVerts_, DEPTH_SCREENVERTS_TOTAL_BYTES);
-	}
-	if (depthIndices_) {
-		FreeMemoryPages(depthIndices_, DEPTH_INDEXBUFFER_BYTES);
-	}
+	delete depthWorker_;
+	depthWorker_ = nullptr;
 }
 
 Mat4F32 ComputeFinalProjMatrix() {
@@ -1191,7 +1176,7 @@ Mat4F32 ComputeFinalProjMatrix() {
 	return m;
 }
 
-// numDecoded is how many vertices this draw will write to depthTransformed_, which is not the
+// numDecoded is how many vertices this draw will write to the batch, which is not the
 // same thing as vertexCount (the number of indices) - an indexed draw can decode far more
 // vertices than it has indices, or far fewer.
 bool DrawEngineCommon::CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, int vertexCount, int numDecoded) {
@@ -1240,18 +1225,21 @@ bool DrawEngineCommon::CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim,
 		_dbg_assert_(gstate.isDepthWriteEnabled());
 	}
 
-	if (depthVertexCount_ + numDecoded > DEPTH_TRANSFORMED_MAX_VERTS) {
-		// Can't add more. We need to flush.
+	if (numDecoded > DEPTH_TRANSFORMED_MAX_VERTS || vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) {
 		return false;
 	}
-	if (depthIndexCount_ + vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) {
-		return false;
+	const DepthBatch *batch = &depthWorker_->Current();
+	if (batch->vertexCount + numDecoded > DEPTH_TRANSFORMED_MAX_VERTS || batch->indexCount + vertexCount > DEPTH_INDEXBUFFER_MAX_INDICES) {
+		// Full, start on another.
+		depthWorker_->Submit();
+		batch = &depthWorker_->Current();
 	}
 
 	draw->depthAddr = gstate.getDepthBufRawAddress() | 0x04000000;
 	draw->depthStride = gstate.DepthBufStride();
-	draw->vertexOffset = depthVertexCount_;
-	draw->indexOffset = depthIndexCount_;
+	depthWorker_->Reserve(numDecoded, vertexCount);
+	draw->vertexOffset = batch->vertexCount;
+	draw->indexOffset = batch->indexCount;
 	draw->vertexCount = vertexCount;
 	draw->cullEnabled = gstate.isCullEnabled();
 	draw->cullMode = gstate.getCullMode();
@@ -1289,6 +1277,7 @@ void DrawEngineCommon::DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDe
 	if (!CalculateDepthDraw(&draw, prim, vertexCount, willDecode)) {
 		return;
 	}
+	DepthBatch &batch = depthWorker_->Current();
 
 	TimeCollector collectStat(&gpuStats.perFrame.msPrepareDepth, g_coreCollectDebugStats);
 
@@ -1302,24 +1291,14 @@ void DrawEngineCommon::DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDe
 			break;
 		}
 		// Decode the verts (and at the same time apply morphing/skinning). Simple.
-		DecodeAndTransformForDepthRaster(depthTransformed_ + (draw.vertexOffset + numDecoded) * 4, worldviewproj, dv.verts, dv.indexLowerBound, dv.indexUpperBound, dec, vertTypeID);
+		DecodeAndTransformForDepthRaster(batch.transformed + (draw.vertexOffset + numDecoded) * 4, worldviewproj, dv.verts, dv.indexLowerBound, dv.indexUpperBound, dec, vertTypeID);
 		numDecoded += dv.indexUpperBound - dv.indexLowerBound + 1;
 	}
 
 	// Copy indices.
-	memcpy(depthIndices_ + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount);
+	memcpy(batch.indices + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount);
 
-	// Commit
-	depthIndexCount_ += vertexCount;
-	depthVertexCount_ += numDecoded;
-
-	if (depthDraws_.empty()) {
-		rasterTimeStart_ = time_now_d();
-	}
-
-	depthDraws_.push_back(draw);
-
-	// FlushQueuedDepth();
+	CommitDepthDraw(draw, vertexCount, numDecoded);
 }
 
 void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, const VertexDecoder *dec, int vertexCount) {
@@ -1331,6 +1310,7 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i
 	if (!CalculateDepthDraw(&draw, prim, vertexCount, numDecoded)) {
 		return;
 	}
+	DepthBatch &batch = depthWorker_->Current();
 
 	TimeCollector collectStat(&gpuStats.perFrame.msPrepareDepth, g_coreCollectDebugStats);
 
@@ -1338,77 +1318,55 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i
 	_dbg_assert_(prim != GE_PRIM_TRIANGLE_STRIP && prim != GE_PRIM_TRIANGLE_FAN);
 
 	if (dec->throughmode) {
-		ConvertPredecodedThroughForDepthRaster(depthTransformed_ + 4 * draw.vertexOffset, decoded_, dec, numDecoded);
+		ConvertPredecodedThroughForDepthRaster(batch.transformed + 4 * draw.vertexOffset, decoded_, dec, numDecoded);
 	} else {
 		if (dec->VertexType() & (GE_VTYPE_WEIGHT_MASK | GE_VTYPE_MORPHCOUNT_MASK)) {
 			return;
 		}
 		float worldviewproj[16];
 		ComputeFinalProjMatrix().Store(worldviewproj);
-		TransformPredecodedForDepthRaster(depthTransformed_ + 4 * draw.vertexOffset, worldviewproj, decoded_, dec, numDecoded);
+		TransformPredecodedForDepthRaster(batch.transformed + 4 * draw.vertexOffset, worldviewproj, decoded_, dec, numDecoded);
 	}
 
 	// Copy indices.
-	memcpy(depthIndices_ + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount);
+	memcpy(batch.indices + draw.indexOffset, decIndex_, sizeof(uint16_t) * vertexCount);
 
-	// Commit
-	depthIndexCount_ += vertexCount;
-	depthVertexCount_ += numDecoded;
+	CommitDepthDraw(draw, vertexCount, numDecoded);
+}
 
-	depthDraws_.push_back(draw);
+void DrawEngineCommon::CommitDepthDraw(const DepthDraw &draw, int vertexCount, int numDecoded) {
+	DepthBatch &batch = depthWorker_->Current();
+	if (batch.draws.empty()) {
+		batch.lowQ = g_Config.iDepthRasterMode == (int)DepthRasterMode::LOW_QUALITY;
+		if (rasterTimeStart_ == 0.0) {
+			rasterTimeStart_ = time_now_d();
+		}
+	}
+	batch.indexCount += vertexCount;
+	batch.vertexCount += numDecoded;
+	batch.draws.push_back(draw);
+}
 
-	if (depthDraws_.empty()) {
-		rasterTimeStart_ = time_now_d();
+void DrawEngineCommon::SubmitQueuedDepth() {
+	if (depthWorker_) {
+		depthWorker_->Submit();
 	}
-	// FlushQueuedDepth();
 }
 
 void DrawEngineCommon::FlushQueuedDepth() {
+	if (!depthWorker_) {
+		return;
+	}
+	depthWorker_->Wait();
 	if (rasterTimeStart_ != 0.0) {
 		gpuStats.perFrame.msRasterTimeAvailable += time_now_d() - rasterTimeStart_;
 		rasterTimeStart_ = 0.0;
 	}
+	depthWorker_->TakeTimes(&gpuStats.perFrame.msCullDepth, &gpuStats.perFrame.msRasterizeDepth);
+}
 
-	const bool collectStats = g_coreCollectDebugStats;
-	const bool lowQ = g_Config.iDepthRasterMode == (int)DepthRasterMode::LOW_QUALITY;
-	for (const auto &draw : depthDraws_) {
-		int *tx = depthScreenVerts_;
-		int *ty = depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT;
-		float *tz = (float *)(depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT * 2);
-
-		int outVertCount = 0;
-
-		const float *vertices = depthTransformed_ + 4 * draw.vertexOffset;
-		const uint16_t *indices = depthIndices_ + draw.indexOffset;
-
-		DepthScissor tileScissor = draw.scissor.Tile(0, 1);
-
-		{
-			TimeCollector collectStat(&gpuStats.perFrame.msCullDepth, collectStats);
-			switch (draw.prim) {
-			case GE_PRIM_RECTANGLES:
-				outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, vertices, indices, draw, tileScissor);
-				break;
-			case GE_PRIM_TRIANGLES:
-				outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, vertices, indices, draw, tileScissor, DEPTH_SCREENVERTS_COMPONENT_COUNT);
-				break;
-			default:
-				_dbg_assert_(false);
-				break;
-			}
-		}
-		if (outVertCount > 0) {
-			TimeCollector collectStat(&gpuStats.perFrame.msRasterizeDepth, collectStats);
-			if (!Memory::IsValid4AlignedAddress(draw.depthAddr)) {
-				continue;
-			}
-			u16 *depthPtr = (uint16_t *)Memory::GetPointerWriteUnchecked(draw.depthAddr);
-			DepthRasterScreenVerts(depthPtr, draw.depthStride, tx, ty, tz, outVertCount, draw, tileScissor, lowQ);
-		}
+void DrawEngineCommon::WaitForDepthWrites(u32 addr, u32 size) {
+	if (depthWorker_) {
+		depthWorker_->WaitIfWriting(addr, size);
 	}
-
-	// Reset queue
-	depthIndexCount_ = 0;
-	depthVertexCount_ = 0;
-	depthDraws_.clear();
 }

GPU/Common/DrawEngineCommon.h

@@ -33,6 +33,7 @@
 
 class VertexDecoder;
 struct DepthDraw;
+class DepthRasterWorker;
 
 enum {
 	VERTEX_BUFFER_MAX = 65536,
@@ -159,7 +160,12 @@ public:
 		return decoded_ + 12 * 65536;
 	}
 
+	// Rasterizes the queued depth draws, and waits for them.
 	void FlushQueuedDepth();
+	// Starts rasterizing the queued depth draws, without waiting.
+	void SubmitQueuedDepth();
+	// Waits if depth rasterization in progress may write to these bytes.
+	void WaitForDepthWrites(u32 addr, u32 size);
 
 protected:
 	bool CheckClipFlags(bool useHwTransform) const;
@@ -181,6 +187,7 @@ protected:
 	void DepthRasterSubmitRaw(GEPrimitiveType prim, const VertexDecoder *dec, uint32_t vertTypeID, int vertexCount);
 	void DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, const VertexDecoder *dec, int vertexCount);
 	bool CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, int vertexCount, int numDecoded);
+	void CommitDepthDraw(const DepthDraw &draw, int vertexCount, int numDecoded);
 
 	static inline int IndexSize(u32 vtype) {
 		const u32 indexType = (vtype & GE_VTYPE_IDX_MASK);
@@ -368,19 +375,12 @@ protected:
 	// Software depth raster
 	bool useDepthRaster_ = false;
 
-	float *depthTransformed_ = nullptr;
-	int *depthScreenVerts_ = nullptr;
-	uint16_t *depthIndices_ = nullptr;
+	DepthRasterWorker *depthWorker_ = nullptr;
 
 	// Depth tracking
 	ClipInfoFlags clipInfoFlags_{};
 	ClipInfoFlags lastClipInfoFlags_{};  // Flags at the last flush. For dirtying.
 
-	// Queue
-	int depthVertexCount_ = 0;
-	int depthIndexCount_ = 0;
-	std::vector<DepthDraw> depthDraws_;
-
 	double rasterTimeStart_ = 0.0;
 
 	bool lastUseHwTransform_ = true;

GPU/Common/FramebufferManagerCommon.h

@@ -313,6 +313,9 @@ public:
 	void SetDrawEngine(DrawEngineCommon *td) {
 		drawEngine_ = td;
 	}
+	DrawEngineCommon *GetDrawEngine() const {
+		return drawEngine_;
+	}
 
 	void Init(int msaaLevel);
 	virtual void BeginFrame(const DisplayLayoutConfig &config);

GPU/Common/TextureCacheCommon.cpp

@@ -36,6 +36,7 @@
 #include "Core/Debugger/MemBlockInfo.h"
 #include "Core/System.h"
 #include "Core/HW/Display.h"
+#include "GPU/Common/DrawEngineCommon.h"
 #include "GPU/Common/FramebufferManagerCommon.h"
 #include "GPU/Common/TextureCacheCommon.h"
 #include "GPU/Common/TextureDecoder.h"
@@ -399,6 +400,13 @@ SamplerCacheKey GetFramebufferSamplingParams(const GEState &gstate, u16 bufferWi
 	return key;
 }
 
+// Before reading texture data: it may be in a depth buffer the software depth rasterizer is writing.
+void TextureCacheCommon::WaitForDepthWrites(u32 addr, u32 bytes) {
+	if (Memory::IsVRAMAddress(addr)) {
+		framebufferManager_->GetDrawEngine()->WaitForDepthWrites(addr, bytes);
+	}
+}
+
 static u32 ComputeTextureHash(TextureReplacer &replacer, u32 addr, int bufw, int w, int h, bool swizzled, const TexCacheEntry *entry) {
 	const GETextureFormat format = entry->format;
 	// Swizzled CLUT4 glyph atlases can fill below the visible UV range while remaining
@@ -723,6 +731,9 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) {
 				// A video texture is new every frame by definition, so hashing it only confirms what
 				// the VIDEO flag already said.
 				const bool skipHash = isVideo;
+				if (!skipHash) {
+					WaitForDepthWrites(entry->addr, (textureBitsPerPixel[entry->format] * entry->bufw * h) / 8);
+				}
 				const u32 newFullHash = skipHash ? 0 : ComputeTextureHash(replacer_, entry->addr, entry->bufw, w, h, swizzled, entry);
 				if (skipHash || newFullHash != entry->fullhash) {
 					// The texture changed. Throw it in the secondary cache. Then we'll create a new entry later.
@@ -932,6 +943,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) {
 
 	const bool skipHash = isVideo;
 	if (!(entry->status & TexStatus::IS_PPGE_ATLAS) && !skipHash) {
+		WaitForDepthWrites(entry->addr, (textureBitsPerPixel[entry->format] * entry->bufw * h) / 8);
 		entry->fullhash = ComputeTextureHash(replacer_, entry->addr, entry->bufw, w, h, swizzled, entry);
 	}
 
@@ -2015,6 +2027,9 @@ TextureAlpha TextureCacheCommon::DecodeTextureLevel(u8 *out, int outPitch, GETex
 
 	const u8 *texptr = isPPGE ? (PPGeAtlasGetData() + ppgeOffset) : Memory::GetPointerOrException(texaddr);
 	const uint32_t byteSize = (textureBitsPerPixel[format] * bufw * h) / 8;
+	if (!isPPGE) {
+		WaitForDepthWrites(texaddr, byteSize);
+	}
 
 	// Validate the texture data fits in mapped RAM, like the DXT path does.
 	// texaddr/bufw/w/h are all guest-controlled via the GE display list.

GPU/Common/TextureCacheCommon.h

@@ -397,6 +397,8 @@ public:
 
 
 protected:
+	void WaitForDepthWrites(u32 addr, u32 bytes);
+
 	bool PrepareBuildTexture(BuildTexturePlan &plan, TexCacheEntry *entry);
 
 	// This updates nextTexture_ / nextFramebufferTexture_, which is then used by ApplyTexture.

GPU/GPUCommon.cpp

@@ -1212,7 +1212,7 @@ void GPUCommon::Execute_Ret(u32 op, u32 diff) {
 
 void GPUCommon::Execute_End(u32 op, u32 diff) {
 	if (flushOnParams_) {
-		drawEngineCommon_->FlushQueuedDepth();
+		drawEngineCommon_->SubmitQueuedDepth();
 		Flush();
 	}
 

GPU/GPUCommonHW.cpp

@@ -927,7 +927,7 @@ void GPUCommonHW::Execute_Prim(u32 op, u32 diff) {
 	}
 
 	if (changed) {
-		drawEngineCommon_->FlushQueuedDepth();
+		drawEngineCommon_->SubmitQueuedDepth();
 	}
 
 	if (gstate_c.dirty & DIRTY_VERTEXSHADER_STATE) {
@@ -1288,7 +1288,7 @@ void GPUCommonHW::Execute_Bezier(u32 op, u32 diff) {
 	bool changed;
 	VirtualFramebuffer *vfb = framebufferManager_->SetRenderFrameBuffer(gstate_c.IsDirty(DIRTY_FRAMEBUF), gstate_c.skipDrawReason, &changed);
 	if (changed) {
-		drawEngineCommon_->FlushQueuedDepth();
+		drawEngineCommon_->SubmitQueuedDepth();
 	}
 	if (gstate_c.skipDrawReason & (SKIPDRAW_SKIPFRAME | SKIPDRAW_NON_DISPLAYED_FB)) {
 		// TODO: Should this eat some cycles?  Probably yes.  Not sure if important.
@@ -1359,7 +1359,7 @@ void GPUCommonHW::Execute_Spline(u32 op, u32 diff) {
 	bool changed;
 	VirtualFramebuffer *vfb = framebufferManager_->SetRenderFrameBuffer(gstate_c.IsDirty(DIRTY_FRAMEBUF), gstate_c.skipDrawReason, &changed);
 	if (changed) {
-		drawEngineCommon_->FlushQueuedDepth();
+		drawEngineCommon_->SubmitQueuedDepth();
 	}
 	if (gstate_c.skipDrawReason & (SKIPDRAW_SKIPFRAME | SKIPDRAW_NON_DISPLAYED_FB)) {
 		// TODO: Should this eat some cycles?  Probably yes.  Not sure if important.
@@ -1468,6 +1468,7 @@ void GPUCommonHW::Execute_TexLevel(u32 op, u32 diff) {
 
 void GPUCommonHW::Execute_LoadClut(u32 op, u32 diff) {
 	gstate_c.Dirty(DIRTY_TEXTURE_PARAMS);
+	drawEngineCommon_->WaitForDepthWrites(gstate.getClutAddress(), gstate.getClutLoadBytes());
 	textureCache_->LoadClut(gstate.getClutAddress(), gstate.getClutLoadBytes(), &recorder_);
 }
 
@@ -1741,6 +1742,10 @@ void GPUCommonHW::Execute_TexFlush(u32 op, u32 diff) {
 	framebufferManager_->DiscardFramebufferCopy();
 }
 
+void GPUCommonHW::FlushPendingDrawing() {
+	drawEngineCommon_->FlushQueuedDepth();
+}
+
 u32 GPUCommonHW::DrawSync(int mode) {
 	drawEngineCommon_->FlushQueuedDepth();
 	return GPUCommon::DrawSync(mode);

GPU/GPUCommonHW.h

@@ -45,6 +45,7 @@ public:
 	void InvalidateCache(u32 addr, int size, GPUInvalidationType type) override;
 
 	u32 DrawSync(int mode) override;
+	void FlushPendingDrawing() override;
 	int ListSync(int listid, int mode) override;
 
 	bool FramebufferDirty() override;