Merge pull request #19774 from hrydgard/depth-raster-refactor

Depth raster refactor

Common/Math/CrossSIMD.h

@@ -145,6 +145,7 @@ struct Vec4S32 {
 	static Vec4S32 Load(const int *src) { return Vec4S32{ _mm_loadu_si128((const __m128i *)src) }; }
 	static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ _mm_load_si128((const __m128i *)src) }; }
 	void Store(int *dst) { _mm_storeu_si128((__m128i *)dst, v); }
+	void Store2(int *dst) { _mm_storel_epi64((__m128i *)dst, v); }
 	void StoreAligned(int *dst) { _mm_store_si128((__m128i *)dst, v);}
 
 	// Swaps the two lower elements. Useful for reversing triangles..
@@ -196,6 +197,7 @@ struct Vec4F32 {
 	static Vec4F32 Load(const float *src) { return Vec4F32{ _mm_loadu_ps(src) }; }
 	static Vec4F32 LoadAligned(const float *src) { return Vec4F32{ _mm_load_ps(src) }; }
 	void Store(float *dst) { _mm_storeu_ps(dst, v); }
+	void Store2(float *dst) { _mm_storel_epi64((__m128i *)dst, _mm_castps_si128(v)); }
 	void StoreAligned (float *dst) { _mm_store_ps(dst, v); }
 	void Store3(float *dst) {
 		// TODO: There might be better ways.
@@ -455,6 +457,7 @@ struct Vec4S32 {
 	static Vec4S32 Load(const int *src) { return Vec4S32{ vld1q_s32(src) }; }
 	static Vec4S32 LoadAligned(const int *src) { return Vec4S32{ vld1q_s32(src) }; }
 	void Store(int *dst) { vst1q_s32(dst, v); }
+	void Store2(int *dst) { vst1_s32(dst, vget_low_s32(v)); }
 	void StoreAligned(int *dst) { vst1q_s32(dst, v); }
 
 	// Swaps the two lower elements, but NOT the two upper ones. Useful for reversing triangles..
@@ -488,6 +491,7 @@ struct Vec4F32 {
 	static Vec4F32 Load(const float *src) { return Vec4F32{ vld1q_f32(src) }; }
 	static Vec4F32 LoadAligned(const float *src) { return Vec4F32{ vld1q_f32(src) }; }
 	void Store(float *dst) { vst1q_f32(dst, v); }
+	void Store2(float *dst) { vst1_f32(dst, vget_low_f32(v)); }
 	void StoreAligned(float *dst) { vst1q_f32(dst, v); }
 	void Store3(float *dst) {
 		// TODO: There might be better ways. Try to avoid this when possible.

GPU/Common/DepthRaster.cpp

@@ -8,15 +8,8 @@
 #include "Common/Math/math_util.h"
 #include "GPU/Common/VertexDecoderCommon.h"
 
-// We only need to support these three modes.
-enum class ZCompareMode {
-	Greater,  // Most common
-	Less,  // Less common
-	Always,  // Fairly common
-};
-
 // x1/x2 etc are the scissor rect.
-void DepthRasterRect(uint16_t *dest, int stride, int x1, int y1, int x2, int y2, int v1x, int v1y, int v2x, int v2y, short depthValue, ZCompareMode compareMode) {
+static void DepthRasterRect(uint16_t *dest, int stride, const DepthScissor scissor, int v1x, int v1y, int v2x, int v2y, short depthValue, ZCompareMode compareMode) {
 	// Swap coordinates if needed, we don't back-face-cull rects.
 	// We also ignore the UV rotation here.
 	if (v1x > v2x) {
@@ -26,21 +19,21 @@ void DepthRasterRect(uint16_t *dest, int stride, int x1, int y1, int x2, int y2,
 		std::swap(v1y, v2y);
 	}
 
-	if (v1x < x1) {
-		v1x = x1;
+	if (v1x < scissor.x1) {
+		v1x = scissor.x1;
 	}
-	if (v2x > x2) {
-		v2x = x2 + 1;  // PSP scissors are inclusive
+	if (v2x > scissor.x2) {
+		v2x = scissor.x2 + 1;  // PSP scissors are inclusive
 	}
 	if (v1x >= v2x) {
 		return;
 	}
 
-	if (v1y < y1) {
-		v1y = y1;
+	if (v1y < scissor.y1) {
+		v1y = scissor.y1;
 	}
-	if (v2y > y2) {
-		v2y = y2 + 1;
+	if (v2y > scissor.y2) {
+		v2y = scissor.y2 + 1;
 	}
 	if (v1y >= v2y) {
 		return;
@@ -116,12 +109,12 @@ constexpr int MIN_TRI_AREA = 10;
 // Started with the scalar version, will SIMD-ify later.
 // x1/y1 etc are the scissor rect.
 template<ZCompareMode compareMode>
-TriangleResult DepthRasterTriangle(uint16_t *depthBuf, int stride, int x1, int y1, int x2, int y2, const int *tx, const int *ty, const float *tz) {
-	int tileStartX = x1;
-	int tileEndX = x2;
+TriangleResult DepthRasterTriangle(uint16_t *depthBuf, int stride, DepthScissor scissor, const int *tx, const int *ty, const float *tz) {
+	const int tileStartX = scissor.x1;
+	const int tileEndX = scissor.x2;
 
-	int tileStartY = y1;
-	int tileEndY = y2;
+	const int tileStartY = scissor.y1;
+	const int tileEndY = scissor.y2;
 
 	// BEGIN triangle setup. This should be done SIMD, four triangles at a time.
 	// Due to the many multiplications, we might want to do it in floating point as 32-bit integer muls
@@ -149,11 +142,11 @@ TriangleResult DepthRasterTriangle(uint16_t *depthBuf, int stride, int x1, int y
 
 	// TODO: Cull really small triangles here - we can increase the threshold a bit probably.
 	int triArea = (v1y - v2y) * v0x + (v2x - v1x) * v0y + (v1x * v2y - v2x * v1y);
-	if (triArea <= 0) {
+	if (triArea < 0) {
 		return TriangleResult::Backface;
 	}
 	if (triArea < MIN_TRI_AREA) {
-		return TriangleResult::TooSmall;
+		return TriangleResult::TooSmall;  // Or zero area.
 	}
 
 	float oneOverTriArea = 1.0f / (float)triArea;
@@ -220,13 +213,14 @@ TriangleResult DepthRasterTriangle(uint16_t *depthBuf, int stride, int x1, int y
 }
 
 template<ZCompareMode compareMode>
-inline void DepthRaster4Triangles(int stats[4], uint16_t *depthBuf, int stride, int x1, int y1, int x2, int y2, const int *tx, const int *ty, const float *tz) {
+inline void DepthRaster4Triangles(int stats[4], uint16_t *depthBuf, int stride, DepthScissor scissor, const int *tx, const int *ty, const float *tz) {
 	for (int i = 0; i < 4; i++) {
-		TriangleResult result = DepthRasterTriangle<compareMode>(depthBuf, stride, x1, y1, x2, y2, tx + i, ty + i, tz + i);
+		TriangleResult result = DepthRasterTriangle<compareMode>(depthBuf, stride, scissor, tx + i, ty + i, tz + i);
 		stats[(int)result]++;
 	}
 }
 
+// This will always run on the main thread. Though, might consider moving the transforms out and just storing verts instead?
 void DecodeAndTransformForDepthRaster(float *dest, const float *worldviewproj, const void *vertexData, int indexLowerBound, int indexUpperBound, VertexDecoder *dec, u32 vertTypeID) {
 	// TODO: Ditch skinned and morphed prims for now since we don't have a fast way to skin without running the full decoder.
 	_dbg_assert_((vertTypeID & (GE_VTYPE_WEIGHT_MASK | GE_VTYPE_MORPHCOUNT_MASK)) == 0);
@@ -290,12 +284,14 @@ void ConvertPredecodedThroughForDepthRaster(float *dest, const void *decodedVert
 	for (int i = 0; i < count; i++) {
 		const float *data = (const float *)(startPtr + i * vertexStride + offset);
 		// Just pass the position straight through - this is through mode!
-		Vec4F32::Load(data).WithLane3Zero().Store(dest + i * 4);
+		// A W of one makes projection a no-op, without branching.
+		Vec4F32::Load(data).WithLane3One().Store(dest + i * 4);
 	}
 }
 
-int DepthRasterClipIndexedRectangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, int count) {
+int DepthRasterClipIndexedRectangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, const DepthDraw &draw) {
 	int outCount = 0;
+	const int count = draw.vertexCount;
 	for (int i = 0; i < count; i += 2) {
 		const float *verts[2] = {
 			transformed + indexBuffer[i] * 4,
@@ -322,29 +318,29 @@ int DepthRasterClipIndexedRectangles(int *tx, int *ty, float *tz, const float *t
 		y *= recipW;
 		z *= recipW;
 
-		Vec4S32FromF32(x).Store(tx + outCount);
-		Vec4S32FromF32(y).Store(ty + outCount);
-		z.Clamp(0.0f, 65535.0f).Store(tz + outCount);
+		Vec4S32FromF32(x).Store2(tx + outCount);
+		Vec4S32FromF32(y).Store2(ty + outCount);
+		z.Clamp(0.0f, 65535.0f).Store2(tz + outCount);
 		outCount += 2;
 	}
 	return outCount;
 }
 
-int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, int count) {
-	bool cullEnabled = gstate.isCullEnabled();
-	GECullMode cullMode = gstate.getCullMode();
-
+int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, const DepthDraw &draw) {
 	int outCount = 0;
 
 	int flipCull = 0;
-	if (cullEnabled && cullMode == GE_CULL_CW) {
+	if (draw.cullEnabled && draw.cullMode == GE_CULL_CW) {
 		flipCull = 3;
 	}
+	const bool cullEnabled = draw.cullEnabled;
 
 	static const float zerovec[4] = {};
 
 	int collected = 0;
+	int planeCulled = 0;
 	const float *verts[12];  // four triangles at a time!
+	const int count = draw.vertexCount;
 	for (int i = 0; i < count; i += 3) {
 		// Collect valid triangles into buffer.
 		const float *v0 = transformed + indexBuffer[i] * 4;
@@ -356,6 +352,8 @@ int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *tr
 			verts[collected + 1] = v1;
 			verts[collected + 2] = v2;
 			collected += 3;
+		} else {
+			planeCulled++;
 		}
 
 		if (i >= count - 3 && collected != 12) {
@@ -390,12 +388,12 @@ int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *tr
 		Vec4F32::Transpose(x1, y1, z1, w1);
 		Vec4F32::Transpose(x2, y2, z2, w2);
 
-		// Now the names are accurate! Since we only have three vertices, the fourth member of each vector is zero
-		// and will not be stored (well it will be stored, but it'll be overwritten by the next vertex).
+		// Now the names are accurate!
+
+		// Let's project all three vertices, for all four triangles.
 		Vec4F32 recipW0 = w0.Recip();
 		Vec4F32 recipW1 = w1.Recip();
 		Vec4F32 recipW2 = w2.Recip();
-
 		x0 *= recipW0;
 		y0 *= recipW0;
 		z0 = (z0 * recipW0).Clamp(0.0f, 65535.0f);
@@ -433,73 +431,23 @@ int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *tr
 			outCount += 12;
 		}
 	}
+
+	gpuStats.numDepthRasterZCulled += planeCulled;
 	return outCount;
 }
 
-void DepthRasterConvertTransformed(int *tx, int *ty, float *tz, const float *transformed, const uint16_t * indexBuffer, int count) {
-	for (int i = 0; i < count; i++) {
-		const float *pos = transformed + indexBuffer[i] * 4;
-		tx[i] = (int)pos[0];
-		ty[i] = (int)pos[1];
-		tz[i] = pos[2];  // clamp?
-	}
-}
-
 // Rasterizes screen-space vertices.
-void DepthRasterScreenVerts(uint16_t *depth, int depthStride, GEPrimitiveType prim, int x1, int y1, int x2, int y2, const int *tx, const int *ty, const float *tz, int count) {
+void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, const int *ty, const float *tz, int count, const DepthDraw &draw) {
 	// Prim should now be either TRIANGLES or RECTs.
-	_dbg_assert_(prim == GE_PRIM_RECTANGLES || prim == GE_PRIM_TRIANGLES);
+	_dbg_assert_(draw.prim == GE_PRIM_RECTANGLES || draw.prim == GE_PRIM_TRIANGLES);
 
-	// Ignore draws where stencil operations are active?
-	if (gstate.isStencilTestEnabled()) {
-		// return;
-	}
-
-	GEComparison compareMode = gstate.getDepthTestFunction();
-
-	ZCompareMode comp;
-	// Ignore some useless compare modes.
-	switch (compareMode) {
-	case GE_COMP_ALWAYS:
-		comp = ZCompareMode::Always;
-		break;
-	case GE_COMP_LEQUAL:
-	case GE_COMP_LESS:
-		comp = ZCompareMode::Less;
-		break;
-	case GE_COMP_GEQUAL:
-	case GE_COMP_GREATER:
-		comp = ZCompareMode::Greater;  // Most common
-		break;
-	case GE_COMP_NEVER:
-	case GE_COMP_EQUAL:
-		// These will never have a useful effect in Z-only raster.
-		[[fallthrough]];
-	case GE_COMP_NOTEQUAL:
-		// This is highly unusual, let's just ignore it.
-		[[fallthrough]];
-	default:
-		return;
-	}
-
-	if (gstate.isModeClear()) {
-		if (!gstate.isClearModeDepthMask()) {
-			return;
-		}
-		comp = ZCompareMode::Always;
-	} else {
-		// These should have been caught earlier.
-		_dbg_assert_(gstate.isDepthTestEnabled());
-		_dbg_assert_(gstate.isDepthWriteEnabled());
-	}
-
-	switch (prim) {
+	switch (draw.prim) {
 	case GE_PRIM_RECTANGLES:
 		for (int i = 0; i < count; i += 2) {
 			uint16_t z = (uint16_t)tz[i + 1];  // depth from second vertex
 			// TODO: Should clip coordinates to the scissor rectangle.
 			// We remove the subpixel information here.
-			DepthRasterRect(depth, depthStride, x1, y1, x2, y2, tx[i], ty[i], tx[i + 1], ty[i + 1], z, comp);
+			DepthRasterRect(depth, depthStride, draw.scissor, tx[i], ty[i], tx[i + 1], ty[i + 1], z, draw.compareMode);
 		}
 		gpuStats.numDepthRasterPrims += count / 2;
 		break;
@@ -508,20 +456,20 @@ void DepthRasterScreenVerts(uint16_t *depth, int depthStride, GEPrimitiveType pr
 		int stats[4]{};
 		// Batches of 4 triangles, as output by the clip function.
 		for (int i = 0; i < count; i += 12) {
-			switch (comp) {
+			switch (draw.compareMode) {
 			case ZCompareMode::Greater:
 			{
-				DepthRaster4Triangles<ZCompareMode::Greater>(stats, depth, depthStride, x1, y1, x2, y2, &tx[i], &ty[i], &tz[i]);
+				DepthRaster4Triangles<ZCompareMode::Greater>(stats, depth, depthStride, draw.scissor, &tx[i], &ty[i], &tz[i]);
 				break;
 			}
 			case ZCompareMode::Less:
 			{
-				DepthRaster4Triangles<ZCompareMode::Less>(stats, depth, depthStride, x1, y1, x2, y2, &tx[i], &ty[i], &tz[i]);
+				DepthRaster4Triangles<ZCompareMode::Less>(stats, depth, depthStride, draw.scissor, &tx[i], &ty[i], &tz[i]);
 				break;
 			}
 			case ZCompareMode::Always:
 			{
-				DepthRaster4Triangles<ZCompareMode::Always>(stats, depth, depthStride, x1, y1, x2, y2, &tx[i], &ty[i], &tz[i]);
+				DepthRaster4Triangles<ZCompareMode::Always>(stats, depth, depthStride, draw.scissor, &tx[i], &ty[i], &tz[i]);
 				break;
 			}
 			}

GPU/Common/DepthRaster.h

@@ -9,6 +9,37 @@ struct DepthScreenVertex {
 	int z;
 };
 
+// X11.h, sigh.
+#ifdef Always
+#undef Always
+#endif
+
+// We only need to support these three modes.
+enum class ZCompareMode {
+	Greater,  // Most common
+	Less,  // Less common
+	Always,  // Mostly used for clears
+};
+
+struct DepthScissor {
+	u16 x1;
+	u16 y1;
+	u16 x2;
+	u16 y2;
+};
+
+struct DepthDraw {
+	GEPrimitiveType prim;
+	ZCompareMode compareMode;
+	bool cullEnabled;
+	int cullMode;
+	DepthScissor scissor;
+	bool through;
+	int transformedStartIndex;
+	int indexOffset;
+	int vertexCount;
+};
+
 // Specialized, very limited depth-only rasterizer.
 // Meant to run in parallel with hardware rendering, in games that read back the depth buffer
 // for effects like lens flare.
@@ -17,13 +48,9 @@ struct DepthScreenVertex {
 class VertexDecoder;
 struct TransformedVertex;
 
-int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, int count);
-int DepthRasterClipIndexedRectangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, int count);
+int DepthRasterClipIndexedTriangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, const DepthDraw &draw);
+int DepthRasterClipIndexedRectangles(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, const DepthDraw &draw);
 void DecodeAndTransformForDepthRaster(float *dest, const float *worldviewproj, const void *vertexData, int indexLowerBound, int indexUpperBound, VertexDecoder *dec, u32 vertTypeID);
 void TransformPredecodedForDepthRaster(float *dest, const float *worldviewproj, const void *decodedVertexData, VertexDecoder *dec, int count);
 void ConvertPredecodedThroughForDepthRaster(float *dest, const void *decodedVertexData, VertexDecoder *dec, int count);
-void DepthRasterConvertTransformed(int *tx, int *ty, float *tz, const float *transformed, const uint16_t *indexBuffer, int count);
-
-// void DepthRasterConvertTransformed(int *tx, int *ty, int *tz, GEPrimitiveType prim, const TransformedVertex *transformed, int count);
-
-void DepthRasterScreenVerts(uint16_t *depth, int depthStride, GEPrimitiveType prim, int x1, int y1, int x2, int y2, const int *tx, const int *ty, const float *tz, int count);
+void DepthRasterScreenVerts(uint16_t *depth, int depthStride, const int *tx, const int *ty, const float *tz, int count, const DepthDraw &draw);

GPU/Common/DrawEngineCommon.cpp

@@ -43,6 +43,7 @@ enum {
 	DEPTH_SCREENVERTS_COMPONENT_COUNT = VERTEX_BUFFER_MAX,
 	DEPTH_SCREENVERTS_COMPONENT_SIZE = DEPTH_SCREENVERTS_COMPONENT_COUNT * sizeof(int) + 384,
 	DEPTH_SCREENVERTS_SIZE = DEPTH_SCREENVERTS_COMPONENT_SIZE * 3,
+	DEPTH_INDEXBUFFER_SIZE = VERTEX_BUFFER_MAX * 3 * sizeof(uint16_t),
 };
 
 DrawEngineCommon::DrawEngineCommon() : decoderMap_(32) {
@@ -76,6 +77,7 @@ DrawEngineCommon::~DrawEngineCommon() {
 	if (depthTransformed_) {
 		FreeMemoryPages(depthTransformed_, DEPTH_TRANSFORMED_SIZE);
 		FreeMemoryPages(depthScreenVerts_, DEPTH_SCREENVERTS_SIZE);
+		FreeMemoryPages(depthIndices_, DEPTH_INDEXBUFFER_SIZE);
 	}
 	delete decJitCache_;
 	decoderMap_.Iterate([&](const uint32_t vtype, VertexDecoder *decoder) {
@@ -789,6 +791,7 @@ void DrawEngineCommon::BeginFrame() {
 	if (!depthTransformed_ && useDepthRaster_) {
 		depthTransformed_ = (float *)AllocateMemoryPages(DEPTH_TRANSFORMED_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
 		depthScreenVerts_ = (int *)AllocateMemoryPages(DEPTH_SCREENVERTS_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
+		depthIndices_ = (uint16_t *)AllocateMemoryPages(DEPTH_INDEXBUFFER_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
 	}
 }
 
@@ -929,31 +932,86 @@ Mat4F32 ComputeFinalProjMatrix() {
 	return m;
 }
 
-void DrawEngineCommon::DepthRasterTransform(GEPrimitiveType prim, VertexDecoder *dec, uint32_t vertTypeID, int vertexCount) {
-	if (!gstate.isModeClear() && (!gstate.isDepthTestEnabled() || !gstate.isDepthWriteEnabled())) {
-		return;
-	}
-
+static bool CalculateDepthDraw(DepthDraw *draw, GEPrimitiveType prim, int vertexCount) {
 	switch (prim) {
 	case GE_PRIM_INVALID:
 	case GE_PRIM_KEEP_PREVIOUS:
 	case GE_PRIM_LINES:
 	case GE_PRIM_LINE_STRIP:
 	case GE_PRIM_POINTS:
-		return;
+		return false;
 	default:
 		break;
 	}
 
+	// Ignore some useless compare modes.
+	switch (gstate.getDepthTestFunction()) {
+	case GE_COMP_ALWAYS:
+		draw->compareMode = ZCompareMode::Always;
+		break;
+	case GE_COMP_LEQUAL:
+	case GE_COMP_LESS:
+		draw->compareMode = ZCompareMode::Less;
+		break;
+	case GE_COMP_GEQUAL:
+	case GE_COMP_GREATER:
+		draw->compareMode = ZCompareMode::Greater;  // Most common
+		break;
+	case GE_COMP_NEVER:
+	case GE_COMP_EQUAL:
+		// These will never have a useful effect in Z-only raster.
+		[[fallthrough]];
+	case GE_COMP_NOTEQUAL:
+		// This is highly unusual, let's just ignore it.
+		[[fallthrough]];
+	default:
+		return false;
+	}
+	if (gstate.isModeClear()) {
+		if (!gstate.isClearModeDepthMask()) {
+			return false;
+		}
+		draw->compareMode = ZCompareMode::Always;
+	} else {
+		// These should have been caught earlier.
+		_dbg_assert_(gstate.isDepthTestEnabled());
+		_dbg_assert_(gstate.isDepthWriteEnabled());
+	}
+
+	draw->transformedStartIndex = 0;
+	draw->indexOffset = 0;
+	draw->vertexCount = vertexCount;
+	draw->cullEnabled = gstate.isCullEnabled();
+	draw->cullMode = gstate.getCullMode();
+	draw->prim = prim;
+	draw->scissor.x1 = gstate.getScissorX1();
+	draw->scissor.y1 = gstate.getScissorY1();
+	draw->scissor.x2 = gstate.getScissorX2();
+	draw->scissor.y2 = gstate.getScissorY2();
+	return true;
+}
+
+void DrawEngineCommon::DepthRasterTransform(GEPrimitiveType prim, VertexDecoder *dec, uint32_t vertTypeID, int vertexCount) {
+	if (!gstate.isModeClear() && (!gstate.isDepthTestEnabled() || !gstate.isDepthWriteEnabled())) {
+		return;
+	}
+
 	if (vertTypeID & (GE_VTYPE_WEIGHT_MASK | GE_VTYPE_MORPHCOUNT_MASK)) {
 		return;
 	}
 
-	TimeCollector collectStat(&gpuStats.msRasterizingDepth, coreCollectDebugStats);
+	_dbg_assert_(prim != GE_PRIM_RECTANGLES);
 
 	float worldviewproj[16];
 	ComputeFinalProjMatrix().Store(worldviewproj);
 
+	DepthDraw draw;
+	if (!CalculateDepthDraw(&draw, prim, vertexCount)) {
+		return;
+	}
+
+	TimeCollector collectStat(&gpuStats.msRasterizingDepth, coreCollectDebugStats);
+
 	// Decode.
 	int numDec = 0;
 	for (int i = 0; i < numDrawVerts_; i++) {
@@ -973,16 +1031,11 @@ void DrawEngineCommon::DepthRasterTransform(GEPrimitiveType prim, VertexDecoder
 		numDec += indexUpperBound - indexLowerBound + 1;
 	}
 
-	int *tx = depthScreenVerts_;
-	int *ty = depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT;
-	float *tz = (float *)(depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT * 2);
+	// Copy indices.
+	memcpy(depthIndices_, decIndex_, sizeof(uint16_t) * vertexCount);
 
-	// Clip and triangulate using the index buffer. It now automatically zero-pads.
-	int outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, depthTransformed_, decIndex_, vertexCount);
-
-	DepthRasterScreenVerts((uint16_t *)Memory::GetPointerWrite(gstate.getDepthBufRawAddress() | 0x04000000), gstate.DepthBufStride(),
-		GE_PRIM_TRIANGLES, gstate.getScissorX1(), gstate.getScissorY1(), gstate.getScissorX2(), gstate.getScissorY2(),
-		tx, ty, tz, outVertCount);
+	// FUTURE SPLIT --- The above will always run on the main thread. The below can be split across workers.
+	FlushDepthDraw(draw);
 }
 
 void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, VertexDecoder *dec, int vertexCount) {
@@ -990,31 +1043,17 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i
 		return;
 	}
 
-	TimeCollector collectStat(&gpuStats.msRasterizingDepth, coreCollectDebugStats);
-
-	switch (prim) {
-	case GE_PRIM_INVALID:
-	case GE_PRIM_KEEP_PREVIOUS:
-	case GE_PRIM_LINES:
-	case GE_PRIM_LINE_STRIP:
-	case GE_PRIM_POINTS:
+	DepthDraw draw;
+	if (!CalculateDepthDraw(&draw, prim, vertexCount)) {
 		return;
-	default:
-		break;
 	}
 
+	TimeCollector collectStat(&gpuStats.msRasterizingDepth, coreCollectDebugStats);
+
 	_dbg_assert_(prim != GE_PRIM_TRIANGLE_STRIP && prim != GE_PRIM_TRIANGLE_FAN);
 
-	int *tx = depthScreenVerts_;
-	int *ty = depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT;
-	float *tz = (float *)(depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT * 2);
-
-	int outVertCount = 0;
-
 	if (dec->throughmode) {
 		ConvertPredecodedThroughForDepthRaster(depthTransformed_, decoded_, dec, numDecoded);
-		DepthRasterConvertTransformed(tx, ty, tz, depthTransformed_, decIndex_, vertexCount);
-		outVertCount = vertexCount;
 	} else {
 		if (dec->VertexType() & (GE_VTYPE_WEIGHT_MASK | GE_VTYPE_MORPHCOUNT_MASK)) {
 			return;
@@ -1022,29 +1061,32 @@ void DrawEngineCommon::DepthRasterPredecoded(GEPrimitiveType prim, const void *i
 		float worldviewproj[16];
 		ComputeFinalProjMatrix().Store(worldviewproj);
 		TransformPredecodedForDepthRaster(depthTransformed_, worldviewproj, decoded_, dec, numDecoded);
-
-		switch (prim) {
-		case GE_PRIM_RECTANGLES:
-			outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, depthTransformed_, decIndex_, vertexCount);
-			break;
-		case GE_PRIM_TRIANGLES:
-			outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, depthTransformed_, decIndex_, vertexCount);
-			break;
-		default:
-			_dbg_assert_(false);
-			break;
-		}
 	}
 
-	if (prim == GE_PRIM_TRIANGLES && (outVertCount & 15) != 0) {
-		// Zero padding
-		for (int i = outVertCount; i < ((outVertCount + 16) & ~15); i++) {
-			tx[i] = 0;
-			ty[i] = 0;
-			tz[i] = 0.0f;
-		}
+	// Copy indices.
+	memcpy(depthIndices_, decIndex_, sizeof(uint16_t) * vertexCount);
+
+	// FUTURE SPLIT --- The above will always run on the main thread. The below can be split across workers.
+	FlushDepthDraw(draw);
+}
+
+void DrawEngineCommon::FlushDepthDraw(const DepthDraw &draw) {
+	int *tx = depthScreenVerts_;
+	int *ty = depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT;
+	float *tz = (float *)(depthScreenVerts_ + DEPTH_SCREENVERTS_COMPONENT_COUNT * 2);
+
+	int outVertCount = 0;
+	switch (draw.prim) {
+	case GE_PRIM_RECTANGLES:
+		outVertCount = DepthRasterClipIndexedRectangles(tx, ty, tz, depthTransformed_, depthIndices_ + draw.indexOffset, draw);
+		break;
+	case GE_PRIM_TRIANGLES:
+		outVertCount = DepthRasterClipIndexedTriangles(tx, ty, tz, depthTransformed_, depthIndices_ + draw.indexOffset, draw);
+		break;
+	default:
+		_dbg_assert_(false);
+		break;
 	}
 	DepthRasterScreenVerts((uint16_t *)Memory::GetPointerWrite(gstate.getDepthBufRawAddress() | 0x04000000), gstate.DepthBufStride(),
-		prim, gstate.getScissorX1(), gstate.getScissorY1(), gstate.getScissorX2(), gstate.getScissorY2(),
-		tx, ty, tz, outVertCount);
+		tx, ty, tz, outVertCount, draw);
 }

GPU/Common/DrawEngineCommon.h

@@ -27,9 +27,9 @@
 #include "GPU/Common/GPUStateUtils.h"
 #include "GPU/Common/IndexGenerator.h"
 #include "GPU/Common/VertexDecoderCommon.h"
-#include "GPU/Common/DepthRaster.h"
 
 class VertexDecoder;
+struct DepthDraw;
 
 enum {
 	VERTEX_BUFFER_MAX = 65536,
@@ -177,6 +177,7 @@ protected:
 
 	void DepthRasterTransform(GEPrimitiveType prim, VertexDecoder *dec, uint32_t vertTypeID, int vertexCount);
 	void DepthRasterPredecoded(GEPrimitiveType prim, const void *inVerts, int numDecoded, VertexDecoder *dec, int vertexCount);
+	void FlushDepthDraw(const DepthDraw &draw);
 
 	static inline int IndexSize(u32 vtype) {
 		const u32 indexType = (vtype & GE_VTYPE_IDX_MASK);
@@ -358,4 +359,5 @@ protected:
 
 	float *depthTransformed_ = nullptr;
 	int *depthScreenVerts_ = nullptr;
+	uint16_t *depthIndices_ = nullptr;
 };

GPU/Common/SoftwareTransformCommon.cpp

@@ -186,7 +186,7 @@ void SoftwareTransform::Transform(int prim, u32 vertType, const DecVtxFormat &de
 			reader.Goto(index);
 			// TODO: Write to a flexible buffer, we don't always need all four components.
 			TransformedVertex &vert = transformed[index];
-			reader.ReadPos(vert.pos);
+			reader.ReadPosThrough(vert.pos);
 			vert.pos_w = 1.0f;
 
 			if (hasColor) {
@@ -225,7 +225,7 @@ void SoftwareTransform::Transform(int prim, u32 vertType, const DecVtxFormat &de
 			float pos[3];
 			Vec3f normal(0, 0, 1);
 			Vec3f worldnormal(0, 0, 1);
-			reader.ReadPos(pos);
+			reader.ReadPosNonThrough(pos);
 
 			float ruv[2] = { 0.0f, 0.0f };
 			if (reader.hasUV())
@@ -984,7 +984,7 @@ u32 NormalizeVertices(SimpleVertex *sverts, u8 *bufPtr, const u8 *inPtr, int low
 				nrm[1] = 0;
 				nrm[2] = 1.0f;
 			}
-			reader.ReadPos(pos);
+			reader.ReadPosAuto(pos);
 
 			// Apply skinning transform directly
 			float weights[8];
@@ -1029,7 +1029,7 @@ u32 NormalizeVertices(SimpleVertex *sverts, u8 *bufPtr, const u8 *inPtr, int low
 				sv.nrm.y = 0.0f;
 				sv.nrm.z = 1.0f;
 			}
-			reader.ReadPos((float *)&sv.pos);
+			reader.ReadPosAuto((float *)&sv.pos);
 		}
 	}
 

GPU/Common/VertexDecoderCommon.cpp

@@ -185,7 +185,7 @@ void PrintDecodedVertex(const VertexReader &vtx) {
 	}
 	// Etc..
 	float pos[3];
-	vtx.ReadPos(pos);
+	vtx.ReadPosAuto(pos);
 	printf("P: %f %f %f\n", pos[0], pos[1], pos[2]);
 }
 
@@ -1420,8 +1420,8 @@ static bool DecodedVertsAreSimilar(const VertexReader &vtx1, const VertexReader
 			return false;
 		}
 	}
-	vtx1.ReadPos(vec1.AsArray());
-	vtx2.ReadPos(vec2.AsArray());
+	vtx1.ReadPosAuto(vec1.AsArray());
+	vtx2.ReadPosAuto(vec2.AsArray());
 	float diff = LargestAbsDiff(vec1, vec2, 3);
 	if (diff >= 1.0 / 512.0f) {
 		WARN_LOG(Log::G3D, "Pos diff %f", diff);

GPU/Common/VertexDecoderCommon.h

@@ -124,7 +124,7 @@ class VertexReader {
 public:
 	VertexReader(const u8 *base, const DecVtxFormat &decFmt, int vtype) : base_(base), data_(base), decFmt_(decFmt), vtype_(vtype) {}
 
-	void ReadPos(float pos[3]) const {
+	void ReadPosAuto(float pos[3]) const {
 		// Only DEC_FLOAT_3 is supported.
 		const float *f = (const float *)(data_ + decFmt_.posoff);
 		pos[0] = f[0];
@@ -137,6 +137,21 @@ public:
 		}
 	}
 
+	void ReadPosThrough(float pos[3]) const {
+		// Only DEC_FLOAT_3 is supported.
+		const float *f = (const float *)(data_ + decFmt_.posoff);
+		pos[0] = f[0];
+		pos[1] = f[1];
+		// Integer value passed in a float. Clamped to 0, 65535.
+		pos[2] = (int)f[2] * (1.0f / 65535.0f);
+	}
+
+	void ReadPosNonThrough(float pos[3]) const {
+		// Only DEC_FLOAT_3 is supported.
+		const float *f = (const float *)(data_ + decFmt_.posoff);
+		memcpy(pos, f, 12);
+	}
+
 	void ReadPosThroughZ16(float pos[3]) const {
 		// Only DEC_FLOAT_3 is supported.
 		const float *f = (const float *)(data_ + decFmt_.posoff);

GPU/GPU.h

@@ -113,6 +113,7 @@ struct GPUStatistics {
 		numDepthRasterBackface = 0;
 		numDepthRasterNoPixels = 0;
 		numDepthRasterTooSmall = 0;
+		numDepthRasterZCulled = 0;
 		vertexGPUCycles = 0;
 		otherGPUCycles = 0;
 	}
@@ -159,6 +160,7 @@ struct GPUStatistics {
 	int numDepthRasterBackface;
 	int numDepthRasterNoPixels;
 	int numDepthRasterTooSmall;
+	int numDepthRasterZCulled;
 	// Flip count. Doesn't really belong here.
 	int numFlips;
 };

GPU/GPUCommonHW.cpp

@@ -1776,7 +1776,7 @@ size_t GPUCommonHW::FormatGPUStatsCommon(char *buffer, size_t size) {
 		"replacer: tracks %d references, %d unique textures\n"
 		"Cpy: depth %d, color %d, reint %d, blend %d, self %d\n"
 		"GPU cycles: %d (%0.1f per vertex)\n"
-		"Depth raster: %0.2f ms, %d prim, %d nopix, %d small, %d backface\n%s",
+		"Depth raster: %0.2f ms, %d prim, %d nopix, %d small, %d back, %d zcull\n%s",
 		gpuStats.msProcessingDisplayLists * 1000.0f,
 		gpuStats.numDrawSyncs,
 		gpuStats.numListSyncs,
@@ -1818,6 +1818,7 @@ size_t GPUCommonHW::FormatGPUStatsCommon(char *buffer, size_t size) {
 		gpuStats.numDepthRasterNoPixels,
 		gpuStats.numDepthRasterTooSmall,
 		gpuStats.numDepthRasterBackface,
+		gpuStats.numDepthRasterZCulled,
 		debugRecording_ ? "(debug-recording)" : ""
 	);
 }