From b955ec70c94cfc2a77c04023c4e9a36a50cb5368 Mon Sep 17 00:00:00 2001
From: "Unknown W. Brackets" <checkins@unknownbrackets.org>
Date: Sun, 10 Feb 2019 09:58:01 -0800
Subject: [PATCH] GPU: Correct depth clamp range in range cull.

It was just wrong before, causing wrong culling when using a non-standard
viewport scale/center for depth.

Fixes #11701, fixes #11781.
---
 GPU/Common/GPUStateUtils.cpp           | 18 +++++++-----------
 GPU/Common/GPUStateUtils.h             |  2 +-
 GPU/Common/ShaderUniforms.cpp          | 24 +++++++++++++++++++++---
 GPU/Common/SoftwareTransformCommon.cpp |  2 +-
 4 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/GPU/Common/GPUStateUtils.cpp b/GPU/Common/GPUStateUtils.cpp
index 515caa74d9..92b1d5f974 100644
--- a/GPU/Common/GPUStateUtils.cpp
+++ b/GPU/Common/GPUStateUtils.cpp
@@ -520,7 +520,7 @@ float DepthSliceFactor() {
 }
 
 // This is used for float values which might not be integers, but are in the integer scale of 65535.
-static float ToScaledDepthFromInteger(float z) {
+float ToScaledDepthFromIntegerScale(float z) {
 	if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 		return z * (1.0f / 65535.0f);
 	}
@@ -536,10 +536,6 @@ static float ToScaledDepthFromInteger(float z) {
 	}
 }
 
-float ToScaledDepth(u16 z) {
-	return ToScaledDepthFromInteger((float)(int)z);
-}
-
 float FromScaledDepth(float z) {
 	if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 		return z * 65535.0f;
@@ -605,8 +601,8 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 		out.viewportY = renderY + displayOffsetY;
 		out.viewportW = curRTWidth * renderWidthFactor;
 		out.viewportH = curRTHeight * renderHeightFactor;
-		out.depthRangeMin = ToScaledDepthFromInteger(0);
-		out.depthRangeMax = ToScaledDepthFromInteger(65536);
+		out.depthRangeMin = ToScaledDepthFromIntegerScale(0);
+		out.depthRangeMax = ToScaledDepthFromIntegerScale(65536);
 	} else {
 		// These we can turn into a glViewport call, offset by offsetX and offsetY. Math after.
 		float vpXScale = gstate.getViewportXScale();
@@ -715,11 +711,11 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 		if (!gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
 			zScale = 1.0f;
 			zOffset = 0.0f;
-			out.depthRangeMin = ToScaledDepthFromInteger(vpZCenter - vpZScale);
-			out.depthRangeMax = ToScaledDepthFromInteger(vpZCenter + vpZScale);
+			out.depthRangeMin = ToScaledDepthFromIntegerScale(vpZCenter - vpZScale);
+			out.depthRangeMax = ToScaledDepthFromIntegerScale(vpZCenter + vpZScale);
 		} else {
-			out.depthRangeMin = ToScaledDepthFromInteger(minz);
-			out.depthRangeMax = ToScaledDepthFromInteger(maxz);
+			out.depthRangeMin = ToScaledDepthFromIntegerScale(minz);
+			out.depthRangeMax = ToScaledDepthFromIntegerScale(maxz);
 		}
 
 		// OpenGL will clamp these for us anyway, and Direct3D will error if not clamped.
diff --git a/GPU/Common/GPUStateUtils.h b/GPU/Common/GPUStateUtils.h
index 957c6c42b3..6cf95f84b7 100644
--- a/GPU/Common/GPUStateUtils.h
+++ b/GPU/Common/GPUStateUtils.h
@@ -72,7 +72,7 @@ struct ViewportAndScissor {
 	bool dirtyDepth;
 };
 void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, float renderHeight, int bufferWidth, int bufferHeight, ViewportAndScissor &out);
-float ToScaledDepth(u16 z);
+float ToScaledDepthFromIntegerScale(float z);
 float FromScaledDepth(float z);
 float DepthSliceFactor();
 
diff --git a/GPU/Common/ShaderUniforms.cpp b/GPU/Common/ShaderUniforms.cpp
index 26bdbd7c17..f8fb3456e9 100644
--- a/GPU/Common/ShaderUniforms.cpp
+++ b/GPU/Common/ShaderUniforms.cpp
@@ -42,9 +42,27 @@ void CalcCullRange(float minValues[4], float maxValues[4], bool flipViewport, bo
 		return (pspViewport - gstate_c.vpYOffset) * heightScale;
 	};
 	auto reverseViewportZ = [hasNegZ](float z) {
-		float pspViewport = (z - gstate.getViewportZCenter()) * (1.0f / gstate.getViewportZScale());
-		// Differs from GLES: depth is 0 to 1, not -1 to 1.
-		float realViewport = (pspViewport - gstate_c.vpZOffset) * gstate_c.vpDepthScale;
+		float vpZScale = gstate.getViewportZScale();
+		float vpZCenter = gstate.getViewportZCenter();
+
+		float scale, center;
+		if (gstate_c.Supports(GPU_SUPPORTS_ACCURATE_DEPTH)) {
+			// These are just the reverse of the formulas in GPUStateUtils.
+			float halfActualZRange = vpZScale * (1.0f / gstate_c.vpDepthScale);
+			float minz = -((gstate_c.vpZOffset * halfActualZRange) - vpZCenter) - halfActualZRange;
+
+			// In accurate depth mode, we're comparing against a value scaled to (minz, maxz).
+			// And minz might be very negative, (e.g. if we're clamping in that direction.)
+			scale = halfActualZRange;
+			center = minz + halfActualZRange;
+		} else {
+			// In old-style depth mode, we're comparing against a value scaled to viewport.
+			// (and possibly incorrectly clipped against it.)
+			scale = vpZScale;
+			center = vpZCenter;
+		}
+
+		float realViewport = (z - center) * (1.0f / scale);
 		return hasNegZ ? realViewport : (realViewport * 0.5f + 0.5f);
 	};
 	auto sortPair = [](float a, float b) {
diff --git a/GPU/Common/SoftwareTransformCommon.cpp b/GPU/Common/SoftwareTransformCommon.cpp
index 66909cb2d0..4a77731f00 100644
--- a/GPU/Common/SoftwareTransformCommon.cpp
+++ b/GPU/Common/SoftwareTransformCommon.cpp
@@ -460,7 +460,7 @@ void SoftwareTransform(
 		if (params->allowSeparateAlphaClear || (alphaMatchesColor && depthMatchesStencil)) {
 			result->color = transformed[1].color0_32;
 			// Need to rescale from a [0, 1] float.  This is the final transformed value.
-			result->depth = ToScaledDepth((s16)(int)(transformed[1].z * 65535.0f));
+			result->depth = ToScaledDepthFromIntegerScale((int)(transformed[1].z * 65535.0f));
 			result->action = SW_CLEAR;
 			gpuStats.numClears++;
 			return;