Re-implement range culling

GPU/Common/ShaderUniforms.cpp

@@ -93,6 +93,7 @@ void BaseUpdateUniforms(UB_VS_FS_Base *ub, uint64_t dirtyUniforms, bool flipView
 		ub->vpScale[0] = gstate.getViewportXScale();
 		ub->vpScale[1] = gstate.getViewportYScale();
 		ub->vpScale[2] = gstate.getViewportZScale();
+		ub->NaN = std::numeric_limits<float>::quiet_NaN();  // Used in the shader for range culling.
 		ub->vpOffset[0] = gstate.getViewportXCenter();
 		ub->vpOffset[1] = gstate.getViewportYCenter();
 		ub->vpOffset[2] = gstate.getViewportZCenter();

GPU/Common/ShaderUniforms.h

@@ -25,7 +25,7 @@ struct alignas(16) UB_VS_FS_Base {
 	float world[12];
 	float tex[12];
 	float xywh[4];  // later, we could invert w and h here to avoid division.
-	float vpScale[3]; float padding1;  // TODO Use padding1/2/3 for something.
+	float vpScale[3]; float NaN;
 	float vpOffset[3]; float padding2;
 	float rasterOffset[2]; float padding3[2];
 	float uvScaleOffset[4];
@@ -51,7 +51,7 @@ R"(  mat4 u_proj;
   mat3x4 u_world;
   mat3x4 u_texmtx;
   vec4 u_xywh;
-  vec4 u_vpScale;  // w = offsetX
+  vec3 u_vpScale; float u_NaN; // w = offsetX
   vec4 u_vpOffset; // w = offsetY
   vec2 u_rasterOffset; vec2 pad0;
   vec4 u_uvscaleoffset;

GPU/Common/VertexShaderGenerator.cpp

@@ -709,6 +709,8 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 		}
 	}
 
+	WRITE(p, "  bool zClipped = false;\n");
+
 	if (!useHWTransform) {
 		// Simple pass-through of vertex data to fragment shader
 		if (texCoordInVec3) {
@@ -834,10 +836,17 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 			}
 		}
 
-		// Perform the perspective projection and viewport transform. (We'll undo it later, after we applied the viewport).
-		// In software transform mode, this is performed in software.
+		// We're in clip space, so here we check for virtual clipping. We don't actually clip, but we check if the actual hardware would have clipped.
+		// If so, we skip the "range culling" (x and y out-of-bounds checks) since they wouldn't have happened, most likely.
+		WRITE(p, "  if (outPos.z < -outPos.w || outPos.z > outPos.w) {\n");
+		WRITE(p, "    zClipped = true;\n");
+		WRITE(p, "  }\n");
+
+		// Perform the perspective projection and viewport transform. (We'll have to undo the division before passing the coordinate along).
+		// In software transform mode, this is performed in on the CPU.
 		WRITE(p, "  outPos.xyz = (outPos.xyz / outPos.w) * u_vpScale.xyz + u_vpOffset.xyz;\n");
 
+
 		// TODO: Declare variables for dots for shade mapping if needed.
 
 		const char *srcCol = "color0";
@@ -1194,6 +1203,16 @@ bool GenerateVertexShader(const VShaderID &id, char *buffer, const ShaderLanguag
 	}
 
 	if (!isModeThrough) {
+		// Cull against X and Y limits (unless the GPU has a certain driver bug).
+		// It's not clear what the limits should be in through mode though, although I'm sure they exist.
+		if (gstate_c.Use(GPU_USE_VS_RANGE_CULLING)) {
+			WRITE(p, "  if (!zClipped && (outPos.x < 0.0 || outPos.y < 0.0 || outPos.x >= 4096.0 || outPos.y >= 4096.0)) {\n");
+			// Discard the whole triangle by setting one vertex to NaN.
+			WRITE(p, "    outPos = vec4(u_NaN, u_NaN, u_NaN, u_NaN);\n");
+			// TODO: We could just return here. But we can also just keep going to avoid branching, the NaNs will propagate.
+			WRITE(p, "  }\n");
+		}
+
 		// Apply raster offset after the range culling.
 		WRITE(p, "  outPos.xy -= u_rasterOffset.xy;\n");
 	}