From f949fd518787089ec9b7018c19f2b3fe2492093e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Mon, 5 Oct 2026 15:09:55 -0600 Subject: [PATCH] SoftGPU: Treat a NaN screen coordinate as outside the range The range check was written as tests for outside, which are all false for NaN, so a NaN vertex passed and was converted to an integer. That differs by CPU: arm64 gives 0, which lands off screen, while x86-64 gives INT_MIN, which wraps, and the triangle's area overflowed into a huge sliver. In the recording-opengl and recording-vulkan dumps that drew a beam across the top right that a PSP replay of the dump doesn't show. Now such a vertex is flagged, and its triangles dropped, on both. Co-Authored-By: Claude Opus 5.5 (1M context) --- GPU/Software/TransformUnit.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/GPU/Software/TransformUnit.cpp b/GPU/Software/TransformUnit.cpp index 4711bb4227..cd020c4ae4 100644 --- a/GPU/Software/TransformUnit.cpp +++ b/GPU/Software/TransformUnit.cpp @@ -219,7 +219,9 @@ static ScreenCoords ClipToScreenInternal(Vec3f scaled, const ClipCoords &coords, if constexpr (depthClamp) { // A vertex the near plane clips away (z < -w) doesn't set the flag, even for x and y. One exactly on // the plane isn't clipped, so its range counts (gpu/clipping/guardband). - if ((alwaysCheckRange || !(coords.z < -coords.w)) && (scaled.x >= SCREEN_BOUND || scaled.y >= SCREEN_BOUND || scaled.x < 0 || scaled.y < 0)) { + // Written as a test for inside, so a NaN coordinate is outside: converting it to an integer below differs + // by CPU, and on x86-64 its triangles became huge slivers. + if ((alwaysCheckRange || !(coords.z < -coords.w)) && !(scaled.x < SCREEN_BOUND && scaled.y < SCREEN_BOUND && scaled.x >= 0 && scaled.y >= 0)) { *outside_range_flag = true; } @@ -227,7 +229,7 @@ static ScreenCoords ClipToScreenInternal(Vec3f scaled, const ClipCoords &coords, scaled.z = 0.f; else if (scaled.z > 65535.0f) scaled.z = 65535.0f; - } else if (scaled.x > SCREEN_BOUND || scaled.y >= SCREEN_BOUND || scaled.x < 0 || scaled.y < 0 || scaled.z < 0.0f || scaled.z >= 65536.0f) { + } else if (!(scaled.x <= SCREEN_BOUND && scaled.y < SCREEN_BOUND && scaled.x >= 0 && scaled.y >= 0 && scaled.z >= 0.0f && scaled.z < 65536.0f)) { *outside_range_flag = true; }