From 7d69ba76f29a03bac00a857643f8e706f0a9bf13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 22 Sep 2026 13:42:46 -0600 Subject: [PATCH] x64 IR JIT: fix vcmp EN/NN, NI and TR EN/NN compared the source against an uninitialized temp, usually the previous lane's all-ones mask (a NaN), so every lane came out as NaN. NI used a less-than compare, which is false for a NaN. TR set the bit and then replaced it with a garbage bit from the temp. Found by cpu/vfpu/specials. Co-Authored-By: Claude Fable 5.1 --- Core/MIPS/x86/X64IRCompFPU.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Core/MIPS/x86/X64IRCompFPU.cpp b/Core/MIPS/x86/X64IRCompFPU.cpp index 1da0535e3f..facef8f324 100644 --- a/Core/MIPS/x86/X64IRCompFPU.cpp +++ b/Core/MIPS/x86/X64IRCompFPU.cpp @@ -434,7 +434,14 @@ void X64JitBackend::CompIR_FCompare(IRInst inst) { break; case VC_EN: case VC_NN: - CMPSS(tempReg, regs_.F(inst.src1), !condNegated ? CMP_UNORD : CMP_ORD); + // Compare src1 against itself: unordered exactly when it's a NaN. (tempReg holds whatever + // the previous lane left there, often an all-ones mask, which is a NaN too.) + if (cpu_info.bAVX) { + VCMPSS(tempReg, regs_.FX(inst.src1), regs_.F(inst.src1), !condNegated ? CMP_UNORD : CMP_ORD); + } else { + MOVAPS(tempReg, regs_.F(inst.src1)); + CMPSS(tempReg, regs_.F(inst.src1), !condNegated ? CMP_UNORD : CMP_ORD); + } break; case VC_EI: case VC_NI: @@ -445,7 +452,8 @@ void X64JitBackend::CompIR_FCompare(IRInst inst) { MOVAPS(tempReg, regs_.F(inst.src1)); ANDPS(tempReg, M(constants.noSignMask)); // rip accessible } - CMPSS(tempReg, M(constants.positiveInfinity), !condNegated ? CMP_EQ : CMP_LT); // rip accessible + // NEQ rather than LT so that a NaN counts as not infinite. + CMPSS(tempReg, M(constants.positiveInfinity), !condNegated ? CMP_EQ : CMP_NEQ); // rip accessible break; case VC_ES: case VC_NS: @@ -460,7 +468,7 @@ void X64JitBackend::CompIR_FCompare(IRInst inst) { break; case VC_TR: OR(32, regs_.R(IRREG_VFPU_CC), Imm8(affectedBit)); - takeBitFromTempReg = true; + takeBitFromTempReg = false; break; case VC_FL: AND(32, regs_.R(IRREG_VFPU_CC), Imm8(~affectedBit));