From 92e39572648b39f2c8ef4ff5532dda51af1403a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 13 Oct 2026 20:39:10 +0200 Subject: [PATCH] VFPU: Produce the PSP's NaNs in the single-lane ops, on every core The VFPU never keeps a NaN's payload (cpu/vfpu/nan_gen): vadd and vsub produce 0x7F800001, and vmul, vdiv, vscl and vcrs the same with the sign of s ^ t. The cores returned whatever NaN the host made, which differs between x86 and ARM64. Now all four cores on both hosts match the PSP for these ops. The horizontal ops (vdot and friends) are left for when they move to the accurate dot product. The IR gets FVfpuNaN and FVfpuProductNaN, scalar and Vec4, run after the arithmetic. The native backends check for a NaN and branch past the fixup, which only runs when there is one. PurgeTemps no longer renames the dest of an FPR op that also reads it, which the product ops are the first to do. vsat0 and vsat1 also keep a NaN's bits on ARM64 now, where FMIN/FMAX quieted a signaling NaN. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/ARM64/Arm64IRCompFPU.cpp | 52 ++++++++++++++----- Core/MIPS/ARM64/Arm64IRCompVec.cpp | 32 ++++++++++++ Core/MIPS/IR/IRCompVFPU.cpp | 50 ++++++++++++++++-- Core/MIPS/IR/IRInst.cpp | 4 ++ Core/MIPS/IR/IRInst.h | 6 +++ Core/MIPS/IR/IRInterpreter.cpp | 53 ++++++++++++++++++++ Core/MIPS/IR/IRNativeCommon.cpp | 4 ++ Core/MIPS/IR/IRPassSimplify.cpp | 12 ++++- Core/MIPS/InterpreterVFPU.cpp | 47 +++++++++++------ Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp | 27 ++++++++++ Core/MIPS/LoongArch64/LoongArch64CompVec.cpp | 49 ++++++++++++++++++ Core/MIPS/RiscV/RiscVCompFPU.cpp | 32 ++++++++++++ Core/MIPS/RiscV/RiscVCompVec.cpp | 32 ++++++++++++ Core/MIPS/x86/X64IRCompFPU.cpp | 30 ++++++++++- Core/MIPS/x86/X64IRCompVec.cpp | 45 +++++++++++++++++ Core/MIPS/x86/X64IRJit.h | 1 + 16 files changed, 441 insertions(+), 35 deletions(-) diff --git a/Core/MIPS/ARM64/Arm64IRCompFPU.cpp b/Core/MIPS/ARM64/Arm64IRCompFPU.cpp index 08dd479256..541de78bc0 100644 --- a/Core/MIPS/ARM64/Arm64IRCompFPU.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompFPU.cpp @@ -76,6 +76,36 @@ void Arm64JitBackend::CompIR_FArith(IRInst inst) { fp_.FNEG(regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1)); + if (inst.dest != inst.src1) + fp_.FMOV(regs_.F(inst.dest), regs_.F(inst.src1)); + FixupBranch notNaN = B(CC_VC); + MOVI2R(SCRATCH1, 0x7F800001); + fp_.FMOV(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + fp_.FCMP(regs_.F(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = B(CC_VC); + fp_.FMOV(SCRATCH1, regs_.F(inst.src1)); + fp_.FMOV(SCRATCH2, regs_.F(inst.src2)); + EOR(SCRATCH1, SCRATCH1, SCRATCH2); + ANDI2R(SCRATCH1, SCRATCH1, 0x80000000, SCRATCH2); + ORRI2R(SCRATCH1, SCRATCH1, 0x7F800001, SCRATCH2); + fp_.FMOV(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; @@ -455,22 +485,20 @@ void Arm64JitBackend::CompIR_FSat(IRInst inst) { switch (inst.op) { case IROp::FSat0_1: - regs_.Map(inst); - fp_.MOVI2F(SCRATCHF1, 1.0f); - // Note that FMAX takes the larger of the two zeros, which is what we want. - fp_.MOVI2F(SCRATCHF2, 0.0f); - - fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1); - fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2); - break; - case IROp::FSatMinus1_1: regs_.Map(inst); fp_.MOVI2F(SCRATCHF1, 1.0f); - fp_.FNEG(SCRATCHF2, SCRATCHF1); + // Note that FMAX takes the larger of the two zeros, which is what we want. + if (inst.op == IROp::FSat0_1) + fp_.MOVI2F(SCRATCHF2, 0.0f); + else + fp_.FNEG(SCRATCHF2, SCRATCHF1); - fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1); - fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2); + // A NaN passes through with its bits, which FMIN/FMAX would quiet (cpu/vfpu/nan_gen). + fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1)); + fp_.FMIN(SCRATCHF1, regs_.F(inst.src1), SCRATCHF1); + fp_.FMAX(SCRATCHF1, SCRATCHF1, SCRATCHF2); + fp_.FCSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, CC_VS); break; default: diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 8426232d29..8a7ae02720 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -140,6 +140,38 @@ void Arm64JitBackend::CompIR_VecArith(IRInst inst) { fp_.SCVTF(32, regs_.FQ(inst.dest), regs_.FQ(inst.src1), inst.src2 & 0x1F); break; + case IROp::Vec4VfpuNaN: + case IROp::Vec4VfpuProductNaN: + { + // The operands of the product are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.op == IROp::Vec4VfpuNaN || (inst.dest != inst.src1 && inst.dest != inst.src2)); + regs_.Map(inst); + if (inst.op == IROp::Vec4VfpuNaN && inst.dest != inst.src1) + fp_.MOV(regs_.FQ(inst.dest), regs_.FQ(inst.src1)); + // FMAXV gives a NaN if any lane is one. + fp_.FMAXV(SCRATCHF1, regs_.FQ(inst.dest)); + fp_.FCMP(SCRATCHF1, SCRATCHF1); + FixupBranch noNaN = B(CC_VC); + + // Rare: replace the NaN lanes with 0x7F800001, signed like the product for that op. + ARM64Reg orderedMask = EncodeRegToQuad(SCRATCHF1); + ARM64Reg nanReg = EncodeRegToQuad(SCRATCHF2); + fp_.FCMEQ(32, orderedMask, regs_.FQ(inst.dest), regs_.FQ(inst.dest)); + if (inst.op == IROp::Vec4VfpuProductNaN) { + fp_.EOR(nanReg, regs_.FQ(inst.src1), regs_.FQ(inst.src2)); + fp_.USHR(32, nanReg, nanReg, 31); + fp_.SHL(32, nanReg, nanReg, 31); + fp_.ORR(32, nanReg, 0x7F, 24); + } else { + fp_.MOVI(32, nanReg, 0x7F, 24); + } + fp_.ORR(32, nanReg, 0x80, 16); + fp_.ORR(32, nanReg, 0x01, 0); + fp_.BIF(regs_.FQ(inst.dest), nanReg, orderedMask); + SetJumpTarget(noNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index caaade1540..326687ba2f 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -1193,11 +1193,25 @@ namespace MIPSComp { break; } + // Sums and products make the VFPU's own NaN. vmin and vmax pass a NaN through, and vslt and + // vsge don't make one. + const bool sum = type == VecDo3Op::VADD || type == VecDo3Op::VSUB; + const bool product = type == VecDo3Op::VMUL || type == VecDo3Op::VDIV; if (IsVec4(sz, dregs) && IsVec4(sz, sregs) && IsVec4(sz, tregs)) { - if (opFunc != IROp::Nop) { - ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); - } else { + if (opFunc == IROp::Nop) DISABLE; + if (!product) { + ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); + if (sum) + ir.Write(IROp::Vec4VfpuNaN, dregs[0], dregs[0]); + } else if (dregs[0] != sregs[0] && dregs[0] != tregs[0]) { + ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); + ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], tregs[0]); + } else { + // The NaN's sign comes from the operands, so they have to survive the op. + ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]); + ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0); } ApplyPrefixD(dregs, sz, _VD); return; @@ -1207,6 +1221,10 @@ namespace MIPSComp { if (opFunc == IROp::Nop) DISABLE; ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]); + if (product) + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]); + else if (sum) + ir.Write(IROp::Vec4VfpuNaN, IRVTEMP_0, IRVTEMP_0); ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); ApplyPrefixD(dregs, sz, _VD); return; @@ -1220,17 +1238,22 @@ namespace MIPSComp { for (int i = 0; i < n; ++i) { switch (type) { + // tempregs[i] is never one of the operands, which the product NaN needs. case VecDo3Op::VADD: // d[i] = s[i] + t[i]; break; //vadd ir.Write(IROp::FAdd, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]); break; case VecDo3Op::VSUB: // d[i] = s[i] - t[i]; break; //vsub ir.Write(IROp::FSub, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]); break; case VecDo3Op::VDIV: // d[i] = s[i] / t[i]; break; //vdiv ir.Write(IROp::FDiv, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]); break; case VecDo3Op::VMUL: // d[i] = s[i] * t[i]; break; //vmul ir.Write(IROp::FMul, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]); break; case VecDo3Op::VMIN: // vmin ir.Write(IROp::FMin, tempregs[i], sregs[i], tregs[i]); @@ -1843,20 +1866,35 @@ namespace MIPSComp { } if (!overlap || (vs == vd && IsOverlapSafe(treg, n, dregs))) { + // The NaN's sign needs t in every lane (the T prefix temp is free, see above.) if (IsVec4(sz, sregs) && IsVec4(sz, dregs)) { - ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg); + int tlane = treg & 3; + ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane)); + if (dregs[0] != sregs[0]) { + ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], IRVTEMP_PFX_T); + } else { + ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T); + ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0); + } ApplyPrefixD(dregs, sz, vd); return; } else if (IsVec3of4(sz, sregs) && IsVec3of4(sz, dregs) && opts.preferVec4) { + int tlane = treg & 3; + ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane)); ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T); ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); ApplyPrefixD(dregs, sz, vd); return; } } + // tempregs[i] is never s[i] or t, which the product NaN needs. for (int i = 0; i < n; i++) { ir.Write(IROp::FMul, tempregs[i], sregs[i], treg); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], treg); } for (int i = 0; i < n; i++) { @@ -2017,6 +2055,7 @@ namespace MIPSComp { ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_S, sregs[0], VFPU_SWIZZLE(1, 2, 0, 3)); ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, tregs[0], VFPU_SWIZZLE(2, 0, 1, 3)); ir.Write(IROp::Vec4Mul, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T); // Now just retain w and blend in our values. ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); } else { @@ -2030,8 +2069,11 @@ namespace MIPSComp { } ir.Write(IROp::FMul, tempregs[0], sregs[1], tregs[2]); + ir.Write(IROp::FVfpuProductNaN, tempregs[0], sregs[1], tregs[2]); ir.Write(IROp::FMul, tempregs[1], sregs[2], tregs[0]); + ir.Write(IROp::FVfpuProductNaN, tempregs[1], sregs[2], tregs[0]); ir.Write(IROp::FMul, tempregs[2], sregs[0], tregs[1]); + ir.Write(IROp::FVfpuProductNaN, tempregs[2], sregs[0], tregs[1]); for (int i = 0; i < n; i++) { if (tempregs[i] != dregs[i]) diff --git a/Core/MIPS/IR/IRInst.cpp b/Core/MIPS/IR/IRInst.cpp index 8cd54e8985..18d60d56af 100644 --- a/Core/MIPS/IR/IRInst.cpp +++ b/Core/MIPS/IR/IRInst.cpp @@ -136,6 +136,8 @@ static const IRMeta irMeta[] = { { IROp::FCmp, "FCmp", "mFF" }, { IROp::FSat0_1, "FSat(0 - 1)", "FF" }, { IROp::FSatMinus1_1, "FSat(-1 - 1)", "FF" }, + { IROp::FVfpuNaN, "FVfpuNaN", "FF" }, + { IROp::FVfpuProductNaN, "FVfpuProductNaN", "FFF", IRFLAG_SRC3DST }, { IROp::FMovFromGPR, "FMovFromGPR", "FG" }, { IROp::FMovToGPR, "FMovToGPR", "GF" }, { IROp::OptFMovToGPRShr8, "OptFMovToGPRShr8", "GF" }, @@ -175,6 +177,8 @@ static const IRMeta irMeta[] = { { IROp::Vec4Slt, "Vec4Slt", "VVV" }, { IROp::Vec4Sge, "Vec4Sge", "VVV" }, { IROp::Vec4CvtScaledSW, "Vec4CvtScaledSW", "VVI" }, + { IROp::Vec4VfpuNaN, "Vec4VfpuNaN", "VV" }, + { IROp::Vec4VfpuProductNaN, "Vec4VfpuProductNaN", "VVV", IRFLAG_SRC3DST }, // Pack/Unpack { IROp::Vec2Unpack16To31, "Vec2Unpack16To31", "2F" }, // Note that the result is shifted down by 1, hence 31 diff --git a/Core/MIPS/IR/IRInst.h b/Core/MIPS/IR/IRInst.h index ab8b5c38d3..dc41416228 100644 --- a/Core/MIPS/IR/IRInst.h +++ b/Core/MIPS/IR/IRInst.h @@ -163,6 +163,10 @@ enum class IROp : uint8_t { FSat0_1, FSatMinus1_1, + // The VFPU's NaNs (see VfpuSumNaN in InterpreterVFPU.cpp): src1 or 0x7F800001 if it's a NaN, and + // dest (read too) or 0x7F800001 with the sign of src1 ^ src2 if dest is a NaN. + FVfpuNaN, + FVfpuProductNaN, FpCondFromReg, FpCondToReg, @@ -208,6 +212,8 @@ enum class IROp : uint8_t { Vec4Sge, // vi2f: each lane's int to float, divided by 2^src2. Vec4CvtScaledSW, + Vec4VfpuNaN, + Vec4VfpuProductNaN, // vx2i Vec2Unpack16To31, // Note that the result is shifted down by 1, hence 31 diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp index b399fe4a2a..14be112dda 100644 --- a/Core/MIPS/IR/IRInterpreter.cpp +++ b/Core/MIPS/IR/IRInterpreter.cpp @@ -50,6 +50,10 @@ alignas(16) static const uint32_t noSignMask[4] = { 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, }; +alignas(16) static const uint32_t vfpuNaN[4] = { + 0x7F800001, 0x7F800001, 0x7F800001, 0x7F800001, +}; + alignas(16) static const uint32_t lowBytesMask[4] = { 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, }; @@ -245,6 +249,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { dispatch[(int)IROp::Vec4Scale] = &&L_Vec4Scale; dispatch[(int)IROp::Vec4Neg] = &&L_Vec4Neg; dispatch[(int)IROp::Vec4Abs] = &&L_Vec4Abs; + dispatch[(int)IROp::Vec4VfpuNaN] = &&L_Vec4VfpuNaN; + dispatch[(int)IROp::Vec4VfpuProductNaN] = &&L_Vec4VfpuProductNaN; dispatch[(int)IROp::Vec2Unpack16To31] = &&L_Vec2Unpack16To31; dispatch[(int)IROp::Vec2Unpack16To32] = &&L_Vec2Unpack16To32; dispatch[(int)IROp::Vec4Unpack8To32] = &&L_Vec4Unpack8To32; @@ -316,6 +322,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { dispatch[(int)IROp::FNeg] = &&L_FNeg; dispatch[(int)IROp::FSat0_1] = &&L_FSat0_1; dispatch[(int)IROp::FSatMinus1_1] = &&L_FSatMinus1_1; + dispatch[(int)IROp::FVfpuNaN] = &&L_FVfpuNaN; + dispatch[(int)IROp::FVfpuProductNaN] = &&L_FVfpuProductNaN; dispatch[(int)IROp::FSign] = &&L_FSign; dispatch[(int)IROp::FpCondFromReg] = &&L_FpCondFromReg; dispatch[(int)IROp::FpCondToReg] = &&L_FpCondToReg; @@ -759,6 +767,44 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { IR_NEXT; } + IR_CASE(Vec4VfpuNaN) + { +#if PPSSPP_ARCH(SSE2) + __m128 v = _mm_load_ps(&mips->f[inst->src1]); + __m128 isNaN = _mm_cmpunord_ps(v, v); + _mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, _mm_load_ps((const float *)vfpuNaN)))); +#elif PPSSPP_ARCH(ARM_NEON) + float32x4_t v = vld1q_f32(&mips->f[inst->src1]); + vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, vld1q_f32((const float *)vfpuNaN))); +#else + for (int i = 0; i < 4; i++) + mips->fi[inst->dest + i] = my_isnan(mips->f[inst->src1 + i]) ? 0x7F800001 : mips->fi[inst->src1 + i]; +#endif + IR_NEXT; + } + + IR_CASE(Vec4VfpuProductNaN) + { +#if PPSSPP_ARCH(SSE2) + __m128 v = _mm_load_ps(&mips->f[inst->dest]); + __m128 isNaN = _mm_cmpunord_ps(v, v); + __m128 sign = _mm_and_ps(_mm_xor_ps(_mm_load_ps(&mips->f[inst->src1]), _mm_load_ps(&mips->f[inst->src2])), _mm_load_ps((const float *)signBits)); + __m128 nan = _mm_or_ps(sign, _mm_load_ps((const float *)vfpuNaN)); + _mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, nan))); +#elif PPSSPP_ARCH(ARM_NEON) + float32x4_t v = vld1q_f32(&mips->f[inst->dest]); + uint32x4_t sign = vandq_u32(veorq_u32(vld1q_u32(&mips->fi[inst->src1]), vld1q_u32(&mips->fi[inst->src2])), vld1q_u32(signBits)); + float32x4_t nan = vreinterpretq_f32_u32(vorrq_u32(sign, vld1q_u32(vfpuNaN))); + vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, nan)); +#else + for (int i = 0; i < 4; i++) { + if (my_isnan(mips->f[inst->dest + i])) + mips->fi[inst->dest + i] = 0x7F800001 | ((mips->fi[inst->src1 + i] ^ mips->fi[inst->src2 + i]) & 0x80000000); + } +#endif + IR_NEXT; + } + IR_CASE(Vec2Unpack16To31) { const u32 dest = inst->dest; @@ -1245,6 +1291,13 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { IR_CASE(FSatMinus1_1) mips->f[inst->dest] = vfpu_clamp(mips->f[inst->src1], -1.0f, 1.0f); IR_NEXT; + IR_CASE(FVfpuNaN) + mips->fi[inst->dest] = my_isnan(mips->f[inst->src1]) ? 0x7F800001 : mips->fi[inst->src1]; + IR_NEXT; + IR_CASE(FVfpuProductNaN) + if (my_isnan(mips->f[inst->dest])) + mips->fi[inst->dest] = 0x7F800001 | ((mips->fi[inst->src1] ^ mips->fi[inst->src2]) & 0x80000000); + IR_NEXT; IR_CASE(FSign) { diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index b24d5a5b84..373e2dc3a7 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -348,6 +348,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { case IROp::FDiv: case IROp::FSqrt: case IROp::FNeg: + case IROp::FVfpuNaN: + case IROp::FVfpuProductNaN: CompIR_FArith(inst); break; @@ -426,6 +428,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { case IROp::Vec4Slt: case IROp::Vec4Sge: case IROp::Vec4CvtScaledSW: + case IROp::Vec4VfpuNaN: + case IROp::Vec4VfpuProductNaN: CompIR_VecArith(inst); break; diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index bc8c5a19b7..92824ca9df 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -1031,6 +1031,14 @@ bool IRMutatesDestGPR(const IRInstMeta &inst, int reg) { return (inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'G' && inst.src3 == reg; } +// Whether inst reads the FPR it writes (reg counted from 32, like the checks in PurgeTemps). +static bool IRMutatesDestFPR(const IRInstMeta &inst, int reg) { + if ((inst.m.flags & IRFLAG_SRC3DST) == 0) + return false; + int len = inst.m.types[0] == 'F' ? 1 : (inst.m.types[0] == 'V' ? 4 : 0); + return reg >= inst.src3 + 32 && reg < inst.src3 + 32 + len; +} + bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { CONDITIONAL_DISABLE; std::vector insts; @@ -1192,7 +1200,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { if (numDest > 1 && check.srcReg >= destRegs[0] + 32 && check.srcReg < destRegs[0] + 32 + numDest) cannotReplace = true; // Nor where it's read as src3 that's also the destination, which renaming would move. - if ((inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'F' && inst.src3 + 32 == check.reg) + if (IRMutatesDestFPR(inst, check.reg)) cannotReplace = true; if (!cannotReplace && check.srcReg >= 32 && lastWrittenTo[check.srcReg] < check.index) { // This is probably not worth doing unless we can get rid of a temp. @@ -1219,7 +1227,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { // Let's not bother. check.reg = 0; } - } else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch) { + } else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch && !IRMutatesDestFPR(insts[check.index], check.reg)) { // A swap could be profitable if this is a temp, and maybe in other cases. // These can happen a lot from mask regs, etc. // But make sure no other changes happened between. diff --git a/Core/MIPS/InterpreterVFPU.cpp b/Core/MIPS/InterpreterVFPU.cpp index fa091cb114..195d14cb7c 100644 --- a/Core/MIPS/InterpreterVFPU.cpp +++ b/Core/MIPS/InterpreterVFPU.cpp @@ -75,7 +75,7 @@ #define M_SQRT1_2 0.707106781186547524401f #endif -// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). +// The VFPU's own handling of denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). static const bool USE_VFPU_RESULT_FIXUP = false; union FloatBits { @@ -84,6 +84,26 @@ union FloatBits { int i[4]; }; +// The VFPU never keeps a NaN's payload: every NaN it produces is 0x7F800001, negative for a product +// or quotient of operands of different signs (cpu/vfpu/nan_gen). Host NaNs differ by host. +static inline float VfpuSumNaN(float r) { + if (!my_isnan(r)) + return r; + FP32 nan; + nan.u = 0x7F800001; + return nan.f; +} + +static inline float VfpuProductNaN(float r, float a, float b) { + if (!my_isnan(r)) + return r; + FP32 fa, fb, nan; + fa.f = a; + fb.f = b; + nan.u = 0x7F800001 | ((fa.u ^ fb.u) & 0x80000000); + return nan.f; +} + // Preserves NaN in first param, takes sign of equal second param. // Technically, std::max may do this but it's undefined. inline float nanmax(float f, float cst) @@ -1436,10 +1456,9 @@ namespace MIPSInt u32 tprefixAdd = VFPU_SWIZZLE(2, 0, 1, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), sz); - d[0] = s[0] * t[0]; - d[1] = s[1] * t[1]; - d[2] = s[2] * t[2]; - d[3] = s[3] * t[3]; + for (int i = 0; i < 4; i++) { + d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); + } ApplyPrefixD(mips, d, sz); WriteVector(mips, d, sz, vd); PC += 4; @@ -1544,7 +1563,7 @@ namespace MIPSInt int n = GetNumVectorElements(sz); for (int i = 0; i < n; i++) { - d[i] = s[i] * t[i]; + d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); } ApplyPrefixD(mips, d, sz); WriteVector(mips, d, sz, vd); @@ -2105,18 +2124,14 @@ namespace MIPSInt for (int i = 0; i < (int)n; i++) { switch (optype) { - case 0: d.f[i] = s[i] + t[i]; break; //vadd - case 1: d.f[i] = s[i] - t[i]; break; //vsub - case 7: d.f[i] = s[i] / t[i]; break; //vdiv - case 8: d.f[i] = s[i] * t[i]; break; //vmul + case 0: d.f[i] = VfpuSumNaN(s[i] + t[i]); break; //vadd + case 1: d.f[i] = VfpuSumNaN(s[i] - t[i]); break; //vsub + case 7: d.f[i] = VfpuProductNaN(s[i] / t[i], s[i], t[i]); break; //vdiv + case 8: d.f[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); break; //vmul } - if (USE_VFPU_RESULT_FIXUP) { - if (my_isnan(d.f[i])) { - d.u[i] = (d.u[i] & 0xff800001) | 1; - } else if ((d.u[i] & 0x7F800000) == 0) { - d.u[i] &= 0xFF800000; - } + if (USE_VFPU_RESULT_FIXUP && (d.u[i] & 0x7F800000) == 0) { + d.u[i] &= 0xFF800000; } } diff --git a/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp b/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp index e1335944c9..f4f20f865b 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp @@ -70,6 +70,33 @@ void LoongArch64JitBackend::CompIR_FArith(IRInst inst) { FNEG_S(regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + regs_.Map(inst); + FCMP_COND_S(FCC0, regs_.F(inst.src1), regs_.F(inst.src1), LoongArch64Fcond::CUN); + QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1); + FSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, FCC0); + break; + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + FCMP_COND_S(FCC0, regs_.F(inst.dest), regs_.F(inst.dest), LoongArch64Fcond::CUN); + MOVCF2GR(SCRATCH1, FCC0); + FixupBranch notNaN = BEQZ(SCRATCH1); + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1)); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLI_W(SCRATCH1, SCRATCH1, 31); + SLLI_W(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + MOVGR2FR_W(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp index 6a32b6239b..2a5bc8a99f 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp @@ -431,6 +431,55 @@ void LoongArch64JitBackend::CompIR_VecArith(IRInst inst) { } break; + case IROp::Vec4VfpuNaN: + regs_.Map(inst); + if (cpu_info.LOONGARCH_LSX) { + VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.src1), regs_.V(inst.src1)); + LI(SCRATCH1, 0x7F800001); + VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1); + VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.src1), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + } else { + QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1); + for (int i = 0; i < 4; ++i) { + FCMP_COND_S(FCC0, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i), LoongArch64Fcond::CUN); + FSEL(regs_.F(inst.dest + i), regs_.F(inst.src1 + i), SCRATCHF1, FCC0); + } + } + break; + + case IROp::Vec4VfpuProductNaN: + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + if (cpu_info.LOONGARCH_LSX) { + // Put the product's sign in the NaN lanes, then OR in 0x7F800001 there. + VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.dest), regs_.V(inst.dest)); + VXOR_V(EncodeRegToV(SCRATCHF2), regs_.V(inst.src1), regs_.V(inst.src2)); + VSRLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31); + VSLLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31); + VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + LI(SCRATCH1, 0x7F800001); + VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1); + VAND_V(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + VOR_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2)); + } else { + for (int i = 0; i < 4; ++i) { + FCMP_COND_S(FCC0, regs_.F(inst.dest + i), regs_.F(inst.dest + i), LoongArch64Fcond::CUN); + MOVCF2GR(SCRATCH1, FCC0); + FixupBranch notNaN = BEQZ(SCRATCH1); + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i)); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2 + i)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLI_W(SCRATCH1, SCRATCH1, 31); + SLLI_W(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + MOVGR2FR_W(regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + } + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/RiscV/RiscVCompFPU.cpp b/Core/MIPS/RiscV/RiscVCompFPU.cpp index e19152dbd8..f90dbddf1b 100644 --- a/Core/MIPS/RiscV/RiscVCompFPU.cpp +++ b/Core/MIPS/RiscV/RiscVCompFPU.cpp @@ -69,6 +69,38 @@ void RiscVJitBackend::CompIR_FArith(IRInst inst) { FNEG(32, regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + FEQ(32, SCRATCH1, regs_.F(inst.src1), regs_.F(inst.src1)); + if (inst.dest != inst.src1) + FMV(32, regs_.F(inst.dest), regs_.F(inst.src1)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + LI(SCRATCH1, 0x7F800001); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + FEQ(32, SCRATCH1, regs_.F(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1)); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLIW(SCRATCH1, SCRATCH1, 31); + SLLIW(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/RiscV/RiscVCompVec.cpp b/Core/MIPS/RiscV/RiscVCompVec.cpp index 10ac530cc1..0dfc8114ac 100644 --- a/Core/MIPS/RiscV/RiscVCompVec.cpp +++ b/Core/MIPS/RiscV/RiscVCompVec.cpp @@ -322,6 +322,38 @@ void RiscVJitBackend::CompIR_VecArith(IRInst inst) { break; } + case IROp::Vec4VfpuNaN: + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + FEQ(32, SCRATCH1, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i)); + if (inst.dest != inst.src1) + FMV(32, regs_.F(inst.dest + i), regs_.F(inst.src1 + i)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + LI(SCRATCH1, 0x7F800001); + FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + break; + + case IROp::Vec4VfpuProductNaN: + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + FEQ(32, SCRATCH1, regs_.F(inst.dest + i), regs_.F(inst.dest + i)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i)); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2 + i)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLIW(SCRATCH1, SCRATCH1, 31); + SLLIW(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRCompFPU.cpp b/Core/MIPS/x86/X64IRCompFPU.cpp index 905b8fa1ef..02040597b0 100644 --- a/Core/MIPS/x86/X64IRCompFPU.cpp +++ b/Core/MIPS/x86/X64IRCompFPU.cpp @@ -46,6 +46,7 @@ void X64JitBackend::EmitFPUConstants() { EmitConst4x32(&constants.positiveZeroes, 0x00000000); EmitConst4x32(&constants.positiveInfinity, 0x7F800000); EmitConst4x32(&constants.qNAN, 0x7FC00000); + EmitConst4x32(&constants.vfpuNaN, 0x7F800001); EmitConst4x32(&constants.positiveOnes, 0x3F800000); EmitConst4x32(&constants.negativeOnes, 0xBF800000); EmitConst4x32(&constants.maxIntBelowAsFloat, 0x4EFFFFFF); @@ -204,6 +205,33 @@ void X64JitBackend::CompIR_FArith(IRInst inst) { } break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + if (inst.dest != inst.src1) + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = J_CC(CC_NP); + MOVSS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = J_CC(CC_NP); + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + XORPS(regs_.FX(inst.dest), regs_.F(inst.src2)); + ANDPS(regs_.FX(inst.dest), M(constants.signBitAll)); // rip accessible + ORPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; @@ -872,7 +900,7 @@ void X64JitBackend::CompIR_FSat(IRInst inst) { tempReg = regs_.MapWithFPRTemp(inst); // The second argument's NAN is taken if either is NAN, so put known first. - MOVSS(tempReg, M(constants.positiveOnes)); + MOVSS(tempReg, M(constants.positiveOnes)); // rip accessible MINSS(tempReg, regs_.F(inst.src1)); // Known first again, so a NAN stays, bits and all. This keeps -0.0 too, since a tie also diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index 9401fda4b1..b18eb87db8 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -175,6 +175,51 @@ void X64JitBackend::CompIR_VecArith(IRInst inst) { } break; + case IROp::Vec4VfpuNaN: + { + X64Reg maskReg = regs_.MapWithFPRTemp(inst); + if (inst.dest != inst.src1) + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + MOVAPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(maskReg, regs_.F(inst.dest)); + MOVMSKPS(SCRATCH1, R(maskReg)); + TEST(32, R(SCRATCH1), R(SCRATCH1)); + FixupBranch noNaN = J_CC(CC_Z); + // Rare: keep the other lanes, replace the NaNs. + ANDNPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(regs_.FX(inst.dest), regs_.F(inst.dest)); + ANDPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + ORPS(regs_.FX(inst.dest), R(maskReg)); + SetJumpTarget(noNaN); + break; + } + + case IROp::Vec4VfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.SpillLockFPR(inst.dest, inst.src1, inst.src2); + X64Reg maskReg = regs_.GetAndLockTempFPR(); + X64Reg nanReg = regs_.GetAndLockTempFPR(); + regs_.Map(inst); + MOVAPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(maskReg, regs_.F(inst.dest)); + MOVMSKPS(SCRATCH1, R(maskReg)); + TEST(32, R(SCRATCH1), R(SCRATCH1)); + FixupBranch noNaN = J_CC(CC_Z); + // Rare: replace the NaNs, with the sign of the operands' product. + MOVAPS(nanReg, regs_.F(inst.src1)); + XORPS(nanReg, regs_.F(inst.src2)); + ANDPS(nanReg, M(constants.signBitAll)); // rip accessible + ORPS(nanReg, M(constants.vfpuNaN)); // rip accessible + ANDPS(nanReg, R(maskReg)); + ANDNPS(maskReg, regs_.F(inst.dest)); + ORPS(nanReg, R(maskReg)); + MOVAPS(regs_.FX(inst.dest), R(nanReg)); + SetJumpTarget(noNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRJit.h b/Core/MIPS/x86/X64IRJit.h index f9a771286c..de2b5ab8d1 100644 --- a/Core/MIPS/x86/X64IRJit.h +++ b/Core/MIPS/x86/X64IRJit.h @@ -161,6 +161,7 @@ private: const void *positiveOnes; const void *negativeOnes; const void *qNAN; + const void *vfpuNaN; const void *maxIntBelowAsFloat; const float *mulTableVi2f; const float *mulTableVf2i;