diff --git a/Core/MIPS/ARM64/Arm64IRCompFPU.cpp b/Core/MIPS/ARM64/Arm64IRCompFPU.cpp index 08dd479256..541de78bc0 100644 --- a/Core/MIPS/ARM64/Arm64IRCompFPU.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompFPU.cpp @@ -76,6 +76,36 @@ void Arm64JitBackend::CompIR_FArith(IRInst inst) { fp_.FNEG(regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1)); + if (inst.dest != inst.src1) + fp_.FMOV(regs_.F(inst.dest), regs_.F(inst.src1)); + FixupBranch notNaN = B(CC_VC); + MOVI2R(SCRATCH1, 0x7F800001); + fp_.FMOV(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + fp_.FCMP(regs_.F(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = B(CC_VC); + fp_.FMOV(SCRATCH1, regs_.F(inst.src1)); + fp_.FMOV(SCRATCH2, regs_.F(inst.src2)); + EOR(SCRATCH1, SCRATCH1, SCRATCH2); + ANDI2R(SCRATCH1, SCRATCH1, 0x80000000, SCRATCH2); + ORRI2R(SCRATCH1, SCRATCH1, 0x7F800001, SCRATCH2); + fp_.FMOV(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; @@ -455,22 +485,20 @@ void Arm64JitBackend::CompIR_FSat(IRInst inst) { switch (inst.op) { case IROp::FSat0_1: - regs_.Map(inst); - fp_.MOVI2F(SCRATCHF1, 1.0f); - // Note that FMAX takes the larger of the two zeros, which is what we want. - fp_.MOVI2F(SCRATCHF2, 0.0f); - - fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1); - fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2); - break; - case IROp::FSatMinus1_1: regs_.Map(inst); fp_.MOVI2F(SCRATCHF1, 1.0f); - fp_.FNEG(SCRATCHF2, SCRATCHF1); + // Note that FMAX takes the larger of the two zeros, which is what we want. + if (inst.op == IROp::FSat0_1) + fp_.MOVI2F(SCRATCHF2, 0.0f); + else + fp_.FNEG(SCRATCHF2, SCRATCHF1); - fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1); - fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2); + // A NaN passes through with its bits, which FMIN/FMAX would quiet (cpu/vfpu/nan_gen). + fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1)); + fp_.FMIN(SCRATCHF1, regs_.F(inst.src1), SCRATCHF1); + fp_.FMAX(SCRATCHF1, SCRATCHF1, SCRATCHF2); + fp_.FCSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, CC_VS); break; default: diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 8426232d29..8a7ae02720 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -140,6 +140,38 @@ void Arm64JitBackend::CompIR_VecArith(IRInst inst) { fp_.SCVTF(32, regs_.FQ(inst.dest), regs_.FQ(inst.src1), inst.src2 & 0x1F); break; + case IROp::Vec4VfpuNaN: + case IROp::Vec4VfpuProductNaN: + { + // The operands of the product are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.op == IROp::Vec4VfpuNaN || (inst.dest != inst.src1 && inst.dest != inst.src2)); + regs_.Map(inst); + if (inst.op == IROp::Vec4VfpuNaN && inst.dest != inst.src1) + fp_.MOV(regs_.FQ(inst.dest), regs_.FQ(inst.src1)); + // FMAXV gives a NaN if any lane is one. + fp_.FMAXV(SCRATCHF1, regs_.FQ(inst.dest)); + fp_.FCMP(SCRATCHF1, SCRATCHF1); + FixupBranch noNaN = B(CC_VC); + + // Rare: replace the NaN lanes with 0x7F800001, signed like the product for that op. + ARM64Reg orderedMask = EncodeRegToQuad(SCRATCHF1); + ARM64Reg nanReg = EncodeRegToQuad(SCRATCHF2); + fp_.FCMEQ(32, orderedMask, regs_.FQ(inst.dest), regs_.FQ(inst.dest)); + if (inst.op == IROp::Vec4VfpuProductNaN) { + fp_.EOR(nanReg, regs_.FQ(inst.src1), regs_.FQ(inst.src2)); + fp_.USHR(32, nanReg, nanReg, 31); + fp_.SHL(32, nanReg, nanReg, 31); + fp_.ORR(32, nanReg, 0x7F, 24); + } else { + fp_.MOVI(32, nanReg, 0x7F, 24); + } + fp_.ORR(32, nanReg, 0x80, 16); + fp_.ORR(32, nanReg, 0x01, 0); + fp_.BIF(regs_.FQ(inst.dest), nanReg, orderedMask); + SetJumpTarget(noNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index caaade1540..326687ba2f 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -1193,11 +1193,25 @@ namespace MIPSComp { break; } + // Sums and products make the VFPU's own NaN. vmin and vmax pass a NaN through, and vslt and + // vsge don't make one. + const bool sum = type == VecDo3Op::VADD || type == VecDo3Op::VSUB; + const bool product = type == VecDo3Op::VMUL || type == VecDo3Op::VDIV; if (IsVec4(sz, dregs) && IsVec4(sz, sregs) && IsVec4(sz, tregs)) { - if (opFunc != IROp::Nop) { - ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); - } else { + if (opFunc == IROp::Nop) DISABLE; + if (!product) { + ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); + if (sum) + ir.Write(IROp::Vec4VfpuNaN, dregs[0], dregs[0]); + } else if (dregs[0] != sregs[0] && dregs[0] != tregs[0]) { + ir.Write(opFunc, dregs[0], sregs[0], tregs[0]); + ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], tregs[0]); + } else { + // The NaN's sign comes from the operands, so they have to survive the op. + ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]); + ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0); } ApplyPrefixD(dregs, sz, _VD); return; @@ -1207,6 +1221,10 @@ namespace MIPSComp { if (opFunc == IROp::Nop) DISABLE; ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]); + if (product) + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]); + else if (sum) + ir.Write(IROp::Vec4VfpuNaN, IRVTEMP_0, IRVTEMP_0); ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); ApplyPrefixD(dregs, sz, _VD); return; @@ -1220,17 +1238,22 @@ namespace MIPSComp { for (int i = 0; i < n; ++i) { switch (type) { + // tempregs[i] is never one of the operands, which the product NaN needs. case VecDo3Op::VADD: // d[i] = s[i] + t[i]; break; //vadd ir.Write(IROp::FAdd, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]); break; case VecDo3Op::VSUB: // d[i] = s[i] - t[i]; break; //vsub ir.Write(IROp::FSub, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]); break; case VecDo3Op::VDIV: // d[i] = s[i] / t[i]; break; //vdiv ir.Write(IROp::FDiv, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]); break; case VecDo3Op::VMUL: // d[i] = s[i] * t[i]; break; //vmul ir.Write(IROp::FMul, tempregs[i], sregs[i], tregs[i]); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]); break; case VecDo3Op::VMIN: // vmin ir.Write(IROp::FMin, tempregs[i], sregs[i], tregs[i]); @@ -1843,20 +1866,35 @@ namespace MIPSComp { } if (!overlap || (vs == vd && IsOverlapSafe(treg, n, dregs))) { + // The NaN's sign needs t in every lane (the T prefix temp is free, see above.) if (IsVec4(sz, sregs) && IsVec4(sz, dregs)) { - ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg); + int tlane = treg & 3; + ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane)); + if (dregs[0] != sregs[0]) { + ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], IRVTEMP_PFX_T); + } else { + ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T); + ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0); + } ApplyPrefixD(dregs, sz, vd); return; } else if (IsVec3of4(sz, sregs) && IsVec3of4(sz, dregs) && opts.preferVec4) { + int tlane = treg & 3; + ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane)); ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T); ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); ApplyPrefixD(dregs, sz, vd); return; } } + // tempregs[i] is never s[i] or t, which the product NaN needs. for (int i = 0; i < n; i++) { ir.Write(IROp::FMul, tempregs[i], sregs[i], treg); + ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], treg); } for (int i = 0; i < n; i++) { @@ -2017,6 +2055,7 @@ namespace MIPSComp { ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_S, sregs[0], VFPU_SWIZZLE(1, 2, 0, 3)); ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, tregs[0], VFPU_SWIZZLE(2, 0, 1, 3)); ir.Write(IROp::Vec4Mul, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T); + ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T); // Now just retain w and blend in our values. ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7); } else { @@ -2030,8 +2069,11 @@ namespace MIPSComp { } ir.Write(IROp::FMul, tempregs[0], sregs[1], tregs[2]); + ir.Write(IROp::FVfpuProductNaN, tempregs[0], sregs[1], tregs[2]); ir.Write(IROp::FMul, tempregs[1], sregs[2], tregs[0]); + ir.Write(IROp::FVfpuProductNaN, tempregs[1], sregs[2], tregs[0]); ir.Write(IROp::FMul, tempregs[2], sregs[0], tregs[1]); + ir.Write(IROp::FVfpuProductNaN, tempregs[2], sregs[0], tregs[1]); for (int i = 0; i < n; i++) { if (tempregs[i] != dregs[i]) diff --git a/Core/MIPS/IR/IRInst.cpp b/Core/MIPS/IR/IRInst.cpp index 8cd54e8985..18d60d56af 100644 --- a/Core/MIPS/IR/IRInst.cpp +++ b/Core/MIPS/IR/IRInst.cpp @@ -136,6 +136,8 @@ static const IRMeta irMeta[] = { { IROp::FCmp, "FCmp", "mFF" }, { IROp::FSat0_1, "FSat(0 - 1)", "FF" }, { IROp::FSatMinus1_1, "FSat(-1 - 1)", "FF" }, + { IROp::FVfpuNaN, "FVfpuNaN", "FF" }, + { IROp::FVfpuProductNaN, "FVfpuProductNaN", "FFF", IRFLAG_SRC3DST }, { IROp::FMovFromGPR, "FMovFromGPR", "FG" }, { IROp::FMovToGPR, "FMovToGPR", "GF" }, { IROp::OptFMovToGPRShr8, "OptFMovToGPRShr8", "GF" }, @@ -175,6 +177,8 @@ static const IRMeta irMeta[] = { { IROp::Vec4Slt, "Vec4Slt", "VVV" }, { IROp::Vec4Sge, "Vec4Sge", "VVV" }, { IROp::Vec4CvtScaledSW, "Vec4CvtScaledSW", "VVI" }, + { IROp::Vec4VfpuNaN, "Vec4VfpuNaN", "VV" }, + { IROp::Vec4VfpuProductNaN, "Vec4VfpuProductNaN", "VVV", IRFLAG_SRC3DST }, // Pack/Unpack { IROp::Vec2Unpack16To31, "Vec2Unpack16To31", "2F" }, // Note that the result is shifted down by 1, hence 31 diff --git a/Core/MIPS/IR/IRInst.h b/Core/MIPS/IR/IRInst.h index ab8b5c38d3..dc41416228 100644 --- a/Core/MIPS/IR/IRInst.h +++ b/Core/MIPS/IR/IRInst.h @@ -163,6 +163,10 @@ enum class IROp : uint8_t { FSat0_1, FSatMinus1_1, + // The VFPU's NaNs (see VfpuSumNaN in InterpreterVFPU.cpp): src1 or 0x7F800001 if it's a NaN, and + // dest (read too) or 0x7F800001 with the sign of src1 ^ src2 if dest is a NaN. + FVfpuNaN, + FVfpuProductNaN, FpCondFromReg, FpCondToReg, @@ -208,6 +212,8 @@ enum class IROp : uint8_t { Vec4Sge, // vi2f: each lane's int to float, divided by 2^src2. Vec4CvtScaledSW, + Vec4VfpuNaN, + Vec4VfpuProductNaN, // vx2i Vec2Unpack16To31, // Note that the result is shifted down by 1, hence 31 diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp index b399fe4a2a..14be112dda 100644 --- a/Core/MIPS/IR/IRInterpreter.cpp +++ b/Core/MIPS/IR/IRInterpreter.cpp @@ -50,6 +50,10 @@ alignas(16) static const uint32_t noSignMask[4] = { 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, }; +alignas(16) static const uint32_t vfpuNaN[4] = { + 0x7F800001, 0x7F800001, 0x7F800001, 0x7F800001, +}; + alignas(16) static const uint32_t lowBytesMask[4] = { 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, }; @@ -245,6 +249,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { dispatch[(int)IROp::Vec4Scale] = &&L_Vec4Scale; dispatch[(int)IROp::Vec4Neg] = &&L_Vec4Neg; dispatch[(int)IROp::Vec4Abs] = &&L_Vec4Abs; + dispatch[(int)IROp::Vec4VfpuNaN] = &&L_Vec4VfpuNaN; + dispatch[(int)IROp::Vec4VfpuProductNaN] = &&L_Vec4VfpuProductNaN; dispatch[(int)IROp::Vec2Unpack16To31] = &&L_Vec2Unpack16To31; dispatch[(int)IROp::Vec2Unpack16To32] = &&L_Vec2Unpack16To32; dispatch[(int)IROp::Vec4Unpack8To32] = &&L_Vec4Unpack8To32; @@ -316,6 +322,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { dispatch[(int)IROp::FNeg] = &&L_FNeg; dispatch[(int)IROp::FSat0_1] = &&L_FSat0_1; dispatch[(int)IROp::FSatMinus1_1] = &&L_FSatMinus1_1; + dispatch[(int)IROp::FVfpuNaN] = &&L_FVfpuNaN; + dispatch[(int)IROp::FVfpuProductNaN] = &&L_FVfpuProductNaN; dispatch[(int)IROp::FSign] = &&L_FSign; dispatch[(int)IROp::FpCondFromReg] = &&L_FpCondFromReg; dispatch[(int)IROp::FpCondToReg] = &&L_FpCondToReg; @@ -759,6 +767,44 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { IR_NEXT; } + IR_CASE(Vec4VfpuNaN) + { +#if PPSSPP_ARCH(SSE2) + __m128 v = _mm_load_ps(&mips->f[inst->src1]); + __m128 isNaN = _mm_cmpunord_ps(v, v); + _mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, _mm_load_ps((const float *)vfpuNaN)))); +#elif PPSSPP_ARCH(ARM_NEON) + float32x4_t v = vld1q_f32(&mips->f[inst->src1]); + vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, vld1q_f32((const float *)vfpuNaN))); +#else + for (int i = 0; i < 4; i++) + mips->fi[inst->dest + i] = my_isnan(mips->f[inst->src1 + i]) ? 0x7F800001 : mips->fi[inst->src1 + i]; +#endif + IR_NEXT; + } + + IR_CASE(Vec4VfpuProductNaN) + { +#if PPSSPP_ARCH(SSE2) + __m128 v = _mm_load_ps(&mips->f[inst->dest]); + __m128 isNaN = _mm_cmpunord_ps(v, v); + __m128 sign = _mm_and_ps(_mm_xor_ps(_mm_load_ps(&mips->f[inst->src1]), _mm_load_ps(&mips->f[inst->src2])), _mm_load_ps((const float *)signBits)); + __m128 nan = _mm_or_ps(sign, _mm_load_ps((const float *)vfpuNaN)); + _mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, nan))); +#elif PPSSPP_ARCH(ARM_NEON) + float32x4_t v = vld1q_f32(&mips->f[inst->dest]); + uint32x4_t sign = vandq_u32(veorq_u32(vld1q_u32(&mips->fi[inst->src1]), vld1q_u32(&mips->fi[inst->src2])), vld1q_u32(signBits)); + float32x4_t nan = vreinterpretq_f32_u32(vorrq_u32(sign, vld1q_u32(vfpuNaN))); + vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, nan)); +#else + for (int i = 0; i < 4; i++) { + if (my_isnan(mips->f[inst->dest + i])) + mips->fi[inst->dest + i] = 0x7F800001 | ((mips->fi[inst->src1 + i] ^ mips->fi[inst->src2 + i]) & 0x80000000); + } +#endif + IR_NEXT; + } + IR_CASE(Vec2Unpack16To31) { const u32 dest = inst->dest; @@ -1245,6 +1291,13 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) { IR_CASE(FSatMinus1_1) mips->f[inst->dest] = vfpu_clamp(mips->f[inst->src1], -1.0f, 1.0f); IR_NEXT; + IR_CASE(FVfpuNaN) + mips->fi[inst->dest] = my_isnan(mips->f[inst->src1]) ? 0x7F800001 : mips->fi[inst->src1]; + IR_NEXT; + IR_CASE(FVfpuProductNaN) + if (my_isnan(mips->f[inst->dest])) + mips->fi[inst->dest] = 0x7F800001 | ((mips->fi[inst->src1] ^ mips->fi[inst->src2]) & 0x80000000); + IR_NEXT; IR_CASE(FSign) { diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index b24d5a5b84..373e2dc3a7 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -348,6 +348,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { case IROp::FDiv: case IROp::FSqrt: case IROp::FNeg: + case IROp::FVfpuNaN: + case IROp::FVfpuProductNaN: CompIR_FArith(inst); break; @@ -426,6 +428,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { case IROp::Vec4Slt: case IROp::Vec4Sge: case IROp::Vec4CvtScaledSW: + case IROp::Vec4VfpuNaN: + case IROp::Vec4VfpuProductNaN: CompIR_VecArith(inst); break; diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index bc8c5a19b7..92824ca9df 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -1031,6 +1031,14 @@ bool IRMutatesDestGPR(const IRInstMeta &inst, int reg) { return (inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'G' && inst.src3 == reg; } +// Whether inst reads the FPR it writes (reg counted from 32, like the checks in PurgeTemps). +static bool IRMutatesDestFPR(const IRInstMeta &inst, int reg) { + if ((inst.m.flags & IRFLAG_SRC3DST) == 0) + return false; + int len = inst.m.types[0] == 'F' ? 1 : (inst.m.types[0] == 'V' ? 4 : 0); + return reg >= inst.src3 + 32 && reg < inst.src3 + 32 + len; +} + bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { CONDITIONAL_DISABLE; std::vector insts; @@ -1192,7 +1200,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { if (numDest > 1 && check.srcReg >= destRegs[0] + 32 && check.srcReg < destRegs[0] + 32 + numDest) cannotReplace = true; // Nor where it's read as src3 that's also the destination, which renaming would move. - if ((inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'F' && inst.src3 + 32 == check.reg) + if (IRMutatesDestFPR(inst, check.reg)) cannotReplace = true; if (!cannotReplace && check.srcReg >= 32 && lastWrittenTo[check.srcReg] < check.index) { // This is probably not worth doing unless we can get rid of a temp. @@ -1219,7 +1227,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) { // Let's not bother. check.reg = 0; } - } else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch) { + } else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch && !IRMutatesDestFPR(insts[check.index], check.reg)) { // A swap could be profitable if this is a temp, and maybe in other cases. // These can happen a lot from mask regs, etc. // But make sure no other changes happened between. diff --git a/Core/MIPS/InterpreterVFPU.cpp b/Core/MIPS/InterpreterVFPU.cpp index fa091cb114..195d14cb7c 100644 --- a/Core/MIPS/InterpreterVFPU.cpp +++ b/Core/MIPS/InterpreterVFPU.cpp @@ -75,7 +75,7 @@ #define M_SQRT1_2 0.707106781186547524401f #endif -// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). +// The VFPU's own handling of denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). static const bool USE_VFPU_RESULT_FIXUP = false; union FloatBits { @@ -84,6 +84,26 @@ union FloatBits { int i[4]; }; +// The VFPU never keeps a NaN's payload: every NaN it produces is 0x7F800001, negative for a product +// or quotient of operands of different signs (cpu/vfpu/nan_gen). Host NaNs differ by host. +static inline float VfpuSumNaN(float r) { + if (!my_isnan(r)) + return r; + FP32 nan; + nan.u = 0x7F800001; + return nan.f; +} + +static inline float VfpuProductNaN(float r, float a, float b) { + if (!my_isnan(r)) + return r; + FP32 fa, fb, nan; + fa.f = a; + fb.f = b; + nan.u = 0x7F800001 | ((fa.u ^ fb.u) & 0x80000000); + return nan.f; +} + // Preserves NaN in first param, takes sign of equal second param. // Technically, std::max may do this but it's undefined. inline float nanmax(float f, float cst) @@ -1436,10 +1456,9 @@ namespace MIPSInt u32 tprefixAdd = VFPU_SWIZZLE(2, 0, 1, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), sz); - d[0] = s[0] * t[0]; - d[1] = s[1] * t[1]; - d[2] = s[2] * t[2]; - d[3] = s[3] * t[3]; + for (int i = 0; i < 4; i++) { + d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); + } ApplyPrefixD(mips, d, sz); WriteVector(mips, d, sz, vd); PC += 4; @@ -1544,7 +1563,7 @@ namespace MIPSInt int n = GetNumVectorElements(sz); for (int i = 0; i < n; i++) { - d[i] = s[i] * t[i]; + d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); } ApplyPrefixD(mips, d, sz); WriteVector(mips, d, sz, vd); @@ -2105,18 +2124,14 @@ namespace MIPSInt for (int i = 0; i < (int)n; i++) { switch (optype) { - case 0: d.f[i] = s[i] + t[i]; break; //vadd - case 1: d.f[i] = s[i] - t[i]; break; //vsub - case 7: d.f[i] = s[i] / t[i]; break; //vdiv - case 8: d.f[i] = s[i] * t[i]; break; //vmul + case 0: d.f[i] = VfpuSumNaN(s[i] + t[i]); break; //vadd + case 1: d.f[i] = VfpuSumNaN(s[i] - t[i]); break; //vsub + case 7: d.f[i] = VfpuProductNaN(s[i] / t[i], s[i], t[i]); break; //vdiv + case 8: d.f[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); break; //vmul } - if (USE_VFPU_RESULT_FIXUP) { - if (my_isnan(d.f[i])) { - d.u[i] = (d.u[i] & 0xff800001) | 1; - } else if ((d.u[i] & 0x7F800000) == 0) { - d.u[i] &= 0xFF800000; - } + if (USE_VFPU_RESULT_FIXUP && (d.u[i] & 0x7F800000) == 0) { + d.u[i] &= 0xFF800000; } } diff --git a/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp b/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp index e1335944c9..f4f20f865b 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompFPU.cpp @@ -70,6 +70,33 @@ void LoongArch64JitBackend::CompIR_FArith(IRInst inst) { FNEG_S(regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + regs_.Map(inst); + FCMP_COND_S(FCC0, regs_.F(inst.src1), regs_.F(inst.src1), LoongArch64Fcond::CUN); + QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1); + FSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, FCC0); + break; + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + FCMP_COND_S(FCC0, regs_.F(inst.dest), regs_.F(inst.dest), LoongArch64Fcond::CUN); + MOVCF2GR(SCRATCH1, FCC0); + FixupBranch notNaN = BEQZ(SCRATCH1); + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1)); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLI_W(SCRATCH1, SCRATCH1, 31); + SLLI_W(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + MOVGR2FR_W(regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp index 6a32b6239b..2a5bc8a99f 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp @@ -431,6 +431,55 @@ void LoongArch64JitBackend::CompIR_VecArith(IRInst inst) { } break; + case IROp::Vec4VfpuNaN: + regs_.Map(inst); + if (cpu_info.LOONGARCH_LSX) { + VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.src1), regs_.V(inst.src1)); + LI(SCRATCH1, 0x7F800001); + VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1); + VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.src1), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + } else { + QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1); + for (int i = 0; i < 4; ++i) { + FCMP_COND_S(FCC0, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i), LoongArch64Fcond::CUN); + FSEL(regs_.F(inst.dest + i), regs_.F(inst.src1 + i), SCRATCHF1, FCC0); + } + } + break; + + case IROp::Vec4VfpuProductNaN: + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + if (cpu_info.LOONGARCH_LSX) { + // Put the product's sign in the NaN lanes, then OR in 0x7F800001 there. + VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.dest), regs_.V(inst.dest)); + VXOR_V(EncodeRegToV(SCRATCHF2), regs_.V(inst.src1), regs_.V(inst.src2)); + VSRLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31); + VSLLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31); + VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + LI(SCRATCH1, 0x7F800001); + VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1); + VAND_V(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1)); + VOR_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2)); + } else { + for (int i = 0; i < 4; ++i) { + FCMP_COND_S(FCC0, regs_.F(inst.dest + i), regs_.F(inst.dest + i), LoongArch64Fcond::CUN); + MOVCF2GR(SCRATCH1, FCC0); + FixupBranch notNaN = BEQZ(SCRATCH1); + MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i)); + MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2 + i)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLI_W(SCRATCH1, SCRATCH1, 31); + SLLI_W(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + MOVGR2FR_W(regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + } + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/RiscV/RiscVCompFPU.cpp b/Core/MIPS/RiscV/RiscVCompFPU.cpp index e19152dbd8..f90dbddf1b 100644 --- a/Core/MIPS/RiscV/RiscVCompFPU.cpp +++ b/Core/MIPS/RiscV/RiscVCompFPU.cpp @@ -69,6 +69,38 @@ void RiscVJitBackend::CompIR_FArith(IRInst inst) { FNEG(32, regs_.F(inst.dest), regs_.F(inst.src1)); break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + FEQ(32, SCRATCH1, regs_.F(inst.src1), regs_.F(inst.src1)); + if (inst.dest != inst.src1) + FMV(32, regs_.F(inst.dest), regs_.F(inst.src1)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + LI(SCRATCH1, 0x7F800001); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + FEQ(32, SCRATCH1, regs_.F(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1)); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLIW(SCRATCH1, SCRATCH1, 31); + SLLIW(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1); + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/RiscV/RiscVCompVec.cpp b/Core/MIPS/RiscV/RiscVCompVec.cpp index 10ac530cc1..0dfc8114ac 100644 --- a/Core/MIPS/RiscV/RiscVCompVec.cpp +++ b/Core/MIPS/RiscV/RiscVCompVec.cpp @@ -322,6 +322,38 @@ void RiscVJitBackend::CompIR_VecArith(IRInst inst) { break; } + case IROp::Vec4VfpuNaN: + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + FEQ(32, SCRATCH1, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i)); + if (inst.dest != inst.src1) + FMV(32, regs_.F(inst.dest + i), regs_.F(inst.src1 + i)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + LI(SCRATCH1, 0x7F800001); + FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + break; + + case IROp::Vec4VfpuProductNaN: + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + for (int i = 0; i < 4; ++i) { + FEQ(32, SCRATCH1, regs_.F(inst.dest + i), regs_.F(inst.dest + i)); + FixupBranch notNaN = BNE(SCRATCH1, R_ZERO); + FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i)); + FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2 + i)); + XOR(SCRATCH1, SCRATCH1, SCRATCH2); + SRLIW(SCRATCH1, SCRATCH1, 31); + SLLIW(SCRATCH1, SCRATCH1, 31); + LI(SCRATCH2, 0x7F800001); + OR(SCRATCH1, SCRATCH1, SCRATCH2); + FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1); + SetJumpTarget(notNaN); + } + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRCompFPU.cpp b/Core/MIPS/x86/X64IRCompFPU.cpp index 905b8fa1ef..02040597b0 100644 --- a/Core/MIPS/x86/X64IRCompFPU.cpp +++ b/Core/MIPS/x86/X64IRCompFPU.cpp @@ -46,6 +46,7 @@ void X64JitBackend::EmitFPUConstants() { EmitConst4x32(&constants.positiveZeroes, 0x00000000); EmitConst4x32(&constants.positiveInfinity, 0x7F800000); EmitConst4x32(&constants.qNAN, 0x7FC00000); + EmitConst4x32(&constants.vfpuNaN, 0x7F800001); EmitConst4x32(&constants.positiveOnes, 0x3F800000); EmitConst4x32(&constants.negativeOnes, 0xBF800000); EmitConst4x32(&constants.maxIntBelowAsFloat, 0x4EFFFFFF); @@ -204,6 +205,33 @@ void X64JitBackend::CompIR_FArith(IRInst inst) { } break; + case IROp::FVfpuNaN: + { + regs_.Map(inst); + if (inst.dest != inst.src1) + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = J_CC(CC_NP); + MOVSS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + SetJumpTarget(notNaN); + break; + } + + case IROp::FVfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.Map(inst); + UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest)); + FixupBranch notNaN = J_CC(CC_NP); + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + XORPS(regs_.FX(inst.dest), regs_.F(inst.src2)); + ANDPS(regs_.FX(inst.dest), M(constants.signBitAll)); // rip accessible + ORPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + SetJumpTarget(notNaN); + break; + } + default: INVALIDOP; break; @@ -872,7 +900,7 @@ void X64JitBackend::CompIR_FSat(IRInst inst) { tempReg = regs_.MapWithFPRTemp(inst); // The second argument's NAN is taken if either is NAN, so put known first. - MOVSS(tempReg, M(constants.positiveOnes)); + MOVSS(tempReg, M(constants.positiveOnes)); // rip accessible MINSS(tempReg, regs_.F(inst.src1)); // Known first again, so a NAN stays, bits and all. This keeps -0.0 too, since a tie also diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index 9401fda4b1..b18eb87db8 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -175,6 +175,51 @@ void X64JitBackend::CompIR_VecArith(IRInst inst) { } break; + case IROp::Vec4VfpuNaN: + { + X64Reg maskReg = regs_.MapWithFPRTemp(inst); + if (inst.dest != inst.src1) + MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1)); + MOVAPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(maskReg, regs_.F(inst.dest)); + MOVMSKPS(SCRATCH1, R(maskReg)); + TEST(32, R(SCRATCH1), R(SCRATCH1)); + FixupBranch noNaN = J_CC(CC_Z); + // Rare: keep the other lanes, replace the NaNs. + ANDNPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(regs_.FX(inst.dest), regs_.F(inst.dest)); + ANDPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible + ORPS(regs_.FX(inst.dest), R(maskReg)); + SetJumpTarget(noNaN); + break; + } + + case IROp::Vec4VfpuProductNaN: + { + // The operands are never the dest, see IRFrontend::Comp_VecDo3. + _dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2); + regs_.SpillLockFPR(inst.dest, inst.src1, inst.src2); + X64Reg maskReg = regs_.GetAndLockTempFPR(); + X64Reg nanReg = regs_.GetAndLockTempFPR(); + regs_.Map(inst); + MOVAPS(maskReg, regs_.F(inst.dest)); + CMPUNORDPS(maskReg, regs_.F(inst.dest)); + MOVMSKPS(SCRATCH1, R(maskReg)); + TEST(32, R(SCRATCH1), R(SCRATCH1)); + FixupBranch noNaN = J_CC(CC_Z); + // Rare: replace the NaNs, with the sign of the operands' product. + MOVAPS(nanReg, regs_.F(inst.src1)); + XORPS(nanReg, regs_.F(inst.src2)); + ANDPS(nanReg, M(constants.signBitAll)); // rip accessible + ORPS(nanReg, M(constants.vfpuNaN)); // rip accessible + ANDPS(nanReg, R(maskReg)); + ANDNPS(maskReg, regs_.F(inst.dest)); + ORPS(nanReg, R(maskReg)); + MOVAPS(regs_.FX(inst.dest), R(nanReg)); + SetJumpTarget(noNaN); + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRJit.h b/Core/MIPS/x86/X64IRJit.h index f9a771286c..de2b5ab8d1 100644 --- a/Core/MIPS/x86/X64IRJit.h +++ b/Core/MIPS/x86/X64IRJit.h @@ -161,6 +161,7 @@ private: const void *positiveOnes; const void *negativeOnes; const void *qNAN; + const void *vfpuNaN; const void *maxIntBelowAsFloat; const float *mulTableVi2f; const float *mulTableVf2i;