VFPU: Produce the PSP's NaNs in the single-lane ops, on every core

The VFPU never keeps a NaN's payload (cpu/vfpu/nan_gen): vadd and vsub
produce 0x7F800001, and vmul, vdiv, vscl and vcrs the same with the
sign of s ^ t. The cores returned whatever NaN the host made, which
differs between x86 and ARM64. Now all four cores on both hosts match
the PSP for these ops. The horizontal ops (vdot and friends) are left
for when they move to the accurate dot product.

The IR gets FVfpuNaN and FVfpuProductNaN, scalar and Vec4, run after
the arithmetic. The native backends check for a NaN and branch past
the fixup, which only runs when there is one. PurgeTemps no longer
renames the dest of an FPR op that also reads it, which the product
ops are the first to do.

vsat0 and vsat1 also keep a NaN's bits on ARM64 now, where FMIN/FMAX
quieted a signaling NaN.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Henrik RydgårdandClaude Opus 5.5 committed 2026-10-31 21:28:07 +01:00
1 parent ac0530e7ca
commit 92e3957264
16 files changed
+441 -35

No files matched your search

+40 -12
View File
@@ -76,6 +76,36 @@ void Arm64JitBackend::CompIR_FArith(IRInst inst) {
fp_.FNEG(regs_.F(inst.dest), regs_.F(inst.src1));
break;
case IROp::FVfpuNaN:
{
regs_.Map(inst);
fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1));
if (inst.dest != inst.src1)
fp_.FMOV(regs_.F(inst.dest), regs_.F(inst.src1));
FixupBranch notNaN = B(CC_VC);
MOVI2R(SCRATCH1, 0x7F800001);
fp_.FMOV(regs_.F(inst.dest), SCRATCH1);
SetJumpTarget(notNaN);
break;
}
case IROp::FVfpuProductNaN:
{
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
fp_.FCMP(regs_.F(inst.dest), regs_.F(inst.dest));
FixupBranch notNaN = B(CC_VC);
fp_.FMOV(SCRATCH1, regs_.F(inst.src1));
fp_.FMOV(SCRATCH2, regs_.F(inst.src2));
EOR(SCRATCH1, SCRATCH1, SCRATCH2);
ANDI2R(SCRATCH1, SCRATCH1, 0x80000000, SCRATCH2);
ORRI2R(SCRATCH1, SCRATCH1, 0x7F800001, SCRATCH2);
fp_.FMOV(regs_.F(inst.dest), SCRATCH1);
SetJumpTarget(notNaN);
break;
}
default:
INVALIDOP;
break;
@@ -455,22 +485,20 @@ void Arm64JitBackend::CompIR_FSat(IRInst inst) {
switch (inst.op) {
case IROp::FSat0_1:
regs_.Map(inst);
fp_.MOVI2F(SCRATCHF1, 1.0f);
// Note that FMAX takes the larger of the two zeros, which is what we want.
fp_.MOVI2F(SCRATCHF2, 0.0f);
fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1);
fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2);
break;
case IROp::FSatMinus1_1:
regs_.Map(inst);
fp_.MOVI2F(SCRATCHF1, 1.0f);
fp_.FNEG(SCRATCHF2, SCRATCHF1);
// Note that FMAX takes the larger of the two zeros, which is what we want.
if (inst.op == IROp::FSat0_1)
fp_.MOVI2F(SCRATCHF2, 0.0f);
else
fp_.FNEG(SCRATCHF2, SCRATCHF1);
fp_.FMIN(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1);
fp_.FMAX(regs_.F(inst.dest), regs_.F(inst.dest), SCRATCHF2);
// A NaN passes through with its bits, which FMIN/FMAX would quiet (cpu/vfpu/nan_gen).
fp_.FCMP(regs_.F(inst.src1), regs_.F(inst.src1));
fp_.FMIN(SCRATCHF1, regs_.F(inst.src1), SCRATCHF1);
fp_.FMAX(SCRATCHF1, SCRATCHF1, SCRATCHF2);
fp_.FCSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, CC_VS);
break;
default:
+32
View File
@@ -140,6 +140,38 @@ void Arm64JitBackend::CompIR_VecArith(IRInst inst) {
fp_.SCVTF(32, regs_.FQ(inst.dest), regs_.FQ(inst.src1), inst.src2 & 0x1F);
break;
case IROp::Vec4VfpuNaN:
case IROp::Vec4VfpuProductNaN:
{
// The operands of the product are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.op == IROp::Vec4VfpuNaN || (inst.dest != inst.src1 && inst.dest != inst.src2));
regs_.Map(inst);
if (inst.op == IROp::Vec4VfpuNaN && inst.dest != inst.src1)
fp_.MOV(regs_.FQ(inst.dest), regs_.FQ(inst.src1));
// FMAXV gives a NaN if any lane is one.
fp_.FMAXV(SCRATCHF1, regs_.FQ(inst.dest));
fp_.FCMP(SCRATCHF1, SCRATCHF1);
FixupBranch noNaN = B(CC_VC);
// Rare: replace the NaN lanes with 0x7F800001, signed like the product for that op.
ARM64Reg orderedMask = EncodeRegToQuad(SCRATCHF1);
ARM64Reg nanReg = EncodeRegToQuad(SCRATCHF2);
fp_.FCMEQ(32, orderedMask, regs_.FQ(inst.dest), regs_.FQ(inst.dest));
if (inst.op == IROp::Vec4VfpuProductNaN) {
fp_.EOR(nanReg, regs_.FQ(inst.src1), regs_.FQ(inst.src2));
fp_.USHR(32, nanReg, nanReg, 31);
fp_.SHL(32, nanReg, nanReg, 31);
fp_.ORR(32, nanReg, 0x7F, 24);
} else {
fp_.MOVI(32, nanReg, 0x7F, 24);
}
fp_.ORR(32, nanReg, 0x80, 16);
fp_.ORR(32, nanReg, 0x01, 0);
fp_.BIF(regs_.FQ(inst.dest), nanReg, orderedMask);
SetJumpTarget(noNaN);
break;
}
default:
INVALIDOP;
break;
+46 -4
View File
@@ -1193,11 +1193,25 @@ namespace MIPSComp {
break;
}
// Sums and products make the VFPU's own NaN. vmin and vmax pass a NaN through, and vslt and
// vsge don't make one.
const bool sum = type == VecDo3Op::VADD || type == VecDo3Op::VSUB;
const bool product = type == VecDo3Op::VMUL || type == VecDo3Op::VDIV;
if (IsVec4(sz, dregs) && IsVec4(sz, sregs) && IsVec4(sz, tregs)) {
if (opFunc != IROp::Nop) {
ir.Write(opFunc, dregs[0], sregs[0], tregs[0]);
} else {
if (opFunc == IROp::Nop)
DISABLE;
if (!product) {
ir.Write(opFunc, dregs[0], sregs[0], tregs[0]);
if (sum)
ir.Write(IROp::Vec4VfpuNaN, dregs[0], dregs[0]);
} else if (dregs[0] != sregs[0] && dregs[0] != tregs[0]) {
ir.Write(opFunc, dregs[0], sregs[0], tregs[0]);
ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], tregs[0]);
} else {
// The NaN's sign comes from the operands, so they have to survive the op.
ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]);
ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]);
ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0);
}
ApplyPrefixD(dregs, sz, _VD);
return;
@@ -1207,6 +1221,10 @@ namespace MIPSComp {
if (opFunc == IROp::Nop)
DISABLE;
ir.Write(opFunc, IRVTEMP_0, sregs[0], tregs[0]);
if (product)
ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], tregs[0]);
else if (sum)
ir.Write(IROp::Vec4VfpuNaN, IRVTEMP_0, IRVTEMP_0);
ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7);
ApplyPrefixD(dregs, sz, _VD);
return;
@@ -1220,17 +1238,22 @@ namespace MIPSComp {
for (int i = 0; i < n; ++i) {
switch (type) {
// tempregs[i] is never one of the operands, which the product NaN needs.
case VecDo3Op::VADD: // d[i] = s[i] + t[i]; break; //vadd
ir.Write(IROp::FAdd, tempregs[i], sregs[i], tregs[i]);
ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]);
break;
case VecDo3Op::VSUB: // d[i] = s[i] - t[i]; break; //vsub
ir.Write(IROp::FSub, tempregs[i], sregs[i], tregs[i]);
ir.Write(IROp::FVfpuNaN, tempregs[i], tempregs[i]);
break;
case VecDo3Op::VDIV: // d[i] = s[i] / t[i]; break; //vdiv
ir.Write(IROp::FDiv, tempregs[i], sregs[i], tregs[i]);
ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]);
break;
case VecDo3Op::VMUL: // d[i] = s[i] * t[i]; break; //vmul
ir.Write(IROp::FMul, tempregs[i], sregs[i], tregs[i]);
ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], tregs[i]);
break;
case VecDo3Op::VMIN: // vmin
ir.Write(IROp::FMin, tempregs[i], sregs[i], tregs[i]);
@@ -1843,20 +1866,35 @@ namespace MIPSComp {
}
if (!overlap || (vs == vd && IsOverlapSafe(treg, n, dregs))) {
// The NaN's sign needs t in every lane (the T prefix temp is free, see above.)
if (IsVec4(sz, sregs) && IsVec4(sz, dregs)) {
ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg);
int tlane = treg & 3;
ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane));
if (dregs[0] != sregs[0]) {
ir.Write(IROp::Vec4Scale, dregs[0], sregs[0], treg);
ir.Write(IROp::Vec4VfpuProductNaN, dregs[0], sregs[0], IRVTEMP_PFX_T);
} else {
ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg);
ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T);
ir.Write(IROp::Vec4Mov, dregs[0], IRVTEMP_0);
}
ApplyPrefixD(dregs, sz, vd);
return;
} else if (IsVec3of4(sz, sregs) && IsVec3of4(sz, dregs) && opts.preferVec4) {
int tlane = treg & 3;
ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, treg & ~3, VFPU_SWIZZLE(tlane, tlane, tlane, tlane));
ir.Write(IROp::Vec4Scale, IRVTEMP_0, sregs[0], treg);
ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, sregs[0], IRVTEMP_PFX_T);
ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7);
ApplyPrefixD(dregs, sz, vd);
return;
}
}
// tempregs[i] is never s[i] or t, which the product NaN needs.
for (int i = 0; i < n; i++) {
ir.Write(IROp::FMul, tempregs[i], sregs[i], treg);
ir.Write(IROp::FVfpuProductNaN, tempregs[i], sregs[i], treg);
}
for (int i = 0; i < n; i++) {
@@ -2017,6 +2055,7 @@ namespace MIPSComp {
ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_S, sregs[0], VFPU_SWIZZLE(1, 2, 0, 3));
ir.Write(IROp::Vec4Shuffle, IRVTEMP_PFX_T, tregs[0], VFPU_SWIZZLE(2, 0, 1, 3));
ir.Write(IROp::Vec4Mul, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T);
ir.Write(IROp::Vec4VfpuProductNaN, IRVTEMP_0, IRVTEMP_PFX_S, IRVTEMP_PFX_T);
// Now just retain w and blend in our values.
ir.Write(IROp::Vec4Blend, dregs[0], dregs[0], IRVTEMP_0, 0x7);
} else {
@@ -2030,8 +2069,11 @@ namespace MIPSComp {
}
ir.Write(IROp::FMul, tempregs[0], sregs[1], tregs[2]);
ir.Write(IROp::FVfpuProductNaN, tempregs[0], sregs[1], tregs[2]);
ir.Write(IROp::FMul, tempregs[1], sregs[2], tregs[0]);
ir.Write(IROp::FVfpuProductNaN, tempregs[1], sregs[2], tregs[0]);
ir.Write(IROp::FMul, tempregs[2], sregs[0], tregs[1]);
ir.Write(IROp::FVfpuProductNaN, tempregs[2], sregs[0], tregs[1]);
for (int i = 0; i < n; i++) {
if (tempregs[i] != dregs[i])
+4
View File
@@ -136,6 +136,8 @@ static const IRMeta irMeta[] = {
{ IROp::FCmp, "FCmp", "mFF" },
{ IROp::FSat0_1, "FSat(0 - 1)", "FF" },
{ IROp::FSatMinus1_1, "FSat(-1 - 1)", "FF" },
{ IROp::FVfpuNaN, "FVfpuNaN", "FF" },
{ IROp::FVfpuProductNaN, "FVfpuProductNaN", "FFF", IRFLAG_SRC3DST },
{ IROp::FMovFromGPR, "FMovFromGPR", "FG" },
{ IROp::FMovToGPR, "FMovToGPR", "GF" },
{ IROp::OptFMovToGPRShr8, "OptFMovToGPRShr8", "GF" },
@@ -175,6 +177,8 @@ static const IRMeta irMeta[] = {
{ IROp::Vec4Slt, "Vec4Slt", "VVV" },
{ IROp::Vec4Sge, "Vec4Sge", "VVV" },
{ IROp::Vec4CvtScaledSW, "Vec4CvtScaledSW", "VVI" },
{ IROp::Vec4VfpuNaN, "Vec4VfpuNaN", "VV" },
{ IROp::Vec4VfpuProductNaN, "Vec4VfpuProductNaN", "VVV", IRFLAG_SRC3DST },
// Pack/Unpack
{ IROp::Vec2Unpack16To31, "Vec2Unpack16To31", "2F" }, // Note that the result is shifted down by 1, hence 31
+6
View File
@@ -163,6 +163,10 @@ enum class IROp : uint8_t {
FSat0_1,
FSatMinus1_1,
// The VFPU's NaNs (see VfpuSumNaN in InterpreterVFPU.cpp): src1 or 0x7F800001 if it's a NaN, and
// dest (read too) or 0x7F800001 with the sign of src1 ^ src2 if dest is a NaN.
FVfpuNaN,
FVfpuProductNaN,
FpCondFromReg,
FpCondToReg,
@@ -208,6 +212,8 @@ enum class IROp : uint8_t {
Vec4Sge,
// vi2f: each lane's int to float, divided by 2^src2.
Vec4CvtScaledSW,
Vec4VfpuNaN,
Vec4VfpuProductNaN,
// vx2i
Vec2Unpack16To31, // Note that the result is shifted down by 1, hence 31
+53
View File
@@ -50,6 +50,10 @@ alignas(16) static const uint32_t noSignMask[4] = {
0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF,
};
alignas(16) static const uint32_t vfpuNaN[4] = {
0x7F800001, 0x7F800001, 0x7F800001, 0x7F800001,
};
alignas(16) static const uint32_t lowBytesMask[4] = {
0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF,
};
@@ -245,6 +249,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
dispatch[(int)IROp::Vec4Scale] = &&L_Vec4Scale;
dispatch[(int)IROp::Vec4Neg] = &&L_Vec4Neg;
dispatch[(int)IROp::Vec4Abs] = &&L_Vec4Abs;
dispatch[(int)IROp::Vec4VfpuNaN] = &&L_Vec4VfpuNaN;
dispatch[(int)IROp::Vec4VfpuProductNaN] = &&L_Vec4VfpuProductNaN;
dispatch[(int)IROp::Vec2Unpack16To31] = &&L_Vec2Unpack16To31;
dispatch[(int)IROp::Vec2Unpack16To32] = &&L_Vec2Unpack16To32;
dispatch[(int)IROp::Vec4Unpack8To32] = &&L_Vec4Unpack8To32;
@@ -316,6 +322,8 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
dispatch[(int)IROp::FNeg] = &&L_FNeg;
dispatch[(int)IROp::FSat0_1] = &&L_FSat0_1;
dispatch[(int)IROp::FSatMinus1_1] = &&L_FSatMinus1_1;
dispatch[(int)IROp::FVfpuNaN] = &&L_FVfpuNaN;
dispatch[(int)IROp::FVfpuProductNaN] = &&L_FVfpuProductNaN;
dispatch[(int)IROp::FSign] = &&L_FSign;
dispatch[(int)IROp::FpCondFromReg] = &&L_FpCondFromReg;
dispatch[(int)IROp::FpCondToReg] = &&L_FpCondToReg;
@@ -759,6 +767,44 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
IR_NEXT;
}
IR_CASE(Vec4VfpuNaN)
{
#if PPSSPP_ARCH(SSE2)
__m128 v = _mm_load_ps(&mips->f[inst->src1]);
__m128 isNaN = _mm_cmpunord_ps(v, v);
_mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, _mm_load_ps((const float *)vfpuNaN))));
#elif PPSSPP_ARCH(ARM_NEON)
float32x4_t v = vld1q_f32(&mips->f[inst->src1]);
vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, vld1q_f32((const float *)vfpuNaN)));
#else
for (int i = 0; i < 4; i++)
mips->fi[inst->dest + i] = my_isnan(mips->f[inst->src1 + i]) ? 0x7F800001 : mips->fi[inst->src1 + i];
#endif
IR_NEXT;
}
IR_CASE(Vec4VfpuProductNaN)
{
#if PPSSPP_ARCH(SSE2)
__m128 v = _mm_load_ps(&mips->f[inst->dest]);
__m128 isNaN = _mm_cmpunord_ps(v, v);
__m128 sign = _mm_and_ps(_mm_xor_ps(_mm_load_ps(&mips->f[inst->src1]), _mm_load_ps(&mips->f[inst->src2])), _mm_load_ps((const float *)signBits));
__m128 nan = _mm_or_ps(sign, _mm_load_ps((const float *)vfpuNaN));
_mm_store_ps(&mips->f[inst->dest], _mm_or_ps(_mm_andnot_ps(isNaN, v), _mm_and_ps(isNaN, nan)));
#elif PPSSPP_ARCH(ARM_NEON)
float32x4_t v = vld1q_f32(&mips->f[inst->dest]);
uint32x4_t sign = vandq_u32(veorq_u32(vld1q_u32(&mips->fi[inst->src1]), vld1q_u32(&mips->fi[inst->src2])), vld1q_u32(signBits));
float32x4_t nan = vreinterpretq_f32_u32(vorrq_u32(sign, vld1q_u32(vfpuNaN)));
vst1q_f32(&mips->f[inst->dest], vbslq_f32(vceqq_f32(v, v), v, nan));
#else
for (int i = 0; i < 4; i++) {
if (my_isnan(mips->f[inst->dest + i]))
mips->fi[inst->dest + i] = 0x7F800001 | ((mips->fi[inst->src1 + i] ^ mips->fi[inst->src2 + i]) & 0x80000000);
}
#endif
IR_NEXT;
}
IR_CASE(Vec2Unpack16To31)
{
const u32 dest = inst->dest;
@@ -1245,6 +1291,13 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
IR_CASE(FSatMinus1_1)
mips->f[inst->dest] = vfpu_clamp(mips->f[inst->src1], -1.0f, 1.0f);
IR_NEXT;
IR_CASE(FVfpuNaN)
mips->fi[inst->dest] = my_isnan(mips->f[inst->src1]) ? 0x7F800001 : mips->fi[inst->src1];
IR_NEXT;
IR_CASE(FVfpuProductNaN)
if (my_isnan(mips->f[inst->dest]))
mips->fi[inst->dest] = 0x7F800001 | ((mips->fi[inst->src1] ^ mips->fi[inst->src2]) & 0x80000000);
IR_NEXT;
IR_CASE(FSign)
{
+4
View File
@@ -348,6 +348,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
case IROp::FDiv:
case IROp::FSqrt:
case IROp::FNeg:
case IROp::FVfpuNaN:
case IROp::FVfpuProductNaN:
CompIR_FArith(inst);
break;
@@ -426,6 +428,8 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
case IROp::Vec4Slt:
case IROp::Vec4Sge:
case IROp::Vec4CvtScaledSW:
case IROp::Vec4VfpuNaN:
case IROp::Vec4VfpuProductNaN:
CompIR_VecArith(inst);
break;
+10 -2
View File
@@ -1031,6 +1031,14 @@ bool IRMutatesDestGPR(const IRInstMeta &inst, int reg) {
return (inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'G' && inst.src3 == reg;
}
// Whether inst reads the FPR it writes (reg counted from 32, like the checks in PurgeTemps).
static bool IRMutatesDestFPR(const IRInstMeta &inst, int reg) {
if ((inst.m.flags & IRFLAG_SRC3DST) == 0)
return false;
int len = inst.m.types[0] == 'F' ? 1 : (inst.m.types[0] == 'V' ? 4 : 0);
return reg >= inst.src3 + 32 && reg < inst.src3 + 32 + len;
}
bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) {
CONDITIONAL_DISABLE;
std::vector<IRInstMeta> insts;
@@ -1192,7 +1200,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) {
if (numDest > 1 && check.srcReg >= destRegs[0] + 32 && check.srcReg < destRegs[0] + 32 + numDest)
cannotReplace = true;
// Nor where it's read as src3 that's also the destination, which renaming would move.
if ((inst.m.flags & IRFLAG_SRC3DST) != 0 && inst.m.types[0] == 'F' && inst.src3 + 32 == check.reg)
if (IRMutatesDestFPR(inst, check.reg))
cannotReplace = true;
if (!cannotReplace && check.srcReg >= 32 && lastWrittenTo[check.srcReg] < check.index) {
// This is probably not worth doing unless we can get rid of a temp.
@@ -1219,7 +1227,7 @@ bool PurgeTemps(const IRWriter &in, IRWriter &out, const IROptions &opts) {
// Let's not bother.
check.reg = 0;
}
} else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch) {
} else if ((inst.op == IROp::FMov || inst.op == IROp::Vec4Mov) && !lenMismatch && !IRMutatesDestFPR(insts[check.index], check.reg)) {
// A swap could be profitable if this is a temp, and maybe in other cases.
// These can happen a lot from mask regs, etc.
// But make sure no other changes happened between.
+31 -16
View File
@@ -75,7 +75,7 @@
#define M_SQRT1_2 0.707106781186547524401f
#endif
// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3).
// The VFPU's own handling of denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3).
static const bool USE_VFPU_RESULT_FIXUP = false;
union FloatBits {
@@ -84,6 +84,26 @@ union FloatBits {
int i[4];
};
// The VFPU never keeps a NaN's payload: every NaN it produces is 0x7F800001, negative for a product
// or quotient of operands of different signs (cpu/vfpu/nan_gen). Host NaNs differ by host.
static inline float VfpuSumNaN(float r) {
if (!my_isnan(r))
return r;
FP32 nan;
nan.u = 0x7F800001;
return nan.f;
}
static inline float VfpuProductNaN(float r, float a, float b) {
if (!my_isnan(r))
return r;
FP32 fa, fb, nan;
fa.f = a;
fb.f = b;
nan.u = 0x7F800001 | ((fa.u ^ fb.u) & 0x80000000);
return nan.f;
}
// Preserves NaN in first param, takes sign of equal second param.
// Technically, std::max may do this but it's undefined.
inline float nanmax(float f, float cst)
@@ -1436,10 +1456,9 @@ namespace MIPSInt
u32 tprefixAdd = VFPU_SWIZZLE(2, 0, 1, 0);
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), sz);
d[0] = s[0] * t[0];
d[1] = s[1] * t[1];
d[2] = s[2] * t[2];
d[3] = s[3] * t[3];
for (int i = 0; i < 4; i++) {
d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]);
}
ApplyPrefixD(mips, d, sz);
WriteVector(mips, d, sz, vd);
PC += 4;
@@ -1544,7 +1563,7 @@ namespace MIPSInt
int n = GetNumVectorElements(sz);
for (int i = 0; i < n; i++) {
d[i] = s[i] * t[i];
d[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]);
}
ApplyPrefixD(mips, d, sz);
WriteVector(mips, d, sz, vd);
@@ -2105,18 +2124,14 @@ namespace MIPSInt
for (int i = 0; i < (int)n; i++) {
switch (optype) {
case 0: d.f[i] = s[i] + t[i]; break; //vadd
case 1: d.f[i] = s[i] - t[i]; break; //vsub
case 7: d.f[i] = s[i] / t[i]; break; //vdiv
case 8: d.f[i] = s[i] * t[i]; break; //vmul
case 0: d.f[i] = VfpuSumNaN(s[i] + t[i]); break; //vadd
case 1: d.f[i] = VfpuSumNaN(s[i] - t[i]); break; //vsub
case 7: d.f[i] = VfpuProductNaN(s[i] / t[i], s[i], t[i]); break; //vdiv
case 8: d.f[i] = VfpuProductNaN(s[i] * t[i], s[i], t[i]); break; //vmul
}
if (USE_VFPU_RESULT_FIXUP) {
if (my_isnan(d.f[i])) {
d.u[i] = (d.u[i] & 0xff800001) | 1;
} else if ((d.u[i] & 0x7F800000) == 0) {
d.u[i] &= 0xFF800000;
}
if (USE_VFPU_RESULT_FIXUP && (d.u[i] & 0x7F800000) == 0) {
d.u[i] &= 0xFF800000;
}
}
@@ -70,6 +70,33 @@ void LoongArch64JitBackend::CompIR_FArith(IRInst inst) {
FNEG_S(regs_.F(inst.dest), regs_.F(inst.src1));
break;
case IROp::FVfpuNaN:
regs_.Map(inst);
FCMP_COND_S(FCC0, regs_.F(inst.src1), regs_.F(inst.src1), LoongArch64Fcond::CUN);
QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1);
FSEL(regs_.F(inst.dest), regs_.F(inst.src1), SCRATCHF1, FCC0);
break;
case IROp::FVfpuProductNaN:
{
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
FCMP_COND_S(FCC0, regs_.F(inst.dest), regs_.F(inst.dest), LoongArch64Fcond::CUN);
MOVCF2GR(SCRATCH1, FCC0);
FixupBranch notNaN = BEQZ(SCRATCH1);
MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1));
MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2));
XOR(SCRATCH1, SCRATCH1, SCRATCH2);
SRLI_W(SCRATCH1, SCRATCH1, 31);
SLLI_W(SCRATCH1, SCRATCH1, 31);
LI(SCRATCH2, 0x7F800001);
OR(SCRATCH1, SCRATCH1, SCRATCH2);
MOVGR2FR_W(regs_.F(inst.dest), SCRATCH1);
SetJumpTarget(notNaN);
break;
}
default:
INVALIDOP;
break;
@@ -431,6 +431,55 @@ void LoongArch64JitBackend::CompIR_VecArith(IRInst inst) {
}
break;
case IROp::Vec4VfpuNaN:
regs_.Map(inst);
if (cpu_info.LOONGARCH_LSX) {
VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.src1), regs_.V(inst.src1));
LI(SCRATCH1, 0x7F800001);
VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1);
VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.src1), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1));
} else {
QuickFLI(32, SCRATCHF1, (uint32_t)0x7F800001, SCRATCH1);
for (int i = 0; i < 4; ++i) {
FCMP_COND_S(FCC0, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i), LoongArch64Fcond::CUN);
FSEL(regs_.F(inst.dest + i), regs_.F(inst.src1 + i), SCRATCHF1, FCC0);
}
}
break;
case IROp::Vec4VfpuProductNaN:
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
if (cpu_info.LOONGARCH_LSX) {
// Put the product's sign in the NaN lanes, then OR in 0x7F800001 there.
VFCMP_CUN_S(EncodeRegToV(SCRATCHF1), regs_.V(inst.dest), regs_.V(inst.dest));
VXOR_V(EncodeRegToV(SCRATCHF2), regs_.V(inst.src1), regs_.V(inst.src2));
VSRLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31);
VSLLI_W(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), 31);
VBITSEL_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1));
LI(SCRATCH1, 0x7F800001);
VREPLGR2VR_W(EncodeRegToV(SCRATCHF2), SCRATCH1);
VAND_V(EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF2), EncodeRegToV(SCRATCHF1));
VOR_V(regs_.V(inst.dest), regs_.V(inst.dest), EncodeRegToV(SCRATCHF2));
} else {
for (int i = 0; i < 4; ++i) {
FCMP_COND_S(FCC0, regs_.F(inst.dest + i), regs_.F(inst.dest + i), LoongArch64Fcond::CUN);
MOVCF2GR(SCRATCH1, FCC0);
FixupBranch notNaN = BEQZ(SCRATCH1);
MOVFR2GR_S(SCRATCH1, regs_.F(inst.src1 + i));
MOVFR2GR_S(SCRATCH2, regs_.F(inst.src2 + i));
XOR(SCRATCH1, SCRATCH1, SCRATCH2);
SRLI_W(SCRATCH1, SCRATCH1, 31);
SLLI_W(SCRATCH1, SCRATCH1, 31);
LI(SCRATCH2, 0x7F800001);
OR(SCRATCH1, SCRATCH1, SCRATCH2);
MOVGR2FR_W(regs_.F(inst.dest + i), SCRATCH1);
SetJumpTarget(notNaN);
}
}
break;
default:
INVALIDOP;
break;
+32
View File
@@ -69,6 +69,38 @@ void RiscVJitBackend::CompIR_FArith(IRInst inst) {
FNEG(32, regs_.F(inst.dest), regs_.F(inst.src1));
break;
case IROp::FVfpuNaN:
{
regs_.Map(inst);
FEQ(32, SCRATCH1, regs_.F(inst.src1), regs_.F(inst.src1));
if (inst.dest != inst.src1)
FMV(32, regs_.F(inst.dest), regs_.F(inst.src1));
FixupBranch notNaN = BNE(SCRATCH1, R_ZERO);
LI(SCRATCH1, 0x7F800001);
FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1);
SetJumpTarget(notNaN);
break;
}
case IROp::FVfpuProductNaN:
{
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
FEQ(32, SCRATCH1, regs_.F(inst.dest), regs_.F(inst.dest));
FixupBranch notNaN = BNE(SCRATCH1, R_ZERO);
FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1));
FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2));
XOR(SCRATCH1, SCRATCH1, SCRATCH2);
SRLIW(SCRATCH1, SCRATCH1, 31);
SLLIW(SCRATCH1, SCRATCH1, 31);
LI(SCRATCH2, 0x7F800001);
OR(SCRATCH1, SCRATCH1, SCRATCH2);
FMV(FMv::W, FMv::X, regs_.F(inst.dest), SCRATCH1);
SetJumpTarget(notNaN);
break;
}
default:
INVALIDOP;
break;
+32
View File
@@ -322,6 +322,38 @@ void RiscVJitBackend::CompIR_VecArith(IRInst inst) {
break;
}
case IROp::Vec4VfpuNaN:
regs_.Map(inst);
for (int i = 0; i < 4; ++i) {
FEQ(32, SCRATCH1, regs_.F(inst.src1 + i), regs_.F(inst.src1 + i));
if (inst.dest != inst.src1)
FMV(32, regs_.F(inst.dest + i), regs_.F(inst.src1 + i));
FixupBranch notNaN = BNE(SCRATCH1, R_ZERO);
LI(SCRATCH1, 0x7F800001);
FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1);
SetJumpTarget(notNaN);
}
break;
case IROp::Vec4VfpuProductNaN:
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
for (int i = 0; i < 4; ++i) {
FEQ(32, SCRATCH1, regs_.F(inst.dest + i), regs_.F(inst.dest + i));
FixupBranch notNaN = BNE(SCRATCH1, R_ZERO);
FMV(FMv::X, FMv::W, SCRATCH1, regs_.F(inst.src1 + i));
FMV(FMv::X, FMv::W, SCRATCH2, regs_.F(inst.src2 + i));
XOR(SCRATCH1, SCRATCH1, SCRATCH2);
SRLIW(SCRATCH1, SCRATCH1, 31);
SLLIW(SCRATCH1, SCRATCH1, 31);
LI(SCRATCH2, 0x7F800001);
OR(SCRATCH1, SCRATCH1, SCRATCH2);
FMV(FMv::W, FMv::X, regs_.F(inst.dest + i), SCRATCH1);
SetJumpTarget(notNaN);
}
break;
default:
INVALIDOP;
break;
+29 -1
View File
@@ -46,6 +46,7 @@ void X64JitBackend::EmitFPUConstants() {
EmitConst4x32(&constants.positiveZeroes, 0x00000000);
EmitConst4x32(&constants.positiveInfinity, 0x7F800000);
EmitConst4x32(&constants.qNAN, 0x7FC00000);
EmitConst4x32(&constants.vfpuNaN, 0x7F800001);
EmitConst4x32(&constants.positiveOnes, 0x3F800000);
EmitConst4x32(&constants.negativeOnes, 0xBF800000);
EmitConst4x32(&constants.maxIntBelowAsFloat, 0x4EFFFFFF);
@@ -204,6 +205,33 @@ void X64JitBackend::CompIR_FArith(IRInst inst) {
}
break;
case IROp::FVfpuNaN:
{
regs_.Map(inst);
if (inst.dest != inst.src1)
MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1));
UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest));
FixupBranch notNaN = J_CC(CC_NP);
MOVSS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible
SetJumpTarget(notNaN);
break;
}
case IROp::FVfpuProductNaN:
{
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.Map(inst);
UCOMISS(regs_.FX(inst.dest), regs_.F(inst.dest));
FixupBranch notNaN = J_CC(CC_NP);
MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1));
XORPS(regs_.FX(inst.dest), regs_.F(inst.src2));
ANDPS(regs_.FX(inst.dest), M(constants.signBitAll)); // rip accessible
ORPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible
SetJumpTarget(notNaN);
break;
}
default:
INVALIDOP;
break;
@@ -872,7 +900,7 @@ void X64JitBackend::CompIR_FSat(IRInst inst) {
tempReg = regs_.MapWithFPRTemp(inst);
// The second argument's NAN is taken if either is NAN, so put known first.
MOVSS(tempReg, M(constants.positiveOnes));
MOVSS(tempReg, M(constants.positiveOnes)); // rip accessible
MINSS(tempReg, regs_.F(inst.src1));
// Known first again, so a NAN stays, bits and all. This keeps -0.0 too, since a tie also
+45
View File
@@ -175,6 +175,51 @@ void X64JitBackend::CompIR_VecArith(IRInst inst) {
}
break;
case IROp::Vec4VfpuNaN:
{
X64Reg maskReg = regs_.MapWithFPRTemp(inst);
if (inst.dest != inst.src1)
MOVAPS(regs_.FX(inst.dest), regs_.F(inst.src1));
MOVAPS(maskReg, regs_.F(inst.dest));
CMPUNORDPS(maskReg, regs_.F(inst.dest));
MOVMSKPS(SCRATCH1, R(maskReg));
TEST(32, R(SCRATCH1), R(SCRATCH1));
FixupBranch noNaN = J_CC(CC_Z);
// Rare: keep the other lanes, replace the NaNs.
ANDNPS(maskReg, regs_.F(inst.dest));
CMPUNORDPS(regs_.FX(inst.dest), regs_.F(inst.dest));
ANDPS(regs_.FX(inst.dest), M(constants.vfpuNaN)); // rip accessible
ORPS(regs_.FX(inst.dest), R(maskReg));
SetJumpTarget(noNaN);
break;
}
case IROp::Vec4VfpuProductNaN:
{
// The operands are never the dest, see IRFrontend::Comp_VecDo3.
_dbg_assert_(inst.dest != inst.src1 && inst.dest != inst.src2);
regs_.SpillLockFPR(inst.dest, inst.src1, inst.src2);
X64Reg maskReg = regs_.GetAndLockTempFPR();
X64Reg nanReg = regs_.GetAndLockTempFPR();
regs_.Map(inst);
MOVAPS(maskReg, regs_.F(inst.dest));
CMPUNORDPS(maskReg, regs_.F(inst.dest));
MOVMSKPS(SCRATCH1, R(maskReg));
TEST(32, R(SCRATCH1), R(SCRATCH1));
FixupBranch noNaN = J_CC(CC_Z);
// Rare: replace the NaNs, with the sign of the operands' product.
MOVAPS(nanReg, regs_.F(inst.src1));
XORPS(nanReg, regs_.F(inst.src2));
ANDPS(nanReg, M(constants.signBitAll)); // rip accessible
ORPS(nanReg, M(constants.vfpuNaN)); // rip accessible
ANDPS(nanReg, R(maskReg));
ANDNPS(maskReg, regs_.F(inst.dest));
ORPS(nanReg, R(maskReg));
MOVAPS(regs_.FX(inst.dest), R(nanReg));
SetJumpTarget(noNaN);
break;
}
default:
INVALIDOP;
break;
+1
View File
@@ -161,6 +161,7 @@ private:
const void *positiveOnes;
const void *negativeOnes;
const void *qNAN;
const void *vfpuNaN;
const void *maxIntBelowAsFloat;
const float *mulTableVi2f;
const float *mulTableVf2i;