From 1952d20805eb834b0e95caee6a82c11c75c825a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 12:48:34 +0200 Subject: [PATCH 01/17] Config: Add AccurateVdot, and --accurate-vdot to set it Not wired to anything yet. It will make every core compute the VFPU's dot products (vdot, vmmul, vtfm and the rest) bit-exactly like the PSP, through vfpu_dot(). Co-Authored-By: Claude Opus 5.5 --- Core/CmdLine.cpp | 5 +++++ Core/CmdLine.h | 1 + Core/Config.cpp | 1 + Core/Config.h | 2 ++ 4 files changed, 9 insertions(+) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 25dee0f601..861bfc755f 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -199,6 +199,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode", CmdLineMode::Headless}, {POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode", CmdLineMode::Headless}, {POFF(oldAtrac), CmdParamType::Bool, "old-atrac", '\0', "Use old ATRAC decoder"}, + {POFF(accurateVdot), CmdParamType::Bool, "accurate-vdot", '\0', "Compute VFPU dot products bit-exactly like the PSP (slower)"}, {POFF(memStickInserted), CmdParamType::BoolInverse, "no-memstick", '\0', "Run with the memory stick removed"}, {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application}, {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, @@ -610,6 +611,10 @@ void CommandLineOptions::ApplyToConfig() const { g_Config.bUseOldAtrac = oldAtrac.value(); g_Config.DoNotSaveSetting(&g_Config.bUseOldAtrac); } + if (accurateVdot.has_value()) { + g_Config.bAccurateVdot = accurateVdot.value(); + g_Config.DoNotSaveSetting(&g_Config.bAccurateVdot); + } if (root.has_value()) { // No DoNotSaveSetting() here - mountRoot isn't an ordinary setting and never gets diff --git a/Core/CmdLine.h b/Core/CmdLine.h index c2a3814b22..66a0f9e9b4 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -166,6 +166,7 @@ struct CommandLineOptions { bool optionS = false; // a legacy option std::optional oldAtrac; + std::optional accurateVdot; // Headless options that may also be mildly useful in application mode std::optional resolutionScale; diff --git a/Core/Config.cpp b/Core/Config.cpp index b8e59fd76c..2b8cb2e513 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -436,6 +436,7 @@ static const ConfigSetting cpuSettings[] = { ConfigSetting("IOTimingMethod", SETTING(g_Config, iIOTimingMethod), IOTIMING_FAST, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("FastMemoryAccess", SETTING(g_Config, bFastMemory), true, CfgFlag::PER_GAME), ConfigSetting("FunctionReplacements", SETTING(g_Config, bFuncReplacements), true, CfgFlag::PER_GAME | CfgFlag::REPORT), + ConfigSetting("AccurateVdot", SETTING(g_Config, bAccurateVdot), false, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("HideSlowWarnings", SETTING(g_Config, bHideSlowWarnings), false, CfgFlag::DEFAULT), ConfigSetting("HideStateWarnings", SETTING(g_Config, bHideStateWarnings), false, CfgFlag::DEFAULT), ConfigSetting("JitDisableFlags", SETTING(g_Config, uJitDisableFlags), (uint32_t)0, CfgFlag::PER_GAME), diff --git a/Core/Config.h b/Core/Config.h index 9458f6274a..5a2bcd0d62 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -235,6 +235,8 @@ public: bool bWow64WarningDismissed; bool bForceLagSync; bool bFuncReplacements; + // Compute the VFPU's dot products (vdot, vmmul, vtfm and the rest) bit-exactly like the PSP. + bool bAccurateVdot; bool bHideSlowWarnings; bool bHideStateWarnings; uint32_t uJitDisableFlags; From 23fd6b454642513512b7b3280ffe6656eba54243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 12:50:47 +0200 Subject: [PATCH 02/17] Interpreter: Use the accurate VFPU dot product when AccurateVdot is set vdot, vhdp, vfad, vavg, vdet, vtfm/vhtfm, vcrsp/vqmul and vmmul compute through vfpu_dot() then, as the PSP does. vmmul still also takes it from the MoreAccurateVMMUL compat flag. With it, signed_zero matches the PSP, and fcr31, nan_gen and specials get closer. What's left there is the single-lane ops' NaNs and the VFPU's rounding. vcrsp.t no longer zeroes infinities before its last dot: inf * 0 gives the VFPU's NaN there on the PSP (cpu/vfpu/nan_gen). The NaN and denormal fixups in vadd, vsub, vmul and vdiv shared the flag but aren't dot products, so they get their own (still off). Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/InterpreterVFPU.cpp | 43 ++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Core/MIPS/InterpreterVFPU.cpp b/Core/MIPS/InterpreterVFPU.cpp index 598fec521c..45331fc06c 100644 --- a/Core/MIPS/InterpreterVFPU.cpp +++ b/Core/MIPS/InterpreterVFPU.cpp @@ -28,6 +28,7 @@ #include "Core/Core.h" #include "Core/MemMap.h" #include "Core/Reporting.h" +#include "Core/Config.h" #include "Core/System.h" #include "Core/MIPS/MIPS.h" @@ -74,7 +75,13 @@ #define M_SQRT1_2 0.707106781186547524401f #endif -static const bool USE_VFPU_DOT = false; +// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). +static const bool USE_VFPU_RESULT_FIXUP = false; + +// The dot products (vdot, vmmul, vtfm and the rest) through vfpu_dot(), bit-exact with the PSP. +static inline bool AccurateVdot() { + return g_Config.bAccurateVdot; +} union FloatBits { float f[4]; @@ -508,7 +515,7 @@ namespace MIPSInt ReadMatrix(mips, t, sz, vt); // TODO: Always use the more accurate path in interpreter? - bool useAccurateDot = USE_VFPU_DOT || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL; + bool useAccurateDot = AccurateVdot() || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL; for (int a = 0; a < n; a++) { for (int b = 0; b < n; b++) { union { float f; uint32_t u; } sum = { 0.0f }; @@ -1208,7 +1215,7 @@ namespace MIPSInt ReadVector(mips, t, sz, vt); ApplySwizzleT(mips, t, V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d.f = vfpu_dot(s, t); if (my_isnan(d.f)) { d.u = 0x7f800001; @@ -1259,7 +1266,7 @@ namespace MIPSInt ApplyPrefixST(mips, s, VFPURewritePrefix(mips, VFPU_CTRL_SPREFIX, sprefixRemove, sprefixAdd), V_Quad); float sum = 0.0f; - if (USE_VFPU_DOT) { + if (AccurateVdot()) { sum = vfpu_dot(s, t); } else { for (int i = 0; i < 4; i++) { @@ -1479,7 +1486,7 @@ namespace MIPSInt u32 tprefixAdd = VFPU_SWIZZLE(1, 0, 0, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { s[1] = -s[1]; d[0] = vfpu_dot(s, t); } else { @@ -1507,7 +1514,7 @@ namespace MIPSInt u32 tprefixAdd = VFPU_MAKE_CONSTANTS(VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d = vfpu_dot(s, t); } else { d = 0.0f; @@ -1545,7 +1552,7 @@ namespace MIPSInt tprefixAdd = 0; ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d = vfpu_dot(s, t); } else { d = 0.0f; @@ -1708,7 +1715,7 @@ namespace MIPSInt ReadMatrix(mips, s, msz, vs); ReadVector(mips, t, sz, vt); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t2[4]; for (int i = 0; i < 4; i++) { if (i < tn) { @@ -1762,7 +1769,7 @@ namespace MIPSInt ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); // Really this is the operation all rows probably use (with constant wiring.) - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d.f[ins] = vfpu_dot(&s[ins * 4], t); if (my_isnan(d.f[ins])) { @@ -2163,7 +2170,7 @@ namespace MIPSInt case 8: d.f[i] = s[i] * t[i]; break; //vmul } - if (USE_VFPU_DOT) { + if (USE_VFPU_RESULT_FIXUP) { if (my_isnan(d.f[i])) { d.u[i] = (d.u[i] & 0xff800001) | 1; } else if ((d.u[i] & 0x7F800000) == 0) { @@ -2206,7 +2213,7 @@ namespace MIPSInt switch (sz) { case V_Triple: // vcrsp.t { - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t0[4] = { 0.0f, t[2], -t[1], 0.0f }; float t1[4] = { -t[2], 0.0f, t[0], 0.0f }; d[0] = vfpu_dot(s, t0); @@ -2220,14 +2227,8 @@ namespace MIPSInt tprefixAdd = VFPU_SWIZZLE(1, 0, 3, 2) | VFPU_NEGATE(0, 1, 0, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); ApplySwizzleS(mips, s, V_Quad); - if (USE_VFPU_DOT) { - // TODO: But flush any infs to 0? This seems sketchy. - for (int i = 0; i < 4; ++i) { - if (my_isinf(s[i])) - s[i] = 0.0f; - if (my_isinf(t[i])) - t[i] = 0.0f; - } + if (AccurateVdot()) { + // Infinities stay: inf * 0 gives the VFPU's NaN here (cpu/vfpu/nan_gen). d[2] = vfpu_dot(s, t); } else { d[2] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3]; @@ -2237,7 +2238,7 @@ namespace MIPSInt case V_Quad: // vqmul.q { - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t0[4] = { t[3], t[2], -t[1], t[0] }; float t1[4] = { -t[2], t[3], t[0], t[1] }; float t2[4] = { t[1], -t[0], t[3], t[2] }; @@ -2254,7 +2255,7 @@ namespace MIPSInt tprefixAdd = VFPU_SWIZZLE(0, 1, 2, 3) | VFPU_NEGATE(1, 1, 1, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); ApplySwizzleS(mips, s, sz); - if (USE_VFPU_DOT) + if (AccurateVdot()) d[3] = vfpu_dot(s, t); else d[3] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3]; From bbe2d79beac2e156820a8b0b63ce788e34190ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 13:08:47 +0200 Subject: [PATCH 03/17] IR: Add Vec4VfpuDot, the VFPU's own dot product, and use it with AccurateVdot The frontend lowers vdot, vhdp, vfad, vavg, vdet, vcrsp/vqmul, vtfm/vhtfm and vmmul to it then, with the operands built from the same lanes, constants and negations the interpreter feeds vfpu_dot(). BuildDotOperands() puts them in Vec4 regs, using the free temps where they aren't one already, and the ops fall back to the interpreter when there aren't enough. The IR interpreter runs it through vfpu_dot(), and the native backends through their generic fallback for now. vmmul takes the same path for the MoreAccurateVMMUL compat flag. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/IR/IRCompVFPU.cpp | 246 +++++++++++++++++++++++++++++++- Core/MIPS/IR/IRFrontend.h | 15 ++ Core/MIPS/IR/IRInst.cpp | 1 + Core/MIPS/IR/IRInst.h | 1 + Core/MIPS/IR/IRInterpreter.cpp | 5 + Core/MIPS/IR/IRNativeCommon.cpp | 5 + Core/MIPS/IR/IRPassSimplify.cpp | 2 + 7 files changed, 269 insertions(+), 6 deletions(-) diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index ac8003b9ec..5d2b2bcaaa 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -150,6 +150,89 @@ namespace MIPSComp { return true; } + bool IRFrontend::UseAccurateVdot() const { + return g_Config.bAccurateVdot; + } + + // The lanes are already a Vec4 reg, used as is. + static bool DotLanesAreVec4(const IRFrontend::DotLane lanes[4]) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg < 0 || lanes[i].negate || lanes[i].reg != lanes[0].reg + i) + return false; + } + return (lanes[0].reg & 3) == 0; + } + + static bool DotLanesRead(const IRFrontend::DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg >= vec && lanes[i].reg < vec + 4) + return true; + } + return false; + } + + // Building lane i into vec + i only clobbers what a later lane still has to read. + static bool DotLanesBuildableIn(const IRFrontend::DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + for (int j = i + 1; j < 4; j++) { + if (lanes[j].reg == vec + i) + return false; + } + } + return true; + } + + bool IRFrontend::BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec) { + const bool sReady = DotLanesAreVec4(s), tReady = DotLanesAreVec4(t); + // The D temp holds the destination when a D prefix masks lanes. + const int temps[] = { IRVTEMP_0, js.prefixD == 0 ? IRVTEMP_PFX_D : -1, IRVTEMP_PFX_T, IRVTEMP_PFX_S }; + + int sTemp = -1, tTemp = -1; + if (!sReady) { + for (int temp : temps) { + if (temp >= 0 && temp != avoid && !DotLanesRead(t, temp) && DotLanesBuildableIn(s, temp)) { + sTemp = temp; + break; + } + } + if (sTemp < 0) + return false; + } + if (!tReady) { + for (int temp : temps) { + // Built after s, so it mustn't overwrite what s is (built or not). + const int sVecReg = sReady ? s[0].reg : sTemp; + const bool hitsS = sVecReg >= temp && sVecReg < temp + 4; + if (temp >= 0 && temp != avoid && temp != sTemp && !hitsS && DotLanesBuildableIn(t, temp)) { + tTemp = temp; + break; + } + } + if (tTemp < 0) + return false; + } + + auto build = [&](const DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg < 0) { + ir.WriteSetConstantFloat(vec + i, lanes[i].constant); + } else if (lanes[i].negate) { + ir.Write(IROp::FNeg, vec + i, lanes[i].reg); + } else if (lanes[i].reg != vec + i) { + ir.Write(IROp::FMov, vec + i, lanes[i].reg); + } + } + }; + // s's temp holds no lane of t, and t's temp isn't where s is, so neither clobbers the other. + if (!sReady) + build(s, sTemp); + if (!tReady) + build(t, tTemp); + *sVec = sReady ? s[0].reg : sTemp; + *tVec = tReady ? t[0].reg : tTemp; + return true; + } + // True if the prefix only touches lanes the op has. A position past the size may only be the // identity, and a position within it may not name a lane past it (which zeroes the result lane // on hardware, see cpu/vfpu/prefix_ctrl - the interpreter handles that). @@ -698,6 +781,21 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, vt); GetVectorRegsPrefixD(dregs, V_Single, vd); + if (UseAccurateVdot()) { + // s's last lane is 1, so t's is added in. + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + sl[i] = i < n - 1 ? DotReg(sregs[i]) : DotConst(i == n - 1 ? 1.0f : 0.0f); + tl[i] = i < n ? DotReg(tregs[i]) : DotConst(0.0f); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, vd); + return; + } + ir.Write(IROp::FMul, IRVTEMP_0, sregs[0], tregs[0]); for (int i = 1; i < n; i++) { @@ -741,6 +839,25 @@ namespace MIPSComp { GetVectorRegsPrefixS(sregs, sz, _VS); GetVectorRegsPrefixD(dregs, V_Single, _VD); + if (UseAccurateVdot()) { + // Dot products with constants from the T prefix in every lane: ones for vfad, 1/n for + // vavg (0 for vavg.s), as in the interpreter. + const bool vavg = ((op >> 16) & 31) == 7; + static const float vavgConst[4] = { 0.0f, 0.5f, 1.0f / 3.0f, 0.25f }; + const float c = vavg ? vavgConst[n - 1] : 1.0f; + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(0.0f); + tl[i] = DotConst(c); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, _VD); + return; + } + // We have to start at +0.000 in case any values are -0.000. ir.WriteSetConstantFloat(IRVTEMP_0, 0.0f); for (int i = 0; i < n; ++i) { @@ -790,6 +907,21 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, vt); GetVectorRegsPrefixD(dregs, V_Single, vd); + if (UseAccurateVdot()) { + // The prefixes stay within n lanes, so the rest are zero, as in the interpreter. + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(0.0f); + tl[i] = i < n ? DotReg(tregs[i]) : DotConst(0.0f); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, vd); + return; + } + if (IsVec4(sz, sregs) && IsVec4(sz, tregs)) { if (IsOverlapSafe(dregs[0], n, sregs, n, tregs)) { ir.Write(IROp::Vec4Dot, dregs[0], sregs[0], tregs[0]); @@ -1615,12 +1747,6 @@ namespace MIPSComp { DISABLE; } - if (PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { - // Fall back to interpreter, which has the accurate implementation. - // Later we might do something more optimized here. - DISABLE; - } - // Matrix multiply (weird prefixes) // D[0 .. N, 0 .. M] = S[0 .. N, 0 .. M]' * T[0 .. N, 0 .. M] // Note: Behaves as if it's implemented through a series of vdots. @@ -1635,6 +1761,35 @@ namespace MIPSComp { MatrixOverlapType soverlap = GetMatrixOverlap(vs, vd, sz); MatrixOverlapType toverlap = GetMatrixOverlap(vt, vd, sz); + if (UseAccurateVdot() || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { + // d[a][b] = s[b] dot t[a], each a row of n lanes and zeros, as in the interpreter. With + // no overlap, the results can go straight to d. + if (soverlap || toverlap) + DISABLE; + u8 sregs[16], tregs[16], dregs[16]; + GetMatrixRegs(sregs, sz, vs); + GetMatrixRegs(tregs, sz, vt); + GetMatrixRegs(dregs, sz, vd); + for (int a = 0; a < n; a++) { + DotLane tl[4]; + for (int c = 0; c < 4; c++) + tl[c] = c < n ? DotReg(tregs[a * 4 + c]) : DotConst(0.0f); + for (int b = 0; b < n; b++) { + DotLane sl[4]; + for (int c = 0; c < 4; c++) + sl[c] = c < n ? DotReg(sregs[b * 4 + c]) : DotConst(0.0f); + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + // Built once per row of t. + for (int c = 0; c < 4; c++) + tl[c] = DotReg(tv + c); + ir.Write(IROp::Vec4VfpuDot, dregs[a * 4 + b], sv, tv); + } + } + return; + } + // A very common arrangment. Rearrange to something we can handle. // A 1x1 matrix has no transposed form, the bit then picks a different register. if (n > 1 && IsMatrixTransposed(vd)) { @@ -1737,6 +1892,30 @@ namespace MIPSComp { GetVectorRegs(tregs, sz, _VT); GetVectorRegs(dregs, sz, _VD); + if (UseAccurateVdot()) { + // A dot per row, against t with the lanes past it zero, and lane ins 1 for the + // homogenous form. The results go to the D temp first, since d may overlap s or t. + const int tn = homogenous ? n - 1 : n; + DotLane tl[4]; + for (int i = 0; i < 4; i++) + tl[i] = i < tn ? DotReg(tregs[i]) : DotConst(homogenous && i == ins ? 1.0f : 0.0f); + for (int i = 0; i < n; i++) { + DotLane sl[4]; + for (int k = 0; k < 4; k++) + sl[k] = k < n ? DotReg(sregs[i * 4 + k]) : DotConst(0.0f); + u8 sv, tv; + if (!BuildDotOperands(sl, tl, IRVTEMP_PFX_D, &sv, &tv)) + DISABLE; + // Built once, t is a Vec4 for the rest. + for (int k = 0; k < 4; k++) + tl[k] = DotReg(tv + k); + ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv); + } + for (int i = 0; i < n; i++) + ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i); + return; + } + // SIMD-optimized implementations - if sregs[0..3] is non-consecutive, it's transposed. if (msz == M_4x4 && IsMatrixTransposedVec4(msz, sregs)) { int s0 = IRVTEMP_0; @@ -1871,6 +2050,18 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, _VT); GetVectorRegsPrefixD(dregs, V_Single, _VD); + if (UseAccurateVdot()) { + // The T prefix's swizzle is forced to yx: s0 * t1 + (-s1) * t0. + const DotLane sl[4] = { DotReg(sregs[0]), DotReg(sregs[1], true), DotConst(0.0f), DotConst(0.0f) }; + const DotLane tl[4] = { DotReg(tregs[1]), DotReg(tregs[0]), DotConst(0.0f), DotConst(0.0f) }; + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, _VD); + return; + } + ir.Write(IROp::FMul, IRVTEMP_0, sregs[1], tregs[0]); ir.Write(IROp::FMul, dregs[0], sregs[0], tregs[1]); ir.Write(IROp::FSub, dregs[0], dregs[0], IRVTEMP_0); @@ -2086,6 +2277,49 @@ namespace MIPSComp { GetVectorRegs(tregs, sz, _VT); GetVectorRegs(dregs, sz, _VD); + if (UseAccurateVdot() && (sz == V_Triple || sz == V_Quad)) { + // The dots the interpreter does, against permuted and negated copies of t. The results + // go to the D temp first, since d may overlap s or t. + const DotLane zero = DotConst(0.0f); + auto tl = [&](int i, bool neg = false) { return i < n ? DotReg(tregs[i], neg) : zero; }; + DotLane tDots[4][4]; + int dots; + if (sz == V_Triple) { + const DotLane t0[4] = { zero, tl(2), tl(1, true), zero }; + const DotLane t1[4] = { tl(2, true), zero, tl(0), zero }; + const DotLane t2[4] = { tl(1), tl(0, true), zero, tl(2) }; + memcpy(tDots[0], t0, sizeof(t0)); + memcpy(tDots[1], t1, sizeof(t1)); + memcpy(tDots[2], t2, sizeof(t2)); + dots = 3; + } else { + const DotLane t0[4] = { tl(3), tl(2), tl(1, true), tl(0) }; + const DotLane t1[4] = { tl(2, true), tl(3), tl(0), tl(1) }; + const DotLane t2[4] = { tl(1), tl(0, true), tl(3), tl(2) }; + const DotLane t3[4] = { tl(0, true), tl(1, true), tl(2, true), tl(3) }; + memcpy(tDots[0], t0, sizeof(t0)); + memcpy(tDots[1], t1, sizeof(t1)); + memcpy(tDots[2], t2, sizeof(t2)); + memcpy(tDots[3], t3, sizeof(t3)); + dots = 4; + } + DotLane sl[4]; + for (int i = 0; i < 4; i++) + sl[i] = i < n ? DotReg(sregs[i]) : zero; + for (int i = 0; i < dots; i++) { + u8 sv, tv; + if (!BuildDotOperands(sl, tDots[i], IRVTEMP_PFX_D, &sv, &tv)) + DISABLE; + // Built once, s is a Vec4 for the rest. + for (int j = 0; j < 4; j++) + sl[j] = DotReg(sv + j); + ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv); + } + for (int i = 0; i < dots; i++) + ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i); + return; + } + if (sz == V_Triple) { u8 tempregs[4]{}; for (int i = 0; i < n; ++i) { diff --git a/Core/MIPS/IR/IRFrontend.h b/Core/MIPS/IR/IRFrontend.h index 2a4be17b8d..2cf80fd63c 100644 --- a/Core/MIPS/IR/IRFrontend.h +++ b/Core/MIPS/IR/IRFrontend.h @@ -99,6 +99,13 @@ public: opts = o; } + // One lane of a Vec4VfpuDot operand: a reg, maybe negated, or a constant (reg < 0). + struct DotLane { + int reg; + float constant; + bool negate; + }; + private: void RestoreRoundingMode(bool force = false); void ApplyRoundingMode(bool force = false); @@ -134,6 +141,14 @@ private: void GetVectorRegs(u8 regs[4], VectorSize N, int vectorReg); void GetMatrixRegs(u8 regs[16], MatrixSize N, int matrixReg); + static DotLane DotReg(int reg, bool negate = false) { return DotLane{ reg, 0.0f, negate }; } + static DotLane DotConst(float f) { return DotLane{ -1, f, false }; } + // Puts both operands of a Vec4VfpuDot in Vec4 regs, building them in free temps where they + // aren't one already. avoid is a Vec4 temp that must not be used (or -1). Returns false if + // there aren't enough free temps, and then nothing was written. + bool BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec); + bool UseAccurateVdot() const; + // State JitState js; IRWriter ir; diff --git a/Core/MIPS/IR/IRInst.cpp b/Core/MIPS/IR/IRInst.cpp index 9bf8fee367..2fd123fa8b 100644 --- a/Core/MIPS/IR/IRInst.cpp +++ b/Core/MIPS/IR/IRInst.cpp @@ -158,6 +158,7 @@ static const IRMeta irMeta[] = { { IROp::Vec4Mul, "Vec4Mul", "VVV" }, { IROp::Vec4Scale, "Vec4Scale", "VVF" }, { IROp::Vec4Dot, "Vec4Dot", "FVV" }, + { IROp::Vec4VfpuDot, "Vec4VfpuDot", "FVV" }, { IROp::Vec4Neg, "Vec4Neg", "VV" }, { IROp::Vec4Abs, "Vec4Abs", "VV" }, diff --git a/Core/MIPS/IR/IRInst.h b/Core/MIPS/IR/IRInst.h index 0f746246ae..32962e2ae1 100644 --- a/Core/MIPS/IR/IRInst.h +++ b/Core/MIPS/IR/IRInst.h @@ -187,6 +187,7 @@ enum class IROp : uint8_t { Vec4Div, Vec4Scale, Vec4Dot, + Vec4VfpuDot, // The VFPU's own dot product, bit-exact (vfpu_dot). Vec4Neg, Vec4Abs, diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp index 6d67305c97..fa514dd566 100644 --- a/Core/MIPS/IR/IRInterpreter.cpp +++ b/Core/MIPS/IR/IRInterpreter.cpp @@ -171,6 +171,7 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { dispatch[(int)IROp::FCmpVfpuAggregate] = &&L_FCmpVfpuAggregate; dispatch[(int)IROp::FCmovVfpuCC] = &&L_FCmovVfpuCC; dispatch[(int)IROp::Vec4Dot] = &&L_Vec4Dot; + dispatch[(int)IROp::Vec4VfpuDot] = &&L_Vec4VfpuDot; dispatch[(int)IROp::FSin] = &&L_FSin; dispatch[(int)IROp::FCos] = &&L_FCos; dispatch[(int)IROp::FRSqrt] = &&L_FRSqrt; @@ -798,6 +799,10 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { IR_NEXT; } + IR_CASE(Vec4VfpuDot) + mips->f[inst->dest] = vfpu_dot(&mips->f[inst->src1], &mips->f[inst->src2]); + IR_NEXT; + IR_CASE(FSin) mips->f[inst->dest] = vfpu_sin(mips->f[inst->src1]); IR_NEXT; diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index 7df5abf18a..b88360bd59 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -424,6 +424,11 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { CompIR_VecHoriz(inst); break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32: case IROp::Vec4Unpack8To32: diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index 1961892748..699cc729ad 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -847,6 +847,7 @@ bool PropagateConstants(const IRWriter &in, IRWriter &out, const IROptions &opts case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: case IROp::Vec4Scale: case IROp::Vec4Shuffle: case IROp::Vec4Blend: @@ -2185,6 +2186,7 @@ bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts) { break; case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: if (overlapped(inst.dest, 1, inst.src1, 4, inst.src2, 4) && findAvailTempVec4()) { out.Write(inst.op, temp, inst.src1, inst.src2, inst.constant); if (usedLaterAsVec4(inst.dest & ~3)) { From dc417ca7cc52e29e08675f6f26041d688d148ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 13:08:47 +0200 Subject: [PATCH 04/17] CPUEquivalence: Add PPSSPP_CPUFUZZ_ACCURATE_VDOT Co-Authored-By: Claude Opus 5.5 --- docs/cpu-equivalence.md | 1 + unittest/JitHarness.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/cpu-equivalence.md b/docs/cpu-equivalence.md index a6e6e5ecab..7c8a73b386 100644 --- a/docs/cpu-equivalence.md +++ b/docs/cpu-equivalence.md @@ -33,6 +33,7 @@ Environment variables: | `PPSSPP_CPUFUZZ_DUMP` | write every core's end state of every case to a file | | `PPSSPP_CPUFUZZ_STRICT` | fail on any difference, not only on a crash | | `PPSSPP_CPUFUZZ_OPS` | comma-separated instruction name prefixes to generate only; `dot` is the dot products and the prefix ops | +| `PPSSPP_CPUFUZZ_ACCURATE_VDOT` | run with the accurate dot products | For quick rounds while working on some instructions, fuzz just those, on every core of the machine: `PPSSPP_CPUFUZZ_OPS=dot Tools/cpu_fuzz_parallel.py build/PPSSPPUnitTest --count 2000` splits the seeds diff --git a/unittest/JitHarness.cpp b/unittest/JitHarness.cpp index 292d6f82ea..1c39d7d668 100644 --- a/unittest/JitHarness.cpp +++ b/unittest/JitHarness.cpp @@ -398,6 +398,7 @@ bool TestJitReplacementHooks() { // PPSSPP_CPUFUZZ_SKIP cores not to run, by digit: 1 IR interpreter, 2 JIT, 3 JIT IR. // PPSSPP_CPUFUZZ_REPORTS how many differences to minimize and print (default 20). // PPSSPP_CPUFUZZ_STRICT fail the test on any difference, not only on a crash. +// PPSSPP_CPUFUZZ_ACCURATE_VDOT run with AccurateVdot, the VFPU's own dot products. // PPSSPP_CPUFUZZ_OPS comma-separated instruction name prefixes to generate only, for fast // runs on the instructions being worked on. "dot" is the dot products // and the prefixes. Without PPSSPP_CPUFUZZ_CLASSES, the class is "v". @@ -899,6 +900,8 @@ bool TestCPUEquivalence() { // terminating syscall doesn't. const bool oldSkipDeadbeef = g_Config.bSkipDeadbeefFilling; g_Config.bSkipDeadbeefFilling = true; + const bool oldAccurateVdot = g_Config.bAccurateVdot; + g_Config.bAccurateVdot = getenv("PPSSPP_CPUFUZZ_ACCURATE_VDOT") != nullptr; const char *countStr = getenv("PPSSPP_CPUFUZZ_COUNT"); const char *seedStr = getenv("PPSSPP_CPUFUZZ_SEED"); @@ -1049,6 +1052,7 @@ bool TestCPUEquivalence() { mipsr4k.UpdateCore(CPUCore::INTERPRETER); DestroyJitHarness(); g_Config.bSkipDeadbeefFilling = oldSkipDeadbeef; + g_Config.bAccurateVdot = oldAccurateVdot; return success; } From 8ac395c566bff42baa78587a942182f2ef67c9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 13:37:00 +0200 Subject: [PATCH 05/17] ARM64 IR JIT: Run Vec4VfpuDot through a shared routine GenerateArm64VfpuDot() generates vfpu_dot() as integer code, once per JIT, to be called with BL. The operands and result go through mips->vfpuDotArgs and vfpuDotResult, and the routine preserves every register but x30, so a call site is two stores, the BL and a load, with nothing flushed. Inf and NaN operands go to vfpu_dot_reference(), with the caller-saved registers saved. Co-Authored-By: Claude Opus 5.5 --- Core/CMakeLists.txt | 1 + Core/Core.vcxproj | 1 + Core/Core.vcxproj.filters | 3 + Core/MIPS/ARM/ArmIRCompVec.cpp | 5 + Core/MIPS/ARM64/Arm64IRAsm.cpp | 3 + Core/MIPS/ARM64/Arm64IRCompVec.cpp | 20 +++ Core/MIPS/ARM64/Arm64IRJit.h | 2 + Core/MIPS/ARM64/Arm64VfpuDot.h | 168 +++++++++++++++++++ Core/MIPS/IR/IRNativeCommon.cpp | 5 +- Core/MIPS/LoongArch64/LoongArch64CompVec.cpp | 5 + Core/MIPS/MIPS.h | 4 + Core/MIPS/RiscV/RiscVCompVec.cpp | 5 + Core/MIPS/x86/X64IRCompVec.cpp | 5 + UWP/CoreUWP/CoreUWP.vcxproj | 1 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 1 + 15 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 Core/MIPS/ARM64/Arm64VfpuDot.h diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 4e691f1a7b..4a94c71484 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -50,6 +50,7 @@ list(APPEND CoreExtra MIPS/ARM64/Arm64IRJit.h MIPS/ARM64/Arm64IRRegCache.cpp MIPS/ARM64/Arm64IRRegCache.h + MIPS/ARM64/Arm64VfpuDot.h Util/DisArm64.cpp ) diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 163be032c6..fed2ddf89d 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -870,6 +870,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 9e17d9263a..b2b2f5d5cc 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -2166,6 +2166,9 @@ MIPS\ARM64 + + MIPS\ARM64 + Util diff --git a/Core/MIPS/ARM/ArmIRCompVec.cpp b/Core/MIPS/ARM/ArmIRCompVec.cpp index fe98a56139..edb5be51e9 100644 --- a/Core/MIPS/ARM/ArmIRCompVec.cpp +++ b/Core/MIPS/ARM/ArmIRCompVec.cpp @@ -170,6 +170,11 @@ void ArmJitBackend::CompIR_VecHoriz(IRInst inst) { VMOV(regs_.F(inst.dest), SCRATCHF1); break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/ARM64/Arm64IRAsm.cpp b/Core/MIPS/ARM64/Arm64IRAsm.cpp index 07296dbe11..02d75b9eb4 100644 --- a/Core/MIPS/ARM64/Arm64IRAsm.cpp +++ b/Core/MIPS/ARM64/Arm64IRAsm.cpp @@ -22,6 +22,7 @@ #include "Common/Log.h" #include "Core/CoreTiming.h" #include "Core/MemMap.h" +#include "Core/MIPS/ARM64/Arm64VfpuDot.h" #include "Core/MIPS/ARM64/Arm64IRJit.h" #include "Core/MIPS/ARM64/Arm64IRRegCache.h" #include "Core/MIPS/JitCommon/JitCommon.h" @@ -264,6 +265,8 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) { RET(); } + vfpuDot_ = GenerateArm64VfpuDot(this, &fp_, CTXREG); + // Leave this at the end, add more stuff above. if (enableDisasm) { std::vector lines = DisassembleArm64(disasmStart, (int)(GetCodePtr() - disasmStart)); diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 45924a088e..634f54c889 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -599,6 +599,26 @@ void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + // Through memory and the shared routine, which preserves every register but x30. + regs_.SpillLockFPR(inst.src1, inst.src2); + regs_.MapVec4(inst.src1); + regs_.MapVec4(inst.src2); + fp_.STR(128, INDEX_UNSIGNED, regs_.FQ(inst.src1), CTXREG, offsetof(MIPSState, vfpuDotArgs)); + fp_.STR(128, INDEX_UNSIGNED, regs_.FQ(inst.src2), CTXREG, offsetof(MIPSState, vfpuDotArgs) + 16); + BL(vfpuDot_); + fp_.LDR(32, INDEX_UNSIGNED, SCRATCHF1, CTXREG, offsetof(MIPSState, vfpuDotResult)); + if (Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { + // The dest lane is in a mapped Vec4. + regs_.MapVec4(inst.dest & ~3, MIPSMap::DIRTY); + fp_.INS(32, regs_.FQ(inst.dest & ~3), inst.dest & 3, EncodeRegToQuad(SCRATCHF1), 0); + } else { + regs_.MapFPR(inst.dest, MIPSMap::NOINIT); + fp_.FMOV(regs_.F(inst.dest), SCRATCHF1); + } + regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index d6efefd516..665838cce7 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -145,6 +145,8 @@ private: // Indexed by FPCR FZ:RN bits for convenience. Uses SCRATCH2. const u8 *convertS0ToSCRATCH1_[8]; + // See GenerateArm64VfpuDot(). + const u8 *vfpuDot_ = nullptr; // Note: mutable state used at runtime. const u8 *currentRoundingFunc_ = nullptr; diff --git a/Core/MIPS/ARM64/Arm64VfpuDot.h b/Core/MIPS/ARM64/Arm64VfpuDot.h new file mode 100644 index 0000000000..49f3cdba48 --- /dev/null +++ b/Core/MIPS/ARM64/Arm64VfpuDot.h @@ -0,0 +1,168 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/Arm64Emitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// Generates the VFPU dot product routine the ARM64 JITs call with BL: vfpu_dot() of the two +// Vec4s in mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. +// Every register is preserved except x30 and the flags. It's integer code, the algorithm of +// vfpu_dot_reference(), and leaves inf and NaN inputs to it. +inline const u8 *GenerateArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) { + using namespace Arm64Gen; + const s32 args = (s32)offsetof(MIPSState, vfpuDotArgs); + const s32 result = (s32)offsetof(MIPSState, vfpuDotResult); + + const u8 *start = e->AlignCode16(); + e->STP(INDEX_PRE, X0, X1, SP, -16); + e->STP(INDEX_PRE, X2, X3, SP, -16); + e->STP(INDEX_PRE, X4, X5, SP, -16); + e->STP(INDEX_PRE, X6, X7, SP, -16); + e->STP(INDEX_PRE, X8, X9, SP, -16); + e->STP(INDEX_PRE, X10, X11, SP, -16); + e->STP(INDEX_PRE, X30, ZR, SP, -16); + + // W0/W1: the lane's operands, W2/W3: their exponents, W4: the largest exponent sum of a + // normal product, W5: the sum. + // First pass: the largest exponent sum, and inf or NaN anywhere sends it to the slow path. + std::vector special; + e->MOVI2R(W4, 0); + for (int i = 0; i < 4; i++) { + e->LDR(INDEX_UNSIGNED, W0, ctx, args + i * 4); + e->LDR(INDEX_UNSIGNED, W1, ctx, args + 16 + i * 4); + e->UBFX(W2, W0, 23, 8); + e->UBFX(W3, W1, 23, 8); + e->CMPI2R(W2, 255); + special.push_back(e->B(CC_EQ)); + e->CMPI2R(W3, 255); + special.push_back(e->B(CC_EQ)); + // A zero or denormal operand makes the product zero, which doesn't count. + e->ADD(W7, W2, W3); + e->CMPI2R(W2, 0); + e->CCMP(W3, 0, 4, CC_NEQ); + e->CSEL(W7, W7, WZR, CC_NEQ); + e->CMP(W4, W7); + e->CSEL(W4, W4, W7, CC_HS); + } + + // Second pass: the 24x24-bit products, kept to 2 extra bits with round-to-odd, aligned to the + // largest by truncation, signed, and summed exactly. + e->MOVI2R(W5, 0); + for (int i = 0; i < 4; i++) { + e->LDR(INDEX_UNSIGNED, W0, ctx, args + i * 4); + e->LDR(INDEX_UNSIGNED, W1, ctx, args + 16 + i * 4); + e->UBFX(W2, W0, 23, 8); + e->UBFX(W3, W1, 23, 8); + e->ANDI2R(W9, W0, 0x007FFFFF); + e->ORRI2R(W9, W9, 0x00800000); + e->ANDI2R(W10, W1, 0x007FFFFF); + e->ORRI2R(W10, W10, 0x00800000); + e->UMULL(X6, W9, W10); + e->LSR(X7, X6, 21); + e->TSTI2R(X6, (1 << 21) - 1); + e->CSET(W8, CC_NEQ); + e->ORR(W7, W7, W8); + // Shift right by the exponent difference, at most 28. + e->ADD(W11, W2, W3); + e->SUB(W11, W4, W11); + e->MOVI2R(W8, 28); + e->CMP(W11, W8); + e->CSEL(W11, W11, W8, CC_LS); + e->LSRV(W7, W7, W11); + e->CMPI2R(W2, 0); + e->CCMP(W3, 0, 4, CC_NEQ); + e->CSEL(W7, W7, WZR, CC_NEQ); + e->EOR(W8, W0, W1); + e->TSTI2R(W8, 0x80000000); + e->CSNEG(W7, W7, W7, CC_EQ); + e->ADD(W5, W5, W7); + } + + // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. + e->ANDI2R(W8, W5, 0x80000000); + e->CMPI2R(W5, 0); + e->CSNEG(W9, W5, W5, CC_GE); + e->LSR(W9, W9, 2); + e->ORRI2R(W10, W9, 1); + e->CLZ(W10, W10); + e->ADDI2R(W10, W10, 32); + e->SUBI2R(W11, W10, 1); + e->LSLV(X6, X9, X11); + e->UBFX(X7, X6, 39, 1); + e->ADD(X6, X6, X7); + e->MOVI2R(X7, (1ULL << 38) - 1); + e->ADD(X6, X6, X7); + e->LSR(X6, X6, 39); + e->SUB(W7, W4, W10); + e->SUBI2R(W7, W7, 88); + e->SXTW(X7, W7); + e->LSL(X7, X7, 23); + e->ADD(X7, X7, X6); + e->MOVI2R(X6, 0x7F800000); + e->CMP(X7, X6); + e->CSEL(X7, X6, X7, CC_GE); + e->MOVI2R(X6, 0x00800000); + e->CMP(X7, X6); + e->CSEL(X7, ZR, X7, CC_LT); + e->CMPI2R(W9, 0); + e->CSEL(W7, WZR, W7, CC_EQ); + e->ORR(W7, W7, W8); + e->STR(INDEX_UNSIGNED, W7, ctx, result); + + const u8 *epilogue = e->GetCodePointer(); + e->LDP(INDEX_POST, X30, ZR, SP, 16); + e->LDP(INDEX_POST, X10, X11, SP, 16); + e->LDP(INDEX_POST, X8, X9, SP, 16); + e->LDP(INDEX_POST, X6, X7, SP, 16); + e->LDP(INDEX_POST, X4, X5, SP, 16); + e->LDP(INDEX_POST, X2, X3, SP, 16); + e->LDP(INDEX_POST, X0, X1, SP, 16); + e->RET(); + + // Inf or NaN: the C version, with everything else a call may clobber saved too. + for (const FixupBranch &b : special) + e->SetJumpTarget(b); + e->STP(INDEX_PRE, X12, X13, SP, -16); + e->STP(INDEX_PRE, X14, X15, SP, -16); + e->STP(INDEX_PRE, X16, X17, SP, -16); + e->STP(INDEX_PRE, X18, X19, SP, -16); + for (int i = 0; i < 32; i += 2) + fp->STP(128, INDEX_PRE, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, -32); + e->ADDI2R(X0, ctx, args); + e->ADDI2R(X1, ctx, args + 16); + e->MOVP2R(X16, &vfpu_dot_reference); + e->BLR(X16); + fp->STR(32, INDEX_UNSIGNED, S0, ctx, result); + for (int i = 30; i >= 0; i -= 2) + fp->LDP(128, INDEX_POST, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, 32); + e->LDP(INDEX_POST, X18, X19, SP, 16); + e->LDP(INDEX_POST, X16, X17, SP, 16); + e->LDP(INDEX_POST, X14, X15, SP, 16); + e->LDP(INDEX_POST, X12, X13, SP, 16); + e->B(epilogue); + return start; +} + +} // namespace MIPSComp diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index b88360bd59..484ed8a313 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -421,13 +421,10 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { break; case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: CompIR_VecHoriz(inst); break; - case IROp::Vec4VfpuDot: - // Through the IR interpreter for now. - CompIR_Generic(inst); - break; case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32: diff --git a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp index ab940a082d..a12804157f 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp @@ -389,6 +389,11 @@ void LoongArch64JitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/MIPS.h b/Core/MIPS/MIPS.h index 9d5e79ef9b..fa38faa157 100644 --- a/Core/MIPS/MIPS.h +++ b/Core/MIPS/MIPS.h @@ -248,6 +248,10 @@ public: float sincostemp[2]; + // The operands (s, then t) and result of the JITs' VFPU dot product routine (vfpu_dot()). + alignas(16) u32 vfpuDotArgs[8]; + u32 vfpuDotResult; + static const u32 FCR0_VALUE = 0x00003351; #if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/RiscV/RiscVCompVec.cpp b/Core/MIPS/RiscV/RiscVCompVec.cpp index c2c00e5589..c68b0877ef 100644 --- a/Core/MIPS/RiscV/RiscVCompVec.cpp +++ b/Core/MIPS/RiscV/RiscVCompVec.cpp @@ -312,6 +312,11 @@ void RiscVJitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index aea78af280..1333b13dab 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -274,6 +274,11 @@ void X64JitBackend::CompIR_VecHoriz(IRInst inst) { break; } + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index cdb0b5f19b..cd6788c90f 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -278,6 +278,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index 6c900373eb..142cbb50eb 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -597,6 +597,7 @@ + From fa25caf9050c0ca3018e992e58d153b4aeb1aef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 13:43:37 +0200 Subject: [PATCH 06/17] x86-64 IR JIT: Run Vec4VfpuDot through a shared routine GenerateX64VfpuDot() is the x86-64 counterpart of GenerateArm64VfpuDot(): the same integer version of vfpu_dot(), preserving every register but the flags, with inf and NaN operands going to vfpu_dot_reference(). A dest lane inside a source Vec4 still takes the generic fallback, like Vec4Dot does. Co-Authored-By: Claude Opus 5.5 --- Core/CMakeLists.txt | 1 + Core/Core.vcxproj | 1 + Core/Core.vcxproj.filters | 3 + Core/MIPS/x86/X64IRAsm.cpp | 4 + Core/MIPS/x86/X64IRCompVec.cpp | 12 +- Core/MIPS/x86/X64IRJit.h | 2 + Core/MIPS/x86/X64VfpuDot.h | 197 ++++++++++++++++++++++++++++ UWP/CoreUWP/CoreUWP.vcxproj | 1 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 1 + 9 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 Core/MIPS/x86/X64VfpuDot.h diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 4a94c71484..8a8a78352b 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -66,6 +66,7 @@ list(APPEND CoreExtra MIPS/x86/X64IRJit.h MIPS/x86/X64IRRegCache.cpp MIPS/x86/X64IRRegCache.h + MIPS/x86/X64VfpuDot.h ) list(APPEND CoreExtra diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index fed2ddf89d..499dbdac51 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -1064,6 +1064,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index b2b2f5d5cc..24a84a5cfd 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -2157,6 +2157,9 @@ MIPS\x86 + + MIPS\x86 + MIPS\x86 diff --git a/Core/MIPS/x86/X64IRAsm.cpp b/Core/MIPS/x86/X64IRAsm.cpp index eb26a06c03..2acb6a23ea 100644 --- a/Core/MIPS/x86/X64IRAsm.cpp +++ b/Core/MIPS/x86/X64IRAsm.cpp @@ -23,6 +23,7 @@ #include "Core/MemMap.h" #include "Core/MIPS/x86/X64IRJit.h" #include "Core/MIPS/x86/X64IRRegCache.h" +#include "Core/MIPS/x86/X64VfpuDot.h" #include "Core/MIPS/JitCommon/JitCommon.h" #include "Core/MIPS/JitCommon/JitState.h" #include "Core/Core.h" @@ -295,6 +296,9 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) { } JMP(quitLoop, true); +#if PPSSPP_ARCH(AMD64) + vfpuDot_ = GenerateX64VfpuDot(this, CTXREG); +#endif // Leave this at the end, add more stuff above. if (enableDisasm) { diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index 1333b13dab..c55ac7781f 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -275,8 +275,16 @@ void X64JitBackend::CompIR_VecHoriz(IRInst inst) { } case IROp::Vec4VfpuDot: - // Through the IR interpreter for now. - CompIR_Generic(inst); + // Through memory and the shared routine, which preserves every register. + if (!vfpuDot_ || Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { + CompIR_Generic(inst); + break; + } + regs_.Map(inst); + MOVUPS(MDisp(CTXREG, offsetof(MIPSState, vfpuDotArgs)), regs_.FX(inst.src1)); + MOVUPS(MDisp(CTXREG, offsetof(MIPSState, vfpuDotArgs) + 16), regs_.FX(inst.src2)); + CALL(vfpuDot_); + MOVSS(regs_.FX(inst.dest), MDisp(CTXREG, offsetof(MIPSState, vfpuDotResult))); break; default: diff --git a/Core/MIPS/x86/X64IRJit.h b/Core/MIPS/x86/X64IRJit.h index edd86594e0..11733dcdeb 100644 --- a/Core/MIPS/x86/X64IRJit.h +++ b/Core/MIPS/x86/X64IRJit.h @@ -141,6 +141,8 @@ private: const u8 *saveStaticRegisters_ = nullptr; const u8 *loadStaticRegisters_ = nullptr; + // See GenerateX64VfpuDot(). + const u8 *vfpuDot_ = nullptr; typedef struct { float f[4]; } Float4Constant; struct Constants { diff --git a/Core/MIPS/x86/X64VfpuDot.h b/Core/MIPS/x86/X64VfpuDot.h new file mode 100644 index 0000000000..e883a20f1d --- /dev/null +++ b/Core/MIPS/x86/X64VfpuDot.h @@ -0,0 +1,197 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include "ppsspp_config.h" + +#if PPSSPP_ARCH(AMD64) + +#include +#include + +#include "Common/x64Emitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// Generates the VFPU dot product routine the x86-64 JITs CALL: vfpu_dot() of the two Vec4s in +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. Every +// register is preserved except the flags. It's integer code, the algorithm of +// vfpu_dot_reference(), and leaves inf and NaN inputs to it. +inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { + using namespace Gen; + const int args = (int)offsetof(MIPSState, vfpuDotArgs); + const int result = (int)offsetof(MIPSState, vfpuDotResult); + static const X64Reg saved[] = { RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11 }; + + const u8 *start = e->AlignCode16(); + for (X64Reg r : saved) + e->PUSH(r); + + // ESI/EDI: the lane's operands, R8D/R9D: their exponents, R10D: the largest exponent sum of a + // normal product, R11D: the sum. + auto loadLane = [&](int i) { + e->MOV(32, R(ESI), MDisp(ctx, args + i * 4)); + e->MOV(32, R(EDI), MDisp(ctx, args + 16 + i * 4)); + e->MOV(32, R(R8), R(ESI)); + e->SHR(32, R(R8), Imm8(23)); + e->AND(32, R(R8), Imm32(255)); + e->MOV(32, R(R9), R(EDI)); + e->SHR(32, R(R9), Imm8(23)); + e->AND(32, R(R9), Imm32(255)); + }; + // A zero or denormal operand makes the product zero, which doesn't count. + auto zeroUnlessNormal = [&](X64Reg r, X64Reg zero) { + e->TEST(32, R(R8), R(R8)); + e->CMOVcc(32, r, R(zero), CC_Z); + e->TEST(32, R(R9), R(R9)); + e->CMOVcc(32, r, R(zero), CC_Z); + }; + + // First pass: the largest exponent sum, and inf or NaN anywhere sends it to the slow path. + std::vector special; + e->XOR(32, R(R10), R(R10)); + for (int i = 0; i < 4; i++) { + loadLane(i); + e->CMP(32, R(R8), Imm32(255)); + special.push_back(e->J_CC(CC_E, true)); + e->CMP(32, R(R9), Imm32(255)); + special.push_back(e->J_CC(CC_E, true)); + e->LEA(32, EAX, MRegSum(R8, R9)); + e->XOR(32, R(EDX), R(EDX)); + zeroUnlessNormal(EAX, EDX); + e->CMP(32, R(R10), R(EAX)); + e->CMOVcc(32, R10, R(EAX), CC_B); + } + + // Second pass: the 24x24-bit products, kept to 2 extra bits with round-to-odd, aligned to the + // largest by truncation, signed, and summed exactly. + e->XOR(32, R(R11), R(R11)); + for (int i = 0; i < 4; i++) { + loadLane(i); + e->MOV(32, R(EAX), R(ESI)); + e->AND(32, R(EAX), Imm32(0x007FFFFF)); + e->OR(32, R(EAX), Imm32(0x00800000)); + e->MOV(32, R(EDX), R(EDI)); + e->AND(32, R(EDX), Imm32(0x007FFFFF)); + e->OR(32, R(EDX), Imm32(0x00800000)); + e->IMUL(64, RAX, R(RDX)); + e->MOV(64, R(RDX), R(RAX)); + e->SHR(64, R(RDX), Imm8(21)); + e->TEST(32, R(EAX), Imm32((1 << 21) - 1)); + e->SETcc(CC_NZ, R(EAX)); + e->MOVZX(32, 8, EAX, R(EAX)); + e->OR(32, R(EDX), R(EAX)); + // Shift right by the exponent difference, at most 28. + e->LEA(32, ECX, MRegSum(R8, R9)); + e->MOV(32, R(EAX), R(R10)); + e->SUB(32, R(EAX), R(ECX)); + e->MOV(32, R(ECX), Imm32(28)); + e->CMP(32, R(EAX), R(ECX)); + e->CMOVcc(32, ECX, R(EAX), CC_B); + e->SHR(32, R(EDX), R(CL)); + e->XOR(32, R(EAX), R(EAX)); + zeroUnlessNormal(EDX, EAX); + e->MOV(32, R(EAX), R(ESI)); + e->XOR(32, R(EAX), R(EDI)); + e->SAR(32, R(EAX), Imm8(31)); + e->XOR(32, R(EDX), R(EAX)); + e->SUB(32, R(EDX), R(EAX)); + e->ADD(32, R(R11), R(EDX)); + } + + // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. + // ESI: the sign, EDI: the significand m, EDX: its leading zeros as 64 bits. + e->MOV(32, R(ESI), R(R11)); + e->AND(32, R(ESI), Imm32(0x80000000)); + e->MOV(32, R(EAX), R(R11)); + e->CDQ(); + e->XOR(32, R(EAX), R(EDX)); + e->SUB(32, R(EAX), R(EDX)); + e->SHR(32, R(EAX), Imm8(2)); + e->MOV(32, R(EDI), R(EAX)); + e->MOV(32, R(ECX), R(EAX)); + e->OR(32, R(ECX), Imm32(1)); + e->BSR(32, ECX, R(ECX)); + e->MOV(32, R(EDX), Imm32(63)); + e->SUB(32, R(EDX), R(ECX)); + e->LEA(32, ECX, MDisp(EDX, -1)); + e->MOV(32, R(EAX), R(EDI)); + e->SHL(64, R(RAX), R(CL)); + e->MOV(64, R(R8), R(RAX)); + e->SHR(64, R(R8), Imm8(39)); + e->AND(32, R(R8), Imm32(1)); + e->ADD(64, R(RAX), R(R8)); + e->MOV(64, R(R8), Imm64((1ULL << 38) - 1)); + e->ADD(64, R(RAX), R(R8)); + e->SHR(64, R(RAX), Imm8(39)); + e->MOV(32, R(R8), R(R10)); + e->SUB(32, R(R8), R(EDX)); + e->SUB(32, R(R8), Imm32(88)); + e->MOVSX(64, 32, R8, R(R8)); + e->SHL(64, R(R8), Imm8(23)); + e->ADD(64, R(R8), R(RAX)); + e->MOV(64, R(RAX), Imm64(0x7F800000)); + e->CMP(64, R(R8), R(RAX)); + e->CMOVcc(64, R8, R(RAX), CC_GE); + e->MOV(64, R(RAX), Imm64(0x00800000)); + e->XOR(32, R(ECX), R(ECX)); + e->CMP(64, R(R8), R(RAX)); + e->CMOVcc(64, R8, R(RCX), CC_L); + e->TEST(32, R(EDI), R(EDI)); + e->CMOVcc(64, R8, R(RCX), CC_Z); + e->OR(32, R(R8), R(ESI)); + e->MOV(32, MDisp(ctx, result), R(R8)); + + const u8 *epilogue = e->GetCodePointer(); + for (int i = (int)(sizeof(saved) / sizeof(saved[0])) - 1; i >= 0; i--) + e->POP(saved[i]); + e->RET(); + + // Inf or NaN: the C version, with the vector registers saved too, on an aligned stack. + for (const FixupBranch &b : special) + e->SetJumpTarget(b); + // [RSP]: the shadow space Windows wants, then the 16 XMMs, then the unaligned RSP. + const int xmmOffset = 32, savedRsp = 32 + 16 * 16, frame = savedRsp + 16; + e->MOV(64, R(RAX), R(RSP)); + e->AND(64, R(RSP), Imm32(-16)); + e->SUB(64, R(RSP), Imm32(frame)); + e->MOV(64, MDisp(RSP, savedRsp), R(RAX)); + for (int i = 0; i < 16; i++) + e->MOVUPS(MDisp(RSP, xmmOffset + i * 16), (X64Reg)(XMM0 + i)); +#ifdef _WIN32 + e->LEA(64, RCX, MDisp(ctx, args)); + e->LEA(64, RDX, MDisp(ctx, args + 16)); +#else + e->LEA(64, RDI, MDisp(ctx, args)); + e->LEA(64, RSI, MDisp(ctx, args + 16)); +#endif + e->MOV(64, R(RAX), ImmPtr((const void *)&vfpu_dot_reference)); + e->CALLptr(R(RAX)); + e->MOVSS(MDisp(ctx, result), XMM0); + for (int i = 0; i < 16; i++) + e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, xmmOffset + i * 16)); + e->MOV(64, R(RSP), MDisp(RSP, savedRsp)); + e->JMP(epilogue, true); + return start; +} + +} // namespace MIPSComp + +#endif diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index cd6788c90f..82f9cd5f09 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -308,6 +308,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index 142cbb50eb..dffad90dd9 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -627,6 +627,7 @@ + From bebe527e48323fd12fd04cf069e1092c52ea6dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 14:17:54 +0200 Subject: [PATCH 07/17] ARM64 emitter: Add vector ADD/SUB, UMAXP, USHL, UMULL/UMULL2, ADDV and USRA The disassembler now decodes the across-lanes and three-different groups, which it only printed as raw words. The encodings match clang's. Co-Authored-By: Claude Opus 5.5 --- Common/Arm64Emitter.cpp | 37 ++++++++++++++++++++ Common/Arm64Emitter.h | 13 +++++++ Core/Util/DisArm64.cpp | 67 ++++++++++++++++++++++++++++++++++-- unittest/TestArm64Disasm.cpp | 16 +++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/Common/Arm64Emitter.cpp b/Common/Arm64Emitter.cpp index fbab83833e..1a3f688600 100644 --- a/Common/Arm64Emitter.cpp +++ b/Common/Arm64Emitter.cpp @@ -3196,6 +3196,37 @@ void ARM64FloatEmitter::UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(1, EncodeSize(size), 0xC, Rd, Rn, Rm); } +void ARM64FloatEmitter::ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(0, EncodeSize(size), 0x10, Rd, Rn, Rm); +} +void ARM64FloatEmitter::SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x10, Rd, Rn, Rm); +} +void ARM64FloatEmitter::UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x14, Rd, Rn, Rm); +} +void ARM64FloatEmitter::USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x8, Rd, Rn, Rm); +} +void ARM64FloatEmitter::UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + _assert_msg_(IsQuad(Rd) && !IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__); + Write32((1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} +void ARM64FloatEmitter::UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + _assert_msg_(IsQuad(Rd) && IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__); + Write32((1 << 30) | (1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} +void ARM64FloatEmitter::ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn) +{ + _assert_msg_(size < 64 && (size < 32 || IsQuad(Rn)), "%s: bad size", __FUNCTION__); + Write32((IsQuad(Rn) << 30) | (0xE << 24) | (EncodeSize(size) << 22) | (0x18 << 17) | (0x1B << 12) | (2 << 10) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} void ARM64FloatEmitter::SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(0, EncodeSize(size), 0xD, Rd, Rn, Rm); @@ -3846,6 +3877,12 @@ void ARM64FloatEmitter::SHL(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { EmitShiftImm(IsQuad(Rd), false, imm >> 3, imm & 7, 0xA, Rd, Rn); } +void ARM64FloatEmitter::USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { + _assert_msg_(shift > 0 && shift <= dest_size, "%s shift amount out of range!", __FUNCTION__); + u32 imm = EncodeImmShiftRight(dest_size, shift); + EmitShiftImm(IsQuad(Rd), true, imm >> 3, imm & 7, 0x2, Rd, Rn); +} + void ARM64FloatEmitter::USHR(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { _assert_msg_(shift < dest_size, "%s shift amount must less than the element size!", __FUNCTION__); u32 imm = EncodeImmShiftRight(dest_size, shift); diff --git a/Common/Arm64Emitter.h b/Common/Arm64Emitter.h index 19fef73745..8edc282a4b 100644 --- a/Common/Arm64Emitter.h +++ b/Common/Arm64Emitter.h @@ -888,6 +888,17 @@ public: void UMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Integer vector add and subtract. + void ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Widening multiplies: Rd (a quad) gets the products of the lower (UMULL) or upper (UMULL2) + // halves of Rn and Rm, src_size bits each. + void UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Sum of all lanes, into a scalar of the lane size. + void ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn); void SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void SMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); @@ -995,6 +1006,8 @@ public: void SHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SHLL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SHRN(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); + // Rd += Rn >> shift, per lane. + void USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); void SHRN2(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); void SXTL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SXTL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); diff --git a/Core/Util/DisArm64.cpp b/Core/Util/DisArm64.cpp index 6e44ae7646..35bc5e07a6 100644 --- a/Core/Util/DisArm64.cpp +++ b/Core/Util/DisArm64.cpp @@ -1357,6 +1357,69 @@ static void SIMDVectorIndexedElement(uint32_t w, Instruction *instr) { Out(instr, "%s %c%d, %c%d, %c%d.%s[%d]", opname, r, Rd, r, Rn, r, Rm, GetArrangement(Q, size & 1), index); } +// Reductions across the lanes of a vector into a scalar. +static void SIMDAcrossLanes(uint32_t w, Instruction *instr) { + int Rd = w & 0x1f; + int Rn = (w >> 5) & 0x1f; + int opcode = (w >> 12) & 0x1F; + int size = (w >> 22) & 3; + bool Q = GetQ(w); + bool U = GetU(w); + char r = Q ? 'q' : 'd'; + if (U && (opcode == 0x0C || opcode == 0x0F)) { + // fmaxnmv, fmaxv, fminnmv, fminv: single precision only. + const char *name = opcode == 0x0C ? ((size & 2) ? "fminnmv" : "fmaxnmv") : ((size & 2) ? "fminv" : "fmaxv"); + Out(instr, "%s s%d, %c%d.4s", name, Rd, r, Rn); + return; + } + const char *name = nullptr; + bool longResult = false; + switch (opcode) { + case 0x03: name = U ? "uaddlv" : "saddlv"; longResult = true; break; + case 0x0A: name = U ? "umaxv" : "smaxv"; break; + case 0x1A: name = U ? "uminv" : "sminv"; break; + case 0x1B: name = U ? nullptr : "addv"; break; + } + if (!name || size == 3) { + Out(instr, "(asimd across lanes %08x)", w); + return; + } + static const char scalarRegs[] = "bhsd"; + Out(instr, "%s.%d %c%d, %c%d", name, 8 << size, scalarRegs[size + (longResult ? 1 : 0)], Rd, r, Rn); +} + +// Long (L), wide (W) and narrowing (HN) ops on lanes of different sizes. +static void SIMDThreeDifferent(uint32_t w, Instruction *instr) { + int Rd = w & 0x1f; + int Rn = (w >> 5) & 0x1f; + int Rm = (w >> 16) & 0x1f; + int opcode = (w >> 12) & 0xF; + int size = (w >> 22) & 3; + bool Q = GetQ(w); + bool U = GetU(w); + static const char * const names[2][16] = { + { "saddl", "saddw", "ssubl", "ssubw", "addhn", "sabal", "subhn", "sabdl", "smlal", "sqdmlal", "smlsl", "sqdmlsl", "smull", "sqdmull", "pmull", nullptr }, + { "uaddl", "uaddw", "usubl", "usubw", "raddhn", "uabal", "rsubhn", "uabdl", "umlal", nullptr, "umlsl", nullptr, "umull", nullptr, nullptr, nullptr }, + }; + const char *name = names[U][opcode]; + if (!name || size == 3) { + Out(instr, "(asimd three-different %08x)", w); + return; + } + const char *two = Q ? "2" : ""; + const char half = Q ? 'q' : 'd'; + const int narrow = 8 << size, wide = 16 << size; + if (opcode == 1 || opcode == 3) { + // Wide: Rd and Rn are the wide lanes. + Out(instr, "%s%s.%d.%d q%d, q%d, %c%d", name, two, wide, narrow, Rd, Rn, half, Rm); + } else if (opcode == 4 || opcode == 6) { + // Narrowing: Rd gets the high halves. + Out(instr, "%s%s.%d.%d %c%d, q%d, q%d", name, two, narrow, wide, half, Rd, Rn, Rm); + } else { + Out(instr, "%s%s.%d.%d q%d, %c%d, %c%d", name, two, wide, narrow, Rd, half, Rn, half, Rm); + } +} + // (w >> 25) & 0xF == 7: AdvSIMD vector instructions. static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) { int Rd = w & 0x1f; @@ -1372,9 +1435,9 @@ static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) { } else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 0) { SIMDTwoRegMisc(w, instr); } else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 8) { - Out(instr, "(asimd across lanes %08x)", w); + SIMDAcrossLanes(w, instr); } else { - Out(instr, "(asimd three-different %08x)", w); + SIMDThreeDifferent(w, instr); } } else if ((w & 0x9FE08400) == 0x0E000400) { SIMDCopy(w, instr); diff --git a/unittest/TestArm64Disasm.cpp b/unittest/TestArm64Disasm.cpp index 245d3ab16e..21375be502 100644 --- a/unittest/TestArm64Disasm.cpp +++ b/unittest/TestArm64Disasm.cpp @@ -772,6 +772,20 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float DIS("umin.8 q0, q3, q4", fp.UMIN(8, Q0, Q3, Q4)); DIS("umax.16 q0, q3, q4", fp.UMAX(16, Q0, Q3, Q4)); DIS("umax.32 d0, d3, d4", fp.UMAX(32, D0, D3, D4)); + DIS("add.32 q0, q3, q4", fp.ADD(32, Q0, Q3, Q4)); + DIS("add.16 d0, d3, d4", fp.ADD(16, D0, D3, D4)); + DIS("sub.32 q0, q3, q4", fp.SUB(32, Q0, Q3, Q4)); + DIS("sub.8 d0, d3, d4", fp.SUB(8, D0, D3, D4)); + DIS("umaxp.32 q0, q3, q4", fp.UMAXP(32, Q0, Q3, Q4)); + DIS("umaxp.8 d0, d3, d4", fp.UMAXP(8, D0, D3, D4)); + DIS("ushl.32 q1, q2, q3", fp.USHL(32, Q1, Q2, Q3)); + DIS("ushl.16 d1, d2, d3", fp.USHL(16, D1, D2, D3)); + DIS("umull.64.32 q1, d2, d3", fp.UMULL(32, Q1, D2, D3)); + DIS("umull.16.8 q4, d5, d6", fp.UMULL(8, Q4, D5, D6)); + DIS("umull2.64.32 q1, q2, q3", fp.UMULL2(32, Q1, Q2, Q3)); + DIS("umull2.32.16 q7, q8, q9", fp.UMULL2(16, Q7, Q8, Q9)); + DIS("addv.32 s1, q2", fp.ADDV(32, S1, Q2)); + DIS("addv.16 h3, d4", fp.ADDV(16, S3, D4)); DIS("smin.8 d0, d3, d4", fp.SMIN(8, D0, D3, D4)); DIS("smin.16 q0, q3, q4", fp.SMIN(16, Q0, Q3, Q4)); DIS("smax.16 d0, d3, d4", fp.SMAX(16, D0, D3, D4)); @@ -967,6 +981,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float DIS("shll2.32.16 q1, q2, #16", fp.SHLL2(16, Q1, Q2)); DIS("shll2.16.8 q1, q2, #8", fp.SHLL2(8, Q1, Q2)); DIS("shrn.32.64 d0, q3, #8", fp.SHRN(32, D0, Q3, 8)); + DIS("usra.32 q1, q2, #21", fp.USRA(32, Q1, Q2, 21)); + DIS("usra.16 d1, d2, #3", fp.USRA(16, D1, D2, 3)); DIS("shrn.8.16 d0, q3, #4", fp.SHRN(8, D0, Q3, 4)); DIS("shrn2.16.32 q0, q3, #16", fp.SHRN2(16, Q0, Q3, 16)); DIS("shrn2.32.64 q0, q3, #1", fp.SHRN2(32, Q0, Q3, 1)); From 3e1198bf22629029d21bf4aaace009cf15013d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 14:24:52 +0200 Subject: [PATCH 08/17] ARM64 JITs: Make the VFPU dot routine NEON, after clang's vfpu_dot_neon It was integer code, about four times the work. It also clobbers q0-q3 and x16/x17 now, which both ARM64 JITs keep free, instead of saving them. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/ARM64/Arm64VfpuDot.h | 224 ++++++++++++++++----------------- 1 file changed, 112 insertions(+), 112 deletions(-) diff --git a/Core/MIPS/ARM64/Arm64VfpuDot.h b/Core/MIPS/ARM64/Arm64VfpuDot.h index 49f3cdba48..f1408c8f6e 100644 --- a/Core/MIPS/ARM64/Arm64VfpuDot.h +++ b/Core/MIPS/ARM64/Arm64VfpuDot.h @@ -26,141 +26,141 @@ namespace MIPSComp { -// Generates the VFPU dot product routine the ARM64 JITs call with BL: vfpu_dot() of the two -// Vec4s in mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. -// Every register is preserved except x30 and the flags. It's integer code, the algorithm of -// vfpu_dot_reference(), and leaves inf and NaN inputs to it. +// Generates the VFPU dot product routine the ARM64 JITs call with BL: vfpu_dot() of the two Vec4s +// in mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It's +// vfpu_dot_neon(), scheduled after what clang makes of it, and leaves inf and NaN to +// vfpu_dot_reference(). It clobbers q0-q3, x16, x17, x30 and the flags, which both ARM64 JITs keep +// free of guest state, and preserves everything else. inline const u8 *GenerateArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) { using namespace Arm64Gen; const s32 args = (s32)offsetof(MIPSState, vfpuDotArgs); const s32 result = (s32)offsetof(MIPSState, vfpuDotResult); const u8 *start = e->AlignCode16(); + fp->STP(128, INDEX_PRE, Q4, Q5, SP, -32); + fp->STP(128, INDEX_PRE, Q6, Q7, SP, -32); e->STP(INDEX_PRE, X0, X1, SP, -16); e->STP(INDEX_PRE, X2, X3, SP, -16); + + // q0 = x, q1 = y. Exponents: a zero or denormal operand makes the product zero, and inf or NaN + // gets 0x7FF added, above any real exponent sum, so one maximum finds the alignment and the + // specials (q3 = e, q5 = normal). + fp->LDR(128, INDEX_UNSIGNED, Q0, ctx, args); + fp->LDR(128, INDEX_UNSIGNED, Q1, ctx, args + 16); + e->MOVI2R(W16, 0x7F800000); + fp->DUP(32, Q2, W16); + fp->AND(Q3, Q0, Q2); + fp->AND(Q4, Q1, Q2); + fp->CMTST(32, Q5, Q0, Q2); + fp->CMTST(32, Q6, Q1, Q2); + fp->AND(Q5, Q5, Q6); + fp->UMAX(32, Q6, Q3, Q4); + fp->CMEQ(32, Q6, Q6, Q2); + fp->ADD(32, Q3, Q3, Q4); + fp->USHR(32, Q3, Q3, 23); + fp->AND(Q3, Q3, Q5); + fp->MOVI(32, Q2, 0x07, 8, true); // 0x7FF + fp->AND(Q6, Q6, Q2); + fp->ADD(32, Q3, Q3, Q6); + fp->UMAXP(32, Q2, Q3, Q3); + fp->UMAXP(32, Q2, Q2, Q2); + fp->UMOV(32, W16, Q2, 0); + e->CMPI2R(W16, 0x7FF); + FixupBranch special = e->B(CC_HS); + + // q4 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q1 = p). + fp->EOR(Q4, Q0, Q1); + fp->SSHR(32, Q4, Q4, 31); + fp->MOVI(32, Q6, 0x7F, 16, true); // 0x7FFFFF + fp->AND(Q0, Q0, Q6); + fp->ORR(32, Q0, 0x80, 16); + fp->AND(Q1, Q1, Q6); + fp->ORR(32, Q1, 0x80, 16); + fp->UMULL(32, Q6, D0, D1); + fp->UMULL2(32, Q0, Q0, Q1); + fp->SHRN(32, D1, Q6, 21); + fp->SHRN2(32, Q1, Q0, 21); + fp->UZP1(32, Q6, Q6, Q0); + fp->MOVI(32, Q0, 0x1F, 16, true); // (1 << 21) - 1 + fp->AND(Q6, Q6, Q0); + fp->MOVI(32, Q0, 1); + fp->UMIN(32, Q6, Q6, Q0); + fp->ORR(Q1, Q1, Q6); + fp->AND(Q1, Q1, Q5); + + // Align to the largest exponent by truncation, apply the signs, and sum exactly. + fp->DUP(32, Q2, Q2, 0); + fp->SUB(32, Q3, Q3, Q2); + fp->MVNI(32, Q2, 27); // -28 + fp->SMAX(32, Q3, Q3, Q2); + fp->USHL(32, Q1, Q1, Q3); + fp->EOR(Q1, Q1, Q4); + fp->SUB(32, Q1, Q1, Q4); + fp->ADDV(32, S1, Q1); + fp->UMOV(32, W17, Q1, 0); + + // vfpu_dot_finish(): w16 = the largest exponent sum, w17 = the sum. + e->ANDI2R(W0, W17, 0x80000000); + e->CMPI2R(W17, 0); + e->CSNEG(W17, W17, W17, CC_GE); + e->LSR(W1, W17, 2); + e->ORRI2R(W2, W1, 1); + e->CLZ(W2, W2); + e->ADDI2R(W3, W2, 31); + e->LSLV(X1, X1, X3); + e->UBFX(X3, X1, 39, 1); + e->ADD(X1, X1, X3); + e->MOVI2R(X3, (1ULL << 38) - 1); + e->ADD(X1, X1, X3); + e->LSR(X1, X1, 39); + e->SUB(W16, W16, W2); + e->SUBI2R(W16, W16, 120); + e->SXTW(X16, W16); + e->ADD(X16, X1, X16, ArithOption(X16, ST_LSL, 23)); + e->MOVI2R(W1, 0x7F7FFFFF); + e->CMP(X16, X1); + e->MOVI2R(W1, 0x7F800000); + e->CSEL(W1, W1, W16, CC_GT); + // Zero if the result is below the smallest normal, or the sum (in w17, still scaled by 4) is. + e->MOVI2R(W2, 0x00800000); + e->CMP(X16, X2); + e->CCMP(W17, 4, 0, CC_GE); + e->CSEL(W1, WZR, W1, CC_LO); + e->ORR(W1, W1, W0); + e->STR(INDEX_UNSIGNED, W1, ctx, result); + + const u8 *epilogue = e->GetCodePointer(); + e->LDP(INDEX_POST, X2, X3, SP, 16); + e->LDP(INDEX_POST, X0, X1, SP, 16); + fp->LDP(128, INDEX_POST, Q6, Q7, SP, 32); + fp->LDP(128, INDEX_POST, Q4, Q5, SP, 32); + e->RET(); + + // Inf or NaN: the C version, with everything a call may clobber saved too. + e->SetJumpTarget(special); e->STP(INDEX_PRE, X4, X5, SP, -16); e->STP(INDEX_PRE, X6, X7, SP, -16); e->STP(INDEX_PRE, X8, X9, SP, -16); e->STP(INDEX_PRE, X10, X11, SP, -16); - e->STP(INDEX_PRE, X30, ZR, SP, -16); - - // W0/W1: the lane's operands, W2/W3: their exponents, W4: the largest exponent sum of a - // normal product, W5: the sum. - // First pass: the largest exponent sum, and inf or NaN anywhere sends it to the slow path. - std::vector special; - e->MOVI2R(W4, 0); - for (int i = 0; i < 4; i++) { - e->LDR(INDEX_UNSIGNED, W0, ctx, args + i * 4); - e->LDR(INDEX_UNSIGNED, W1, ctx, args + 16 + i * 4); - e->UBFX(W2, W0, 23, 8); - e->UBFX(W3, W1, 23, 8); - e->CMPI2R(W2, 255); - special.push_back(e->B(CC_EQ)); - e->CMPI2R(W3, 255); - special.push_back(e->B(CC_EQ)); - // A zero or denormal operand makes the product zero, which doesn't count. - e->ADD(W7, W2, W3); - e->CMPI2R(W2, 0); - e->CCMP(W3, 0, 4, CC_NEQ); - e->CSEL(W7, W7, WZR, CC_NEQ); - e->CMP(W4, W7); - e->CSEL(W4, W4, W7, CC_HS); - } - - // Second pass: the 24x24-bit products, kept to 2 extra bits with round-to-odd, aligned to the - // largest by truncation, signed, and summed exactly. - e->MOVI2R(W5, 0); - for (int i = 0; i < 4; i++) { - e->LDR(INDEX_UNSIGNED, W0, ctx, args + i * 4); - e->LDR(INDEX_UNSIGNED, W1, ctx, args + 16 + i * 4); - e->UBFX(W2, W0, 23, 8); - e->UBFX(W3, W1, 23, 8); - e->ANDI2R(W9, W0, 0x007FFFFF); - e->ORRI2R(W9, W9, 0x00800000); - e->ANDI2R(W10, W1, 0x007FFFFF); - e->ORRI2R(W10, W10, 0x00800000); - e->UMULL(X6, W9, W10); - e->LSR(X7, X6, 21); - e->TSTI2R(X6, (1 << 21) - 1); - e->CSET(W8, CC_NEQ); - e->ORR(W7, W7, W8); - // Shift right by the exponent difference, at most 28. - e->ADD(W11, W2, W3); - e->SUB(W11, W4, W11); - e->MOVI2R(W8, 28); - e->CMP(W11, W8); - e->CSEL(W11, W11, W8, CC_LS); - e->LSRV(W7, W7, W11); - e->CMPI2R(W2, 0); - e->CCMP(W3, 0, 4, CC_NEQ); - e->CSEL(W7, W7, WZR, CC_NEQ); - e->EOR(W8, W0, W1); - e->TSTI2R(W8, 0x80000000); - e->CSNEG(W7, W7, W7, CC_EQ); - e->ADD(W5, W5, W7); - } - - // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. - e->ANDI2R(W8, W5, 0x80000000); - e->CMPI2R(W5, 0); - e->CSNEG(W9, W5, W5, CC_GE); - e->LSR(W9, W9, 2); - e->ORRI2R(W10, W9, 1); - e->CLZ(W10, W10); - e->ADDI2R(W10, W10, 32); - e->SUBI2R(W11, W10, 1); - e->LSLV(X6, X9, X11); - e->UBFX(X7, X6, 39, 1); - e->ADD(X6, X6, X7); - e->MOVI2R(X7, (1ULL << 38) - 1); - e->ADD(X6, X6, X7); - e->LSR(X6, X6, 39); - e->SUB(W7, W4, W10); - e->SUBI2R(W7, W7, 88); - e->SXTW(X7, W7); - e->LSL(X7, X7, 23); - e->ADD(X7, X7, X6); - e->MOVI2R(X6, 0x7F800000); - e->CMP(X7, X6); - e->CSEL(X7, X6, X7, CC_GE); - e->MOVI2R(X6, 0x00800000); - e->CMP(X7, X6); - e->CSEL(X7, ZR, X7, CC_LT); - e->CMPI2R(W9, 0); - e->CSEL(W7, WZR, W7, CC_EQ); - e->ORR(W7, W7, W8); - e->STR(INDEX_UNSIGNED, W7, ctx, result); - - const u8 *epilogue = e->GetCodePointer(); - e->LDP(INDEX_POST, X30, ZR, SP, 16); - e->LDP(INDEX_POST, X10, X11, SP, 16); - e->LDP(INDEX_POST, X8, X9, SP, 16); - e->LDP(INDEX_POST, X6, X7, SP, 16); - e->LDP(INDEX_POST, X4, X5, SP, 16); - e->LDP(INDEX_POST, X2, X3, SP, 16); - e->LDP(INDEX_POST, X0, X1, SP, 16); - e->RET(); - - // Inf or NaN: the C version, with everything else a call may clobber saved too. - for (const FixupBranch &b : special) - e->SetJumpTarget(b); e->STP(INDEX_PRE, X12, X13, SP, -16); e->STP(INDEX_PRE, X14, X15, SP, -16); - e->STP(INDEX_PRE, X16, X17, SP, -16); - e->STP(INDEX_PRE, X18, X19, SP, -16); - for (int i = 0; i < 32; i += 2) + e->STP(INDEX_PRE, X18, X30, SP, -16); + for (int i = 8; i < 32; i += 2) fp->STP(128, INDEX_PRE, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, -32); e->ADDI2R(X0, ctx, args); e->ADDI2R(X1, ctx, args + 16); e->MOVP2R(X16, &vfpu_dot_reference); e->BLR(X16); fp->STR(32, INDEX_UNSIGNED, S0, ctx, result); - for (int i = 30; i >= 0; i -= 2) + for (int i = 30; i >= 8; i -= 2) fp->LDP(128, INDEX_POST, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, 32); - e->LDP(INDEX_POST, X18, X19, SP, 16); - e->LDP(INDEX_POST, X16, X17, SP, 16); + e->LDP(INDEX_POST, X18, X30, SP, 16); e->LDP(INDEX_POST, X14, X15, SP, 16); e->LDP(INDEX_POST, X12, X13, SP, 16); + e->LDP(INDEX_POST, X10, X11, SP, 16); + e->LDP(INDEX_POST, X8, X9, SP, 16); + e->LDP(INDEX_POST, X6, X7, SP, 16); + e->LDP(INDEX_POST, X4, X5, SP, 16); e->B(epilogue); return start; } From 605057864e1cbb48654f511037c14f5640d0c87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 14:50:32 +0200 Subject: [PATCH 09/17] ARM64 emitter: Add vector BIC, and use it for the dot routine's normal mask Co-Authored-By: Claude Opus 5.5 --- Common/Arm64Emitter.cpp | 4 ++++ Common/Arm64Emitter.h | 2 ++ Core/MIPS/ARM64/Arm64VfpuDot.h | 4 ++-- unittest/TestArm64Disasm.cpp | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Common/Arm64Emitter.cpp b/Common/Arm64Emitter.cpp index 1a3f688600..931cc23b26 100644 --- a/Common/Arm64Emitter.cpp +++ b/Common/Arm64Emitter.cpp @@ -3080,6 +3080,10 @@ void ARM64FloatEmitter::AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(0, 0, 3, Rd, Rn, Rm); } +void ARM64FloatEmitter::BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(0, 1, 3, Rd, Rn, Rm); +} void ARM64FloatEmitter::EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(1, 0, 3, Rd, Rn, Rm); diff --git a/Common/Arm64Emitter.h b/Common/Arm64Emitter.h index 8edc282a4b..28b0262c25 100644 --- a/Common/Arm64Emitter.h +++ b/Common/Arm64Emitter.h @@ -854,6 +854,8 @@ public: // Vector void AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Rn & ~Rm. + void BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void BSL(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void BIT(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); diff --git a/Core/MIPS/ARM64/Arm64VfpuDot.h b/Core/MIPS/ARM64/Arm64VfpuDot.h index f1408c8f6e..f40b97461c 100644 --- a/Core/MIPS/ARM64/Arm64VfpuDot.h +++ b/Core/MIPS/ARM64/Arm64VfpuDot.h @@ -52,8 +52,8 @@ inline const u8 *GenerateArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM6 fp->AND(Q3, Q0, Q2); fp->AND(Q4, Q1, Q2); fp->CMTST(32, Q5, Q0, Q2); - fp->CMTST(32, Q6, Q1, Q2); - fp->AND(Q5, Q5, Q6); + fp->CMEQ(32, Q6, Q4); + fp->BIC(Q5, Q5, Q6); fp->UMAX(32, Q6, Q3, Q4); fp->CMEQ(32, Q6, Q6, Q2); fp->ADD(32, Q3, Q3, Q4); diff --git a/unittest/TestArm64Disasm.cpp b/unittest/TestArm64Disasm.cpp index 21375be502..b1c4d68ef8 100644 --- a/unittest/TestArm64Disasm.cpp +++ b/unittest/TestArm64Disasm.cpp @@ -698,6 +698,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float // Vector logical DIS("and q1, q2, q3", fp.AND(Q1, Q2, Q3)); DIS("and d1, d2, d3", fp.AND(D1, D2, D3)); + DIS("bic q1, q2, q3", fp.BIC(Q1, Q2, Q3)); + DIS("bic d1, d2, d3", fp.BIC(D1, D2, D3)); DIS("eor q0, q1, q2", fp.EOR(Q0, Q1, Q2)); DIS("eor d0, d1, d2", fp.EOR(D0, D1, D2)); DIS("bsl q1, q2, q3", fp.BSL(Q1, Q2, Q3)); From 27ef4e3c09ec99c0ea4657794f80a09eab80d169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 14:50:32 +0200 Subject: [PATCH 10/17] x86-64 JITs: Make the VFPU dot routine SSE2, after vfpu_dot_sse2 Inf and NaN still go to vfpu_dot_reference(). Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/x86/X64VfpuDot.h | 226 +++++++++++++++++++++++-------------- 1 file changed, 140 insertions(+), 86 deletions(-) diff --git a/Core/MIPS/x86/X64VfpuDot.h b/Core/MIPS/x86/X64VfpuDot.h index e883a20f1d..ce4778f5f7 100644 --- a/Core/MIPS/x86/X64VfpuDot.h +++ b/Core/MIPS/x86/X64VfpuDot.h @@ -22,6 +22,7 @@ #if PPSSPP_ARCH(AMD64) #include +#include #include #include "Common/x64Emitter.h" @@ -31,90 +32,138 @@ namespace MIPSComp { // Generates the VFPU dot product routine the x86-64 JITs CALL: vfpu_dot() of the two Vec4s in -// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. Every -// register is preserved except the flags. It's integer code, the algorithm of -// vfpu_dot_reference(), and leaves inf and NaN inputs to it. +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It's +// vfpu_dot_sse2(), SSE2 only, and leaves inf and NaN to vfpu_dot_reference(). Every register is +// preserved except the flags. inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { using namespace Gen; const int args = (int)offsetof(MIPSState, vfpuDotArgs); const int result = (int)offsetof(MIPSState, vfpuDotResult); - static const X64Reg saved[] = { RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11 }; + static const X64Reg saved[] = { RAX, RCX, RDX, RSI, RDI, R8, R10, R11 }; + const int numXmm = 8; + + // The constants, RIP-relative. + auto constant4 = [&](u32 a, u32 b, u32 c, u32 d) { + const u8 *p = e->AlignCode16(); + u8 *writable = e->GetWritableCodePtr(); + e->ReserveCodeSpace(16); + const u32 values[4] = { a, b, c, d }; + memcpy(writable, values, sizeof(values)); + return p; + }; + auto splat = [&](u32 v) { return constant4(v, v, v, v); }; + const u8 *expBits = splat(0x7F800000); + const u8 *mantMask = splat(0x007FFFFF); + const u8 *implicitBit = splat(0x00800000); + const u8 *lowMask = constant4((1 << 21) - 1, 0, (1 << 21) - 1, 0); + const u8 *ones = splat(1); + const u8 *max28 = splat(28); + const u8 *scaleBase = splat(127 + 28); const u8 *start = e->AlignCode16(); for (X64Reg r : saved) e->PUSH(r); + e->SUB(64, R(RSP), Imm32(16 * numXmm)); + for (int i = 0; i < numXmm; i++) + e->MOVUPS(MDisp(RSP, i * 16), (X64Reg)(XMM0 + i)); - // ESI/EDI: the lane's operands, R8D/R9D: their exponents, R10D: the largest exponent sum of a - // normal product, R11D: the sum. - auto loadLane = [&](int i) { - e->MOV(32, R(ESI), MDisp(ctx, args + i * 4)); - e->MOV(32, R(EDI), MDisp(ctx, args + 16 + i * 4)); - e->MOV(32, R(R8), R(ESI)); - e->SHR(32, R(R8), Imm8(23)); - e->AND(32, R(R8), Imm32(255)); - e->MOV(32, R(R9), R(EDI)); - e->SHR(32, R(R9), Imm8(23)); - e->AND(32, R(R9), Imm32(255)); - }; - // A zero or denormal operand makes the product zero, which doesn't count. - auto zeroUnlessNormal = [&](X64Reg r, X64Reg zero) { - e->TEST(32, R(R8), R(R8)); - e->CMOVcc(32, r, R(zero), CC_Z); - e->TEST(32, R(R9), R(R9)); - e->CMOVcc(32, r, R(zero), CC_Z); - }; + // xmm0 = x, xmm1 = y. Exponents: a zero or denormal operand makes the product zero (xmm4 = + // flush), and inf or NaN gets 0x7FF, above any real exponent sum, so one maximum finds the + // alignment and the specials (xmm5 = e). They all fit in 15 bits, so the 16-bit max does. + e->MOVUPS(XMM0, MDisp(ctx, args)); + e->MOVUPS(XMM1, MDisp(ctx, args + 16)); + e->PXOR(XMM7, R(XMM7)); + e->MOVDQA(XMM2, R(XMM0)); + e->PAND(XMM2, M(expBits)); + e->MOVDQA(XMM3, R(XMM1)); + e->PAND(XMM3, M(expBits)); + e->MOVDQA(XMM4, R(XMM2)); + e->PCMPEQD(XMM4, R(XMM7)); + e->MOVDQA(XMM5, R(XMM3)); + e->PCMPEQD(XMM5, R(XMM7)); + e->POR(XMM4, R(XMM5)); + e->MOVDQA(XMM6, R(XMM2)); + e->PCMPEQD(XMM6, M(expBits)); + e->MOVDQA(XMM5, R(XMM3)); + e->PCMPEQD(XMM5, M(expBits)); + e->POR(XMM6, R(XMM5)); + e->PSRLD(XMM6, 21); + e->MOVDQA(XMM5, R(XMM2)); + e->PADDD(XMM5, R(XMM3)); + e->PSRLD(XMM5, 23); + e->MOVDQA(XMM2, R(XMM4)); + e->PANDN(XMM2, R(XMM5)); + e->MOVDQA(XMM5, R(XMM2)); + e->POR(XMM5, R(XMM6)); + e->PSHUFD(XMM2, R(XMM5), 0x4E); + e->PMAXSW(XMM2, R(XMM5)); + e->PSHUFD(XMM3, R(XMM2), 0xB1); + e->PMAXSW(XMM2, R(XMM3)); + e->MOVD_xmm(R(EAX), XMM2); + e->CMP(32, R(EAX), Imm32(0x7FF)); + FixupBranch special = e->J_CC(CC_AE, true); - // First pass: the largest exponent sum, and inf or NaN anywhere sends it to the slow path. - std::vector special; - e->XOR(32, R(R10), R(R10)); - for (int i = 0; i < 4; i++) { - loadLane(i); - e->CMP(32, R(R8), Imm32(255)); - special.push_back(e->J_CC(CC_E, true)); - e->CMP(32, R(R9), Imm32(255)); - special.push_back(e->J_CC(CC_E, true)); - e->LEA(32, EAX, MRegSum(R8, R9)); - e->XOR(32, R(EDX), R(EDX)); - zeroUnlessNormal(EAX, EDX); - e->CMP(32, R(R10), R(EAX)); - e->CMOVcc(32, R10, R(EAX), CC_B); - } + // 24x24-bit products, even and odd lanes separately, kept to 2 extra bits with round-to-odd + // (xmm3 = p). + e->MOVDQA(XMM3, R(XMM0)); + e->PAND(XMM3, M(mantMask)); + e->POR(XMM3, M(implicitBit)); + e->MOVDQA(XMM6, R(XMM1)); + e->PAND(XMM6, M(mantMask)); + e->POR(XMM6, M(implicitBit)); + e->MOVDQA(XMM7, R(XMM3)); + e->PMULUDQ(XMM7, R(XMM6)); // even products + e->PSRLQ(XMM3, 32); + e->PSRLQ(XMM6, 32); + e->PMULUDQ(XMM3, R(XMM6)); // odd products + e->MOVDQA(XMM6, R(XMM7)); + e->PSRLQ(XMM6, 21); + e->PAND(XMM7, M(lowMask)); + e->MOVDQA(XMM1, R(XMM3)); + e->PSRLQ(XMM1, 21); + e->PSLLQ(XMM1, 32); + e->POR(XMM6, R(XMM1)); // kept + e->PAND(XMM3, M(lowMask)); + e->PSLLQ(XMM3, 32); + e->POR(XMM3, R(XMM7)); // dropped + e->PXOR(XMM7, R(XMM7)); + e->PCMPEQD(XMM3, R(XMM7)); + e->PANDN(XMM3, M(ones)); // sticky + e->POR(XMM3, R(XMM6)); + e->PANDN(XMM4, R(XMM3)); // p, in xmm4 - // Second pass: the 24x24-bit products, kept to 2 extra bits with round-to-odd, aligned to the - // largest by truncation, signed, and summed exactly. - e->XOR(32, R(R11), R(R11)); - for (int i = 0; i < 4; i++) { - loadLane(i); - e->MOV(32, R(EAX), R(ESI)); - e->AND(32, R(EAX), Imm32(0x007FFFFF)); - e->OR(32, R(EAX), Imm32(0x00800000)); - e->MOV(32, R(EDX), R(EDI)); - e->AND(32, R(EDX), Imm32(0x007FFFFF)); - e->OR(32, R(EDX), Imm32(0x00800000)); - e->IMUL(64, RAX, R(RDX)); - e->MOV(64, R(RDX), R(RAX)); - e->SHR(64, R(RDX), Imm8(21)); - e->TEST(32, R(EAX), Imm32((1 << 21) - 1)); - e->SETcc(CC_NZ, R(EAX)); - e->MOVZX(32, 8, EAX, R(EAX)); - e->OR(32, R(EDX), R(EAX)); - // Shift right by the exponent difference, at most 28. - e->LEA(32, ECX, MRegSum(R8, R9)); - e->MOV(32, R(EAX), R(R10)); - e->SUB(32, R(EAX), R(ECX)); - e->MOV(32, R(ECX), Imm32(28)); - e->CMP(32, R(EAX), R(ECX)); - e->CMOVcc(32, ECX, R(EAX), CC_B); - e->SHR(32, R(EDX), R(CL)); - e->XOR(32, R(EAX), R(EAX)); - zeroUnlessNormal(EDX, EAX); - e->MOV(32, R(EAX), R(ESI)); - e->XOR(32, R(EAX), R(EDI)); - e->SAR(32, R(EAX), Imm8(31)); - e->XOR(32, R(EDX), R(EAX)); - e->SUB(32, R(EDX), R(EAX)); - e->ADD(32, R(R11), R(EDX)); - } + // Align to the largest exponent by truncation. SSE2 has no per-lane shift, so p >> d is done + // as (p * 2^(28 - d)) >> 28, with the power of two built as float bits and converted by + // truncation (always a normal float, so DAZ doesn't matter). + e->PSHUFD(XMM2, R(XMM2), 0x00); + e->PSUBD(XMM2, R(XMM5)); + e->PMINSW(XMM2, M(max28)); + e->MOVDQA(XMM3, M(scaleBase)); + e->PSUBD(XMM3, R(XMM2)); + e->PSLLD(XMM3, 23); + e->CVTTPS2DQ(XMM3, R(XMM3)); + e->MOVDQA(XMM5, R(XMM4)); + e->PMULUDQ(XMM5, R(XMM3)); + e->PSRLQ(XMM5, 28); + e->PSRLQ(XMM4, 32); + e->PSRLQ(XMM3, 32); + e->PMULUDQ(XMM4, R(XMM3)); + e->PSRLQ(XMM4, 28); + e->PSLLQ(XMM4, 32); + e->POR(XMM4, R(XMM5)); + + // Apply the signs and sum exactly. + e->MOVUPS(XMM1, MDisp(ctx, args + 16)); + e->PXOR(XMM0, R(XMM1)); + e->PSRAD(XMM0, 31); + e->PXOR(XMM4, R(XMM0)); + e->PSUBD(XMM4, R(XMM0)); + e->PSHUFD(XMM0, R(XMM4), 0x4E); + e->PADDD(XMM4, R(XMM0)); + e->PSHUFD(XMM0, R(XMM4), 0xB1); + e->PADDD(XMM4, R(XMM0)); + e->MOVD_xmm(R(R11), XMM4); + e->MOV(32, R(R10), R(EAX)); // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. // ESI: the sign, EDI: the significand m, EDX: its leading zeros as 64 bits. @@ -160,21 +209,24 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { e->MOV(32, MDisp(ctx, result), R(R8)); const u8 *epilogue = e->GetCodePointer(); + for (int i = 0; i < numXmm; i++) + e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, i * 16)); + e->ADD(64, R(RSP), Imm32(16 * numXmm)); for (int i = (int)(sizeof(saved) / sizeof(saved[0])) - 1; i >= 0; i--) e->POP(saved[i]); e->RET(); - // Inf or NaN: the C version, with the vector registers saved too, on an aligned stack. - for (const FixupBranch &b : special) - e->SetJumpTarget(b); - // [RSP]: the shadow space Windows wants, then the 16 XMMs, then the unaligned RSP. - const int xmmOffset = 32, savedRsp = 32 + 16 * 16, frame = savedRsp + 16; - e->MOV(64, R(RAX), R(RSP)); + // Inf or NaN: the C version, with the other registers a call may clobber saved too, on an + // aligned stack. + e->SetJumpTarget(special); + e->PUSH(R9); + e->PUSH(RBX); + e->MOV(64, R(RBX), R(RSP)); e->AND(64, R(RSP), Imm32(-16)); - e->SUB(64, R(RSP), Imm32(frame)); - e->MOV(64, MDisp(RSP, savedRsp), R(RAX)); - for (int i = 0; i < 16; i++) - e->MOVUPS(MDisp(RSP, xmmOffset + i * 16), (X64Reg)(XMM0 + i)); + // The upper XMMs, and the shadow space Windows wants. + e->SUB(64, R(RSP), Imm32(8 * 16 + 32)); + for (int i = 8; i < 16; i++) + e->MOVUPS(MDisp(RSP, 32 + (i - 8) * 16), (X64Reg)(XMM0 + i)); #ifdef _WIN32 e->LEA(64, RCX, MDisp(ctx, args)); e->LEA(64, RDX, MDisp(ctx, args + 16)); @@ -185,9 +237,11 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { e->MOV(64, R(RAX), ImmPtr((const void *)&vfpu_dot_reference)); e->CALLptr(R(RAX)); e->MOVSS(MDisp(ctx, result), XMM0); - for (int i = 0; i < 16; i++) - e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, xmmOffset + i * 16)); - e->MOV(64, R(RSP), MDisp(RSP, savedRsp)); + for (int i = 8; i < 16; i++) + e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, 32 + (i - 8) * 16)); + e->MOV(64, R(RSP), R(RBX)); + e->POP(RBX); + e->POP(R9); e->JMP(epilogue, true); return start; } From 0b4b70e67e0b6084ae88330a40a5ca0c90f57742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 20:28:42 +0200 Subject: [PATCH 11/17] ARM64 IR JIT: Call the dot routine with the operands in registers The operands go in q0 and q1 and the result comes back in w1. The routine saves nothing: the call site flushes whatever guest state is in the few registers it clobbers (q4-q6, w0-w3), which come late in the allocation order, so usually there's nothing to do. That's about 20 bytes a call, with nothing going through memory, and as fast as having the whole thing inline, which made a vmmul.q about 5 KB of code. Inf and NaN go through a routine that calls vfpu_dot_reference(). Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/ARM64/Arm64IRAsm.cpp | 3 +- Core/MIPS/ARM64/Arm64IRCompVec.cpp | 17 +- Core/MIPS/ARM64/Arm64IRJit.cpp | 7 +- Core/MIPS/ARM64/Arm64IRJit.h | 4 +- Core/MIPS/ARM64/Arm64IRRegCache.cpp | 9 + Core/MIPS/ARM64/Arm64IRRegCache.h | 5 + Core/MIPS/ARM64/Arm64VfpuDot.h | 269 +++++++++++++++------------- 7 files changed, 184 insertions(+), 130 deletions(-) diff --git a/Core/MIPS/ARM64/Arm64IRAsm.cpp b/Core/MIPS/ARM64/Arm64IRAsm.cpp index 02d75b9eb4..a4c6aa0ec4 100644 --- a/Core/MIPS/ARM64/Arm64IRAsm.cpp +++ b/Core/MIPS/ARM64/Arm64IRAsm.cpp @@ -265,7 +265,8 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) { RET(); } - vfpuDot_ = GenerateArm64VfpuDot(this, &fp_, CTXREG); + vfpuDotSpecial_ = GenerateArm64VfpuDotSpecial(this, &fp_, CTXREG); + vfpuDot_ = GenerateArm64VfpuDotCall(this, &fp_, CTXREG, vfpuDotSpecial_); // Leave this at the end, add more stuff above. if (enableDisasm) { diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 634f54c889..ca308caa33 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -24,6 +24,7 @@ #include "Core/MemMap.h" #include "Core/MIPS/ARM64/Arm64IRJit.h" #include "Core/MIPS/ARM64/Arm64IRRegCache.h" +#include "Core/MIPS/ARM64/Arm64VfpuDot.h" // This file contains compilation for vector instructions. // @@ -600,14 +601,20 @@ void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) { break; case IROp::Vec4VfpuDot: - // Through memory and the shared routine, which preserves every register but x30. + { + // The operands go to the shared routine in q0 and q1, and the result comes back in w1. The + // routine clobbers q4-q6 and x0-x3 too, so whatever guest state is there gets flushed: these + // come late in the allocation order, so usually that's nothing. regs_.SpillLockFPR(inst.src1, inst.src2); regs_.MapVec4(inst.src1); regs_.MapVec4(inst.src2); - fp_.STR(128, INDEX_UNSIGNED, regs_.FQ(inst.src1), CTXREG, offsetof(MIPSState, vfpuDotArgs)); - fp_.STR(128, INDEX_UNSIGNED, regs_.FQ(inst.src2), CTXREG, offsetof(MIPSState, vfpuDotArgs) + 16); + fp_.MOV(Q0, regs_.FQ(inst.src1)); + fp_.MOV(Q1, regs_.FQ(inst.src2)); + regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); + regs_.FlushHostRegs({ Q4, Q5, Q6, W0, W1, W2, W3 }); BL(vfpuDot_); - fp_.LDR(32, INDEX_UNSIGNED, SCRATCHF1, CTXREG, offsetof(MIPSState, vfpuDotResult)); + fp_.FMOV(SCRATCHF1, W1); + if (Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { // The dest lane is in a mapped Vec4. regs_.MapVec4(inst.dest & ~3, MIPSMap::DIRTY); @@ -616,8 +623,8 @@ void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) { regs_.MapFPR(inst.dest, MIPSMap::NOINIT); fp_.FMOV(regs_.F(inst.dest), SCRATCHF1); } - regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); break; + } default: INVALIDOP; diff --git a/Core/MIPS/ARM64/Arm64IRJit.cpp b/Core/MIPS/ARM64/Arm64IRJit.cpp index 56b0adbc93..ca4136c020 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.cpp +++ b/Core/MIPS/ARM64/Arm64IRJit.cpp @@ -71,7 +71,12 @@ bool Arm64JitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) { return false; IRBlock *block = irBlockCache->GetBlock(block_num); - BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32)); + // An accurate dot product can flush several registers around its call. + size_t sizeEstimate = 0; + const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block); + for (int i = 0; i < block->GetNumIRInstructions(); ++i) + sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32; + BeginWrite(std::min(GetSpaceLeft(), sizeEstimate)); u32 startPC = block->GetOriginalStart(); bool wroteCheckedOffset = false; diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index 665838cce7..20d428ae26 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -145,7 +145,9 @@ private: // Indexed by FPCR FZ:RN bits for convenience. Uses SCRATCH2. const u8 *convertS0ToSCRATCH1_[8]; - // See GenerateArm64VfpuDot(). + // See GenerateArm64VfpuDotSpecial(). + const u8 *vfpuDotSpecial_ = nullptr; + // See GenerateArm64VfpuDotCall(). const u8 *vfpuDot_ = nullptr; // Note: mutable state used at runtime. diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.cpp b/Core/MIPS/ARM64/Arm64IRRegCache.cpp index f7bea20c00..556fa4ea39 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.cpp +++ b/Core/MIPS/ARM64/Arm64IRRegCache.cpp @@ -147,6 +147,15 @@ void Arm64IRRegCache::EmitSaveStaticRegisters() { } } +void Arm64IRRegCache::FlushHostRegs(std::initializer_list regs) { + for (ARM64Reg r : regs) { + if (IsVector(r)) + FlushNativeReg(VFPToNativeReg(EncodeRegToSingle(r))); + else + FlushNativeReg(GPRToNativeReg(DecodeReg(r))); + } +} + void Arm64IRRegCache::FlushBeforeCall() { // These registers are not preserved by function calls. auto isGPRSaved = [&](IRNativeReg nreg) { diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.h b/Core/MIPS/ARM64/Arm64IRRegCache.h index f3f0b58b6f..d444913d97 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.h +++ b/Core/MIPS/ARM64/Arm64IRRegCache.h @@ -18,6 +18,9 @@ #pragma once #include "ppsspp_config.h" + +#include + // In other words, PPSSPP_ARCH(ARM64) || DISASM_ALL. #if PPSSPP_ARCH(ARM64) || (PPSSPP_PLATFORM(WINDOWS) && !defined(__LIBRETRO__)) @@ -65,6 +68,8 @@ public: Arm64Gen::ARM64Reg MapWithFPRTemp(const IRInst &inst); void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list regs); void FlushAll(bool gprs = true, bool fprs = true) override; Arm64Gen::ARM64Reg GetAndLockTempGPR(); diff --git a/Core/MIPS/ARM64/Arm64VfpuDot.h b/Core/MIPS/ARM64/Arm64VfpuDot.h index f40b97461c..ce8d64601f 100644 --- a/Core/MIPS/ARM64/Arm64VfpuDot.h +++ b/Core/MIPS/ARM64/Arm64VfpuDot.h @@ -26,142 +26,167 @@ namespace MIPSComp { -// Generates the VFPU dot product routine the ARM64 JITs call with BL: vfpu_dot() of the two Vec4s -// in mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It's -// vfpu_dot_neon(), scheduled after what clang makes of it, and leaves inf and NaN to -// vfpu_dot_reference(). It clobbers q0-q3, x16, x17, x30 and the flags, which both ARM64 JITs keep -// free of guest state, and preserves everything else. -inline const u8 *GenerateArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) { +// The registers EmitArm64VfpuDot() works in. x and y are only read. q are seven vector temps and w +// six GPR temps, none of them x or y, except that q[0] and q[1] may be x and y themselves. +struct Arm64VfpuDotRegs { + Arm64Gen::ARM64Reg x; + Arm64Gen::ARM64Reg y; + Arm64Gen::ARM64Reg q[7]; + Arm64Gen::ARM64Reg w[6]; +}; + +// Emits vfpu_dot() of the Vec4s x and y: vfpu_dot_neon(), scheduled after what clang makes of it. +// The result ends up in w[3]. Inf and NaN aren't handled: the returned branch is taken for them, +// and the caller sends those to GenerateArm64VfpuDotSpecial()'s routine. Clobbers the flags. +inline Arm64Gen::FixupBranch EmitArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, const Arm64VfpuDotRegs &r) { + using namespace Arm64Gen; + const ARM64Reg x = EncodeRegToQuad(r.x); + const ARM64Reg y = EncodeRegToQuad(r.y); + ARM64Reg q[7]; + for (int i = 0; i < 7; i++) + q[i] = EncodeRegToQuad(r.q[i]); + const ARM64Reg emax = DecodeReg(r.w[0]); + const ARM64Reg sum = DecodeReg(r.w[1]); + const ARM64Reg sign = DecodeReg(r.w[2]); + const ARM64Reg m = DecodeReg(r.w[3]); + const ARM64Reg t2 = DecodeReg(r.w[4]); + const ARM64Reg t3 = DecodeReg(r.w[5]); + + // Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF + // added, above any real exponent sum, so one maximum finds the alignment and the specials + // (q3 = e, q5 = normal). + e->MOVI2R(emax, 0x7F800000); + fp->DUP(32, q[2], emax); + fp->AND(q[3], x, q[2]); + fp->AND(q[4], y, q[2]); + fp->CMTST(32, q[5], x, q[2]); + fp->CMEQ(32, q[6], q[4]); + fp->BIC(q[5], q[5], q[6]); + fp->UMAX(32, q[6], q[3], q[4]); + fp->CMEQ(32, q[6], q[6], q[2]); + fp->ADD(32, q[3], q[3], q[4]); + fp->USHR(32, q[3], q[3], 23); + fp->AND(q[3], q[3], q[5]); + fp->MOVI(32, q[2], 0x07, 8, true); // 0x7FF + fp->AND(q[6], q[6], q[2]); + fp->ADD(32, q[3], q[3], q[6]); + fp->UMAXP(32, q[2], q[3], q[3]); + fp->UMAXP(32, q[2], q[2], q[2]); + fp->UMOV(32, emax, q[2], 0); + e->CMPI2R(emax, 0x7FF); + FixupBranch special = e->B(CC_HS); + + // q4 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q1 = p). + fp->EOR(q[4], x, y); + fp->SSHR(32, q[4], q[4], 31); + fp->MOVI(32, q[6], 0x7F, 16, true); // 0x7FFFFF + fp->AND(q[0], x, q[6]); + fp->ORR(32, q[0], 0x80, 16); + fp->AND(q[1], y, q[6]); + fp->ORR(32, q[1], 0x80, 16); + fp->UMULL(32, q[6], EncodeRegToDouble(q[0]), EncodeRegToDouble(q[1])); + fp->UMULL2(32, q[0], q[0], q[1]); + fp->SHRN(32, EncodeRegToDouble(q[1]), q[6], 21); + fp->SHRN2(32, q[1], q[0], 21); + fp->UZP1(32, q[6], q[6], q[0]); + fp->MOVI(32, q[0], 0x1F, 16, true); // (1 << 21) - 1 + fp->AND(q[6], q[6], q[0]); + fp->MOVI(32, q[0], 1); + fp->UMIN(32, q[6], q[6], q[0]); + fp->ORR(q[1], q[1], q[6]); + fp->AND(q[1], q[1], q[5]); + + // Align to the largest exponent by truncation, apply the signs, and sum exactly. + fp->DUP(32, q[2], q[2], 0); + fp->SUB(32, q[3], q[3], q[2]); + fp->MVNI(32, q[2], 27); // -28 + fp->SMAX(32, q[3], q[3], q[2]); + fp->USHL(32, q[1], q[1], q[3]); + fp->EOR(q[1], q[1], q[4]); + fp->SUB(32, q[1], q[1], q[4]); + fp->ADDV(32, EncodeRegToSingle(q[1]), q[1]); + fp->UMOV(32, sum, q[1], 0); + + // vfpu_dot_finish(): emax = the largest exponent sum, sum = the sum. + e->ANDI2R(sign, sum, 0x80000000); + e->CMPI2R(sum, 0); + e->CSNEG(sum, sum, sum, CC_GE); + e->LSR(m, sum, 2); + e->ORRI2R(t2, m, 1); + e->CLZ(t2, t2); + e->ADDI2R(t3, t2, 31); + e->LSLV(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->UBFX(EncodeRegTo64(t3), EncodeRegTo64(m), 39, 1); + e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->MOVI2R(EncodeRegTo64(t3), (1ULL << 38) - 1); + e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->LSR(EncodeRegTo64(m), EncodeRegTo64(m), 39); + e->SUB(emax, emax, t2); + e->SUBI2R(emax, emax, 120); + e->SXTW(EncodeRegTo64(emax), emax); + e->ADD(EncodeRegTo64(emax), EncodeRegTo64(m), EncodeRegTo64(emax), ArithOption(EncodeRegTo64(emax), ST_LSL, 23)); + e->MOVI2R(m, 0x7F7FFFFF); + e->CMP(EncodeRegTo64(emax), EncodeRegTo64(m)); + e->MOVI2R(m, 0x7F800000); + e->CSEL(m, m, emax, CC_GT); + // Zero if the result is below the smallest normal, or the sum (still scaled by 4) is. + e->MOVI2R(t2, 0x00800000); + e->CMP(EncodeRegTo64(emax), EncodeRegTo64(t2)); + e->CCMP(sum, 4, 0, CC_GE); + e->CSEL(m, WZR, m, CC_LO); + e->ORR(m, m, sign); + return special; +} + +// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It clobbers +// q0-q3, x16, x17, x30 and the flags, which the IR JIT keeps free of guest state, and preserves +// everything else. +inline const u8 *GenerateArm64VfpuDotSpecial(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) { using namespace Arm64Gen; const s32 args = (s32)offsetof(MIPSState, vfpuDotArgs); const s32 result = (s32)offsetof(MIPSState, vfpuDotResult); const u8 *start = e->AlignCode16(); - fp->STP(128, INDEX_PRE, Q4, Q5, SP, -32); - fp->STP(128, INDEX_PRE, Q6, Q7, SP, -32); - e->STP(INDEX_PRE, X0, X1, SP, -16); - e->STP(INDEX_PRE, X2, X3, SP, -16); - - // q0 = x, q1 = y. Exponents: a zero or denormal operand makes the product zero, and inf or NaN - // gets 0x7FF added, above any real exponent sum, so one maximum finds the alignment and the - // specials (q3 = e, q5 = normal). - fp->LDR(128, INDEX_UNSIGNED, Q0, ctx, args); - fp->LDR(128, INDEX_UNSIGNED, Q1, ctx, args + 16); - e->MOVI2R(W16, 0x7F800000); - fp->DUP(32, Q2, W16); - fp->AND(Q3, Q0, Q2); - fp->AND(Q4, Q1, Q2); - fp->CMTST(32, Q5, Q0, Q2); - fp->CMEQ(32, Q6, Q4); - fp->BIC(Q5, Q5, Q6); - fp->UMAX(32, Q6, Q3, Q4); - fp->CMEQ(32, Q6, Q6, Q2); - fp->ADD(32, Q3, Q3, Q4); - fp->USHR(32, Q3, Q3, 23); - fp->AND(Q3, Q3, Q5); - fp->MOVI(32, Q2, 0x07, 8, true); // 0x7FF - fp->AND(Q6, Q6, Q2); - fp->ADD(32, Q3, Q3, Q6); - fp->UMAXP(32, Q2, Q3, Q3); - fp->UMAXP(32, Q2, Q2, Q2); - fp->UMOV(32, W16, Q2, 0); - e->CMPI2R(W16, 0x7FF); - FixupBranch special = e->B(CC_HS); - - // q4 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q1 = p). - fp->EOR(Q4, Q0, Q1); - fp->SSHR(32, Q4, Q4, 31); - fp->MOVI(32, Q6, 0x7F, 16, true); // 0x7FFFFF - fp->AND(Q0, Q0, Q6); - fp->ORR(32, Q0, 0x80, 16); - fp->AND(Q1, Q1, Q6); - fp->ORR(32, Q1, 0x80, 16); - fp->UMULL(32, Q6, D0, D1); - fp->UMULL2(32, Q0, Q0, Q1); - fp->SHRN(32, D1, Q6, 21); - fp->SHRN2(32, Q1, Q0, 21); - fp->UZP1(32, Q6, Q6, Q0); - fp->MOVI(32, Q0, 0x1F, 16, true); // (1 << 21) - 1 - fp->AND(Q6, Q6, Q0); - fp->MOVI(32, Q0, 1); - fp->UMIN(32, Q6, Q6, Q0); - fp->ORR(Q1, Q1, Q6); - fp->AND(Q1, Q1, Q5); - - // Align to the largest exponent by truncation, apply the signs, and sum exactly. - fp->DUP(32, Q2, Q2, 0); - fp->SUB(32, Q3, Q3, Q2); - fp->MVNI(32, Q2, 27); // -28 - fp->SMAX(32, Q3, Q3, Q2); - fp->USHL(32, Q1, Q1, Q3); - fp->EOR(Q1, Q1, Q4); - fp->SUB(32, Q1, Q1, Q4); - fp->ADDV(32, S1, Q1); - fp->UMOV(32, W17, Q1, 0); - - // vfpu_dot_finish(): w16 = the largest exponent sum, w17 = the sum. - e->ANDI2R(W0, W17, 0x80000000); - e->CMPI2R(W17, 0); - e->CSNEG(W17, W17, W17, CC_GE); - e->LSR(W1, W17, 2); - e->ORRI2R(W2, W1, 1); - e->CLZ(W2, W2); - e->ADDI2R(W3, W2, 31); - e->LSLV(X1, X1, X3); - e->UBFX(X3, X1, 39, 1); - e->ADD(X1, X1, X3); - e->MOVI2R(X3, (1ULL << 38) - 1); - e->ADD(X1, X1, X3); - e->LSR(X1, X1, 39); - e->SUB(W16, W16, W2); - e->SUBI2R(W16, W16, 120); - e->SXTW(X16, W16); - e->ADD(X16, X1, X16, ArithOption(X16, ST_LSL, 23)); - e->MOVI2R(W1, 0x7F7FFFFF); - e->CMP(X16, X1); - e->MOVI2R(W1, 0x7F800000); - e->CSEL(W1, W1, W16, CC_GT); - // Zero if the result is below the smallest normal, or the sum (in w17, still scaled by 4) is. - e->MOVI2R(W2, 0x00800000); - e->CMP(X16, X2); - e->CCMP(W17, 4, 0, CC_GE); - e->CSEL(W1, WZR, W1, CC_LO); - e->ORR(W1, W1, W0); - e->STR(INDEX_UNSIGNED, W1, ctx, result); - - const u8 *epilogue = e->GetCodePointer(); - e->LDP(INDEX_POST, X2, X3, SP, 16); - e->LDP(INDEX_POST, X0, X1, SP, 16); - fp->LDP(128, INDEX_POST, Q6, Q7, SP, 32); - fp->LDP(128, INDEX_POST, Q4, Q5, SP, 32); - e->RET(); - - // Inf or NaN: the C version, with everything a call may clobber saved too. - e->SetJumpTarget(special); - e->STP(INDEX_PRE, X4, X5, SP, -16); - e->STP(INDEX_PRE, X6, X7, SP, -16); - e->STP(INDEX_PRE, X8, X9, SP, -16); - e->STP(INDEX_PRE, X10, X11, SP, -16); - e->STP(INDEX_PRE, X12, X13, SP, -16); - e->STP(INDEX_PRE, X14, X15, SP, -16); + for (int i = 0; i < 16; i += 2) + e->STP(INDEX_PRE, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, -16); e->STP(INDEX_PRE, X18, X30, SP, -16); - for (int i = 8; i < 32; i += 2) + for (int i = 4; i < 32; i += 2) fp->STP(128, INDEX_PRE, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, -32); e->ADDI2R(X0, ctx, args); e->ADDI2R(X1, ctx, args + 16); e->MOVP2R(X16, &vfpu_dot_reference); e->BLR(X16); fp->STR(32, INDEX_UNSIGNED, S0, ctx, result); - for (int i = 30; i >= 8; i -= 2) + for (int i = 30; i >= 4; i -= 2) fp->LDP(128, INDEX_POST, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, 32); e->LDP(INDEX_POST, X18, X30, SP, 16); - e->LDP(INDEX_POST, X14, X15, SP, 16); - e->LDP(INDEX_POST, X12, X13, SP, 16); - e->LDP(INDEX_POST, X10, X11, SP, 16); - e->LDP(INDEX_POST, X8, X9, SP, 16); - e->LDP(INDEX_POST, X6, X7, SP, 16); - e->LDP(INDEX_POST, X4, X5, SP, 16); - e->B(epilogue); + for (int i = 14; i >= 0; i -= 2) + e->LDP(INDEX_POST, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, 16); + e->RET(); + return start; +} + +// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of q0 and q1, into w1, +// with special (from GenerateArm64VfpuDotSpecial()) for inf and NaN. It clobbers q0-q6, x0-x3, x16, +// x17, x30 and the flags, and saves nothing: the call site flushes whatever of those holds guest +// state, which in most blocks is nothing. +inline const u8 *GenerateArm64VfpuDotCall(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx, const u8 *special) { + using namespace Arm64Gen; + const u8 *start = e->AlignCode16(); + const Arm64VfpuDotRegs regs{ Q0, Q1, { Q0, Q1, Q2, Q3, Q4, Q5, Q6 }, { W16, W17, W0, W1, W2, W3 } }; + FixupBranch specialBranch = EmitArm64VfpuDot(e, fp, regs); + e->RET(); + + // The operands are still in q0 and q1. + e->SetJumpTarget(specialBranch); + e->ADDI2R(X16, ctx, offsetof(MIPSState, vfpuDotArgs)); + fp->STP(128, INDEX_SIGNED, Q0, Q1, X16, 0); + e->STR(INDEX_PRE, X30, SP, -16); + e->BL(special); + e->LDR(INDEX_POST, X30, SP, 16); + e->LDR(INDEX_UNSIGNED, W1, ctx, offsetof(MIPSState, vfpuDotResult)); + e->RET(); return start; } From 0e77a2d1cdbef7cfa39fb91ef27b3cb7476e92dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 17:44:54 +0200 Subject: [PATCH 12/17] VFPU: Make AccurateVdot the default The accurate dot products cost a few percent at most in the IR JIT, apart from GTA (about 20% on a game that's cheap to run anyway). The fuzzer follows the default too, unless PPSSPP_CPUFUZZ_ACCURATE_VDOT=0. Co-Authored-By: Claude Opus 5.5 --- Core/CmdLine.cpp | 2 +- Core/Config.cpp | 2 +- docs/cpu-equivalence.md | 2 +- unittest/JitHarness.cpp | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 861bfc755f..de55ed159f 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -199,7 +199,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode", CmdLineMode::Headless}, {POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode", CmdLineMode::Headless}, {POFF(oldAtrac), CmdParamType::Bool, "old-atrac", '\0', "Use old ATRAC decoder"}, - {POFF(accurateVdot), CmdParamType::Bool, "accurate-vdot", '\0', "Compute VFPU dot products bit-exactly like the PSP (slower)"}, + {POFF(accurateVdot), CmdParamType::Bool, "accurate-vdot", '\0', "Compute VFPU dot products bit-exactly like the PSP (default; =false for the faster approximation)"}, {POFF(memStickInserted), CmdParamType::BoolInverse, "no-memstick", '\0', "Run with the memory stick removed"}, {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application}, {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, diff --git a/Core/Config.cpp b/Core/Config.cpp index 2b8cb2e513..5961e2d207 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -436,7 +436,7 @@ static const ConfigSetting cpuSettings[] = { ConfigSetting("IOTimingMethod", SETTING(g_Config, iIOTimingMethod), IOTIMING_FAST, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("FastMemoryAccess", SETTING(g_Config, bFastMemory), true, CfgFlag::PER_GAME), ConfigSetting("FunctionReplacements", SETTING(g_Config, bFuncReplacements), true, CfgFlag::PER_GAME | CfgFlag::REPORT), - ConfigSetting("AccurateVdot", SETTING(g_Config, bAccurateVdot), false, CfgFlag::PER_GAME | CfgFlag::REPORT), + ConfigSetting("AccurateVdot", SETTING(g_Config, bAccurateVdot), true, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("HideSlowWarnings", SETTING(g_Config, bHideSlowWarnings), false, CfgFlag::DEFAULT), ConfigSetting("HideStateWarnings", SETTING(g_Config, bHideStateWarnings), false, CfgFlag::DEFAULT), ConfigSetting("JitDisableFlags", SETTING(g_Config, uJitDisableFlags), (uint32_t)0, CfgFlag::PER_GAME), diff --git a/docs/cpu-equivalence.md b/docs/cpu-equivalence.md index 7c8a73b386..804f26b375 100644 --- a/docs/cpu-equivalence.md +++ b/docs/cpu-equivalence.md @@ -33,7 +33,7 @@ Environment variables: | `PPSSPP_CPUFUZZ_DUMP` | write every core's end state of every case to a file | | `PPSSPP_CPUFUZZ_STRICT` | fail on any difference, not only on a crash | | `PPSSPP_CPUFUZZ_OPS` | comma-separated instruction name prefixes to generate only; `dot` is the dot products and the prefix ops | -| `PPSSPP_CPUFUZZ_ACCURATE_VDOT` | run with the accurate dot products | +| `PPSSPP_CPUFUZZ_ACCURATE_VDOT` | `0` to run without the accurate dot products, which are on by default | For quick rounds while working on some instructions, fuzz just those, on every core of the machine: `PPSSPP_CPUFUZZ_OPS=dot Tools/cpu_fuzz_parallel.py build/PPSSPPUnitTest --count 2000` splits the seeds diff --git a/unittest/JitHarness.cpp b/unittest/JitHarness.cpp index 1c39d7d668..ccbcca9782 100644 --- a/unittest/JitHarness.cpp +++ b/unittest/JitHarness.cpp @@ -398,7 +398,7 @@ bool TestJitReplacementHooks() { // PPSSPP_CPUFUZZ_SKIP cores not to run, by digit: 1 IR interpreter, 2 JIT, 3 JIT IR. // PPSSPP_CPUFUZZ_REPORTS how many differences to minimize and print (default 20). // PPSSPP_CPUFUZZ_STRICT fail the test on any difference, not only on a crash. -// PPSSPP_CPUFUZZ_ACCURATE_VDOT run with AccurateVdot, the VFPU's own dot products. +// PPSSPP_CPUFUZZ_ACCURATE_VDOT 0 to run without AccurateVdot (the VFPU's own dot products), which is on by default. // PPSSPP_CPUFUZZ_OPS comma-separated instruction name prefixes to generate only, for fast // runs on the instructions being worked on. "dot" is the dot products // and the prefixes. Without PPSSPP_CPUFUZZ_CLASSES, the class is "v". @@ -901,7 +901,8 @@ bool TestCPUEquivalence() { const bool oldSkipDeadbeef = g_Config.bSkipDeadbeefFilling; g_Config.bSkipDeadbeefFilling = true; const bool oldAccurateVdot = g_Config.bAccurateVdot; - g_Config.bAccurateVdot = getenv("PPSSPP_CPUFUZZ_ACCURATE_VDOT") != nullptr; + const char *accurateVdot = getenv("PPSSPP_CPUFUZZ_ACCURATE_VDOT"); + g_Config.bAccurateVdot = !accurateVdot || strcmp(accurateVdot, "0") != 0; const char *countStr = getenv("PPSSPP_CPUFUZZ_COUNT"); const char *seedStr = getenv("PPSSPP_CPUFUZZ_SEED"); From bd54c3e6598711e0489760861455a095007bb92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 17:47:06 +0200 Subject: [PATCH 13/17] Compat: Replace MoreAccurateVMMUL with ForceAccurateVdot The games that needed vmmul's accurate dot products (Tekken 6, Warriors Orochi, Tomb Raider Anniversary) now get AccurateVdot for every dot product, even with the setting turned off. Co-Authored-By: Claude Opus 5.5 --- Core/Compatibility.cpp | 2 +- Core/Compatibility.h | 2 +- Core/MIPS/IR/IRCompVFPU.cpp | 4 ++-- Core/MIPS/InterpreterVFPU.cpp | 5 ++--- assets/compat.ini | 5 ++--- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Core/Compatibility.cpp b/Core/Compatibility.cpp index 37c10f6e47..127e85ac71 100644 --- a/Core/Compatibility.cpp +++ b/Core/Compatibility.cpp @@ -114,7 +114,7 @@ void Compatibility::CheckSettings(IniFile &iniFile, const std::string &gameID) { CheckSetting(iniFile, gameID, "GoWFramerateHack60", &flags_.GoWFramerateHack60); CheckSetting(iniFile, gameID, "FramerateHack30", &flags_.FramerateHack30); CheckSetting(iniFile, gameID, "HideISOFiles", &flags_.HideISOFiles); - CheckSetting(iniFile, gameID, "MoreAccurateVMMUL", &flags_.MoreAccurateVMMUL); + CheckSetting(iniFile, gameID, "ForceAccurateVdot", &flags_.ForceAccurateVdot); CheckSetting(iniFile, gameID, "ForceSoftwareRenderer", &flags_.ForceSoftwareRenderer); CheckSetting(iniFile, gameID, "DarkStalkersPresentHack", &flags_.DarkStalkersPresentHack); CheckSetting(iniFile, gameID, "ReportSmallMemstick", &flags_.ReportSmallMemstick); diff --git a/Core/Compatibility.h b/Core/Compatibility.h index 344b301aa8..be4ed125e2 100644 --- a/Core/Compatibility.h +++ b/Core/Compatibility.h @@ -69,7 +69,7 @@ struct CompatFlags { bool GoWFramerateHack60; bool FramerateHack30; bool HideISOFiles; - bool MoreAccurateVMMUL; + bool ForceAccurateVdot; bool ForceSoftwareRenderer; bool DarkStalkersPresentHack; bool ReportSmallMemstick; diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index 5d2b2bcaaa..7a270c6423 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -151,7 +151,7 @@ namespace MIPSComp { } bool IRFrontend::UseAccurateVdot() const { - return g_Config.bAccurateVdot; + return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot; } // The lanes are already a Vec4 reg, used as is. @@ -1761,7 +1761,7 @@ namespace MIPSComp { MatrixOverlapType soverlap = GetMatrixOverlap(vs, vd, sz); MatrixOverlapType toverlap = GetMatrixOverlap(vt, vd, sz); - if (UseAccurateVdot() || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { + if (UseAccurateVdot()) { // d[a][b] = s[b] dot t[a], each a row of n lanes and zeros, as in the interpreter. With // no overlap, the results can go straight to d. if (soverlap || toverlap) diff --git a/Core/MIPS/InterpreterVFPU.cpp b/Core/MIPS/InterpreterVFPU.cpp index 45331fc06c..333d9081bb 100644 --- a/Core/MIPS/InterpreterVFPU.cpp +++ b/Core/MIPS/InterpreterVFPU.cpp @@ -80,7 +80,7 @@ static const bool USE_VFPU_RESULT_FIXUP = false; // The dot products (vdot, vmmul, vtfm and the rest) through vfpu_dot(), bit-exact with the PSP. static inline bool AccurateVdot() { - return g_Config.bAccurateVdot; + return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot; } union FloatBits { @@ -514,8 +514,7 @@ namespace MIPSInt ReadMatrix(mips, s, sz, vs); ReadMatrix(mips, t, sz, vt); - // TODO: Always use the more accurate path in interpreter? - bool useAccurateDot = AccurateVdot() || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL; + bool useAccurateDot = AccurateVdot(); for (int a = 0; a < n; a++) { for (int b = 0; b < n; b++) { union { float f; uint32_t u; } sum = { 0.0f }; diff --git a/assets/compat.ini b/assets/compat.ini index acd0fc2ee8..0af7549b72 100644 --- a/assets/compat.ini +++ b/assets/compat.ini @@ -1100,9 +1100,8 @@ NPJH50559 = true NPEH00030 = true ULKS46191 = true -[MoreAccurateVMMUL] -# Fixes leg shaking in Tekken 6. The potential for slowdown in other games is large enough -# that we will not generally apply this accurate mode where not needed. +[ForceAccurateVdot] +# The VFPU's own dot products, even with AccurateVdot turned off. Fixes leg shaking in Tekken 6. ULUS10466 = true ULES01376 = true ULJS00224 = true From bc7afa884545e2a23ca2791d9c4cf98a9c2f20bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 20:28:52 +0200 Subject: [PATCH 14/17] ARM32 IR JIT: Call a NEON dot routine for the accurate dot products After clang's ARMv7 output for vfpu_dot_neon(), in Q8-Q15, which the register cache leaves alone. The end is done in 32-bit integers instead of 64-bit ones, checked bit-exact against vfpu_dot_finish(). The call site gathers the lanes into Q8 and Q9 and flushes whatever guest state is in R2, R3, R12 and LR, usually nothing. Inf and NaN go through a routine that calls vfpu_dot_reference(). Without NEON, Vec4VfpuDot still goes through the IR interpreter. Co-Authored-By: Claude Opus 5.5 --- Core/CMakeLists.txt | 1 + Core/Core.vcxproj | 1 + Core/Core.vcxproj.filters | 3 + Core/MIPS/ARM/ArmIRAsm.cpp | 3 + Core/MIPS/ARM/ArmIRCompVec.cpp | 26 +++- Core/MIPS/ARM/ArmIRJit.cpp | 7 +- Core/MIPS/ARM/ArmIRJit.h | 4 + Core/MIPS/ARM/ArmIRRegCache.cpp | 5 + Core/MIPS/ARM/ArmIRRegCache.h | 4 + Core/MIPS/ARM/ArmVfpuDot.h | 199 ++++++++++++++++++++++++++++ UWP/CoreUWP/CoreUWP.vcxproj | 1 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 1 + 12 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 Core/MIPS/ARM/ArmVfpuDot.h diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 8a8a78352b..34e9f16f7c 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -36,6 +36,7 @@ list(APPEND CoreExtra MIPS/ARM/ArmIRRegCache.cpp MIPS/ARM/ArmIRJit.h MIPS/ARM/ArmIRRegCache.h + MIPS/ARM/ArmVfpuDot.h ) list(APPEND CoreExtra diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 499dbdac51..b6a47a8d6f 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -1019,6 +1019,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 24a84a5cfd..f293f6ac42 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -1725,6 +1725,9 @@ MIPS\ARM + + MIPS\ARM + Font diff --git a/Core/MIPS/ARM/ArmIRAsm.cpp b/Core/MIPS/ARM/ArmIRAsm.cpp index d8755a52bd..e4a2d6c636 100644 --- a/Core/MIPS/ARM/ArmIRAsm.cpp +++ b/Core/MIPS/ARM/ArmIRAsm.cpp @@ -21,6 +21,7 @@ #include "Common/Log.h" #include "Common/MemoryUtil.h" #include "Core/CoreTiming.h" +#include "Core/MIPS/ARM/ArmVfpuDot.h" #include "Core/MemMap.h" #include "Core/MIPS/ARM/ArmIRJit.h" #include "Core/MIPS/ARM/ArmIRRegCache.h" @@ -100,6 +101,8 @@ void ArmJitBackend::GenerateFixedCode(MIPSState *mipsState) { B(R_LR); } + vfpuDot_ = GenerateArmVfpuDotCall(this, CTXREG, GenerateArmVfpuDotSpecial(this, CTXREG)); + hooks_.enterDispatcher = (IRNativeFuncNoArg)AlignCode16(); // Nine registers, so we're misaligned by 4 for calls, which the SUB fixes. diff --git a/Core/MIPS/ARM/ArmIRCompVec.cpp b/Core/MIPS/ARM/ArmIRCompVec.cpp index edb5be51e9..42bb52c7c8 100644 --- a/Core/MIPS/ARM/ArmIRCompVec.cpp +++ b/Core/MIPS/ARM/ArmIRCompVec.cpp @@ -18,8 +18,10 @@ #include "ppsspp_config.h" #if PPSSPP_ARCH(ARM) +#include "Common/CPUDetect.h" #include "Core/MIPS/ARM/ArmIRJit.h" #include "Core/MIPS/ARM/ArmIRRegCache.h" +#include "Core/MIPS/ARM/ArmVfpuDot.h" // This file contains compilation for Vec4 operations. // @@ -171,8 +173,11 @@ void ArmJitBackend::CompIR_VecHoriz(IRInst inst) { break; case IROp::Vec4VfpuDot: - // Through the IR interpreter for now. - CompIR_Generic(inst); + if (cpu_info.bNEON) { + CompVfpuDot(inst); + } else { + CompIR_Generic(inst); + } break; default: @@ -181,6 +186,23 @@ void ArmJitBackend::CompIR_VecHoriz(IRInst inst) { } } +void ArmJitBackend::CompVfpuDot(IRInst inst) { + regs_.Map(inst); + // The lanes into Q8 and Q9, through S0-S3 (Q0), since Q8-Q15 have no S registers. + for (int i = 0; i < 4; i++) + VMOV((ARMReg)(S0 + i), regs_.F(inst.src1 + i)); + VMOV_neon(Q8, Q0); + for (int i = 0; i < 4; i++) + VMOV((ARMReg)(S0 + i), regs_.F(inst.src2 + i)); + VMOV_neon(Q9, Q0); + + // The shared routine clobbers these too, so whatever guest state is there gets flushed. They come + // late in the allocation order, so usually that's nothing. + regs_.FlushHostRegs({ R2, R3, R12, R14 }); + BL(vfpuDot_); + VMOV(regs_.F(inst.dest), R3); +} + void ArmJitBackend::CompIR_VecPack(IRInst inst) { CONDITIONAL_DISABLE; diff --git a/Core/MIPS/ARM/ArmIRJit.cpp b/Core/MIPS/ARM/ArmIRJit.cpp index 605c39a180..a6f961e7f9 100644 --- a/Core/MIPS/ARM/ArmIRJit.cpp +++ b/Core/MIPS/ARM/ArmIRJit.cpp @@ -63,7 +63,12 @@ bool ArmJitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) { return false; IRBlock *block = irBlockCache->GetBlock(block_num); - BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32)); + // An accurate dot product gathers its lanes and can flush a few registers around its call. + size_t sizeEstimate = 0; + const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block); + for (int i = 0; i < block->GetNumIRInstructions(); ++i) + sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32; + BeginWrite(std::min(GetSpaceLeft(), sizeEstimate)); u32 startPC = block->GetOriginalStart(); bool wroteCheckedOffset = false; diff --git a/Core/MIPS/ARM/ArmIRJit.h b/Core/MIPS/ARM/ArmIRJit.h index 2c9951990e..56d08e6cf5 100644 --- a/Core/MIPS/ARM/ArmIRJit.h +++ b/Core/MIPS/ARM/ArmIRJit.h @@ -116,6 +116,8 @@ private: void CompFloatToIntWithRounding(ArmGen::ARMReg dest, ArmGen::ARMReg src, int rmode); // Puts the masked address in SCRATCH1, as an offset from MEMBASEREG. Destroys SCRATCH2. void PrepareSrc1Address(IRInst inst); + // Vec4VfpuDot with NEON (see ArmVfpuDot.h). + void CompVfpuDot(IRInst inst); JitOptions &jo; ArmIRRegCache regs_; @@ -127,6 +129,8 @@ private: const u8 *dispatcherNoCheck_ = nullptr; const u8 *restoreRoundingMode_ = nullptr; const u8 *applyRoundingMode_ = nullptr; + // See GenerateArmVfpuDotCall(). + const u8 *vfpuDot_ = nullptr; int jitStartOffset_ = 0; int compilingBlockNum_ = -1; diff --git a/Core/MIPS/ARM/ArmIRRegCache.cpp b/Core/MIPS/ARM/ArmIRRegCache.cpp index 9e2f3a31cf..4ce356e94e 100644 --- a/Core/MIPS/ARM/ArmIRRegCache.cpp +++ b/Core/MIPS/ARM/ArmIRRegCache.cpp @@ -71,6 +71,11 @@ const int *ArmIRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c } } +void ArmIRRegCache::FlushHostRegs(std::initializer_list regs) { + for (ARMReg r : regs) + FlushNativeReg((IRNativeReg)r); +} + void ArmIRRegCache::FlushBeforeCall() { // These are not preserved by calls. for (int r : { R2, R3, R12, R14 }) diff --git a/Core/MIPS/ARM/ArmIRRegCache.h b/Core/MIPS/ARM/ArmIRRegCache.h index 5508dd5ae2..e2cf8dda8f 100644 --- a/Core/MIPS/ARM/ArmIRRegCache.h +++ b/Core/MIPS/ARM/ArmIRRegCache.h @@ -18,6 +18,8 @@ #pragma once #include "ppsspp_config.h" + +#include #if PPSSPP_ARCH(ARM) #include "Common/ArmEmitter.h" @@ -58,6 +60,8 @@ public: ArmGen::ARMReg MapWithFPRTemp(const IRInst &inst); void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list regs); void FlushAll(bool gprs = true, bool fprs = true) override; ArmGen::ARMReg GetAndLockTempGPR(); diff --git a/Core/MIPS/ARM/ArmVfpuDot.h b/Core/MIPS/ARM/ArmVfpuDot.h new file mode 100644 index 0000000000..3a163d493c --- /dev/null +++ b/Core/MIPS/ARM/ArmVfpuDot.h @@ -0,0 +1,199 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/ArmEmitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// The core registers EmitArmVfpuDot() works in, all different. The result ends up in result. +struct ArmVfpuDotRegs { + ArmGen::ARMReg sum; + ArmGen::ARMReg ehi; + ArmGen::ARMReg sign; + ArmGen::ARMReg result; + ArmGen::ARMReg lz; + ArmGen::ARMReg temp; +}; + +// Emits vfpu_dot() of the Vec4s in Q8 (x) and Q9 (y): vfpu_dot_neon(), in ARMv7 NEON after what clang +// makes of it, with Q0 and Q8-Q15 as temps. The end is in 32-bit integers rather than +// vfpu_dot_finish()'s 64-bit ones (checked bit-exact against it): the significand goes to bit 30 +// instead of 62, the exponent is clamped to [-2, 254], and the zero test is on the result plus 2^24, so +// that it and the inf test are unsigned compares. Inf and NaN aren't handled: the returned branch is +// taken for them, with Q8 and Q9 intact. Clobbers the flags. +inline ArmGen::FixupBranch EmitArmVfpuDot(ArmGen::ARMXEmitter *e, const ArmVfpuDotRegs &r) { + using namespace ArmGen; + const ARMReg x = Q8, y = Q9; + + // Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF added, + // above any real exponent sum, so one maximum finds the alignment and the specials (q11 = e, + // q13 = normal). + e->MOVI2R(r.ehi, 0x7F800000); + e->VDUP(I_32, Q10, r.ehi); + e->VAND(Q11, x, Q10); + e->VAND(Q12, y, Q10); + e->VTST(I_32, Q13, x, Q10); + e->VTST(I_32, Q14, y, Q10); + e->VAND(Q13, Q13, Q14); + e->VMAX(I_32 | I_UNSIGNED, Q14, Q11, Q12); + e->VCEQ(I_32, Q14, Q14, Q10); + e->VADD(I_32, Q11, Q11, Q12); + e->VSHR(I_32 | I_UNSIGNED, Q11, Q11, 23); + e->VAND(Q11, Q11, Q13); + e->VMOV_imm(I_32, Q12, VIMM__x1__x1, 0x07); // 0x7FF + e->VAND(Q14, Q14, Q12); + e->VADD(I_32, Q11, Q11, Q14); + e->VPMAX(I_32 | I_UNSIGNED, D24, D22, D23); + e->VPMAX(I_32 | I_UNSIGNED, D24, D24, D24); + e->VMOV_neon(I_32, r.ehi, D24, 0); + // Real exponent sums are at most 508. + e->CMP(r.ehi, AssumeMakeOperand2(0x7F0)); + FixupBranch special = e->B_CC(CC_HS); + + // q0 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q10 = p). + e->VEOR(Q0, x, y); + e->VSHR(I_32 | I_SIGNED, Q0, Q0, 31); + e->VMOV_imm(I_32, Q10, VIMM_x11_x11, 0x7F); // 0x7FFFFF + e->VAND(Q15, x, Q10); + e->VORR_imm(I_32, Q15, VIMM_x___x__, 0x80); + e->VAND(Q14, y, Q10); + e->VORR_imm(I_32, Q14, VIMM_x___x__, 0x80); + e->VMULL(I_32 | I_UNSIGNED, Q8, D30, D28); + e->VMULL(I_32 | I_UNSIGNED, Q9, D31, D29); + e->VSHRN(I_64, D20, Q8, 21); + e->VSHRN(I_64, D21, Q9, 21); + e->VMOVN(I_64, D28, Q8); + e->VMOVN(I_64, D29, Q9); + e->VMOV_imm(I_32, Q15, VIMM_x11_x11, 0x1F); // (1 << 21) - 1 + e->VAND(Q14, Q14, Q15); + e->VMOV_imm(I_32, Q15, VIMM___x___x, 1); + e->VMIN(I_32 | I_UNSIGNED, Q14, Q14, Q15); + e->VORR(Q10, Q10, Q14); + e->VAND(Q10, Q10, Q13); + + // Align to the largest exponent by truncation, apply the signs, and sum exactly. + e->VDUP(I_32, Q12, D24, 0); + e->VSUB(I_32, Q11, Q11, Q12); + e->VMVN_imm(I_32, Q15, VIMM___x___x, 27); // -28 + e->VMAX(I_32 | I_SIGNED, Q11, Q11, Q15); + e->VSHL(I_32 | I_UNSIGNED, Q10, Q10, Q11); + e->VEOR(Q10, Q10, Q0); + e->VSUB(I_32, Q10, Q10, Q0); + e->VADD(I_32, D20, D20, D21); + e->VPADD(I_32, D20, D20, D20); + e->VMOV_neon(I_32, r.sum, D20, 0); + + // The end of vfpu_dot(): ehi = the largest exponent sum, sum = the sum, still scaled by 4. + e->AND(r.sign, r.sum, AssumeMakeOperand2(0x80000000)); + e->CMP(r.sum, Operand2(0)); + e->SetCC(CC_LT); + e->RSB(r.sum, r.sum, Operand2(0)); + e->SetCC(CC_AL); + e->MOV(r.result, Operand2(r.sum, ST_LSR, 2)); + e->ORR(r.lz, r.result, Operand2(1)); + e->CLZ(r.lz, r.lz); + // Round to 24 bits, nearest even, with the top bit at bit 30. + e->SUB(r.temp, r.lz, Operand2(1)); + e->MOV(r.result, Operand2(r.result, ST_LSL, r.temp)); + e->UBFX(r.temp, r.result, 7, 1); + e->ADD(r.result, r.result, r.temp); + e->ADD(r.result, r.result, Operand2(0x3F)); + e->MOV(r.result, Operand2(r.result, ST_LSR, 7)); + // The exponent, clamped to [-2, 254], which doesn't change the result. + e->SUB(r.ehi, r.ehi, r.lz); + e->SUB(r.ehi, r.ehi, Operand2(120)); + e->CMN(r.ehi, Operand2(2)); + e->SetCC(CC_LT); + e->MVN(r.ehi, Operand2(1)); + e->SetCC(CC_AL); + e->CMP(r.ehi, Operand2(254)); + e->SetCC(CC_GT); + e->MOV(r.ehi, Operand2(254)); + e->SetCC(CC_AL); + // Adding the significand, implicit bit and all, lets a carry from the rounding bump the exponent. + e->ADD(r.result, r.result, Operand2(r.ehi, ST_LSL, 23)); + e->ADD(r.temp, r.result, AssumeMakeOperand2(0x01000000)); + // 0x7F800000 isn't an immediate, so compare the exponent field. + e->MOV(r.lz, Operand2(r.result, ST_LSR, 23)); + e->CMP(r.lz, Operand2(0xFF)); + e->SetCC(CC_HS); + e->MOVI2R(r.result, 0x7F800000); + e->SetCC(CC_AL); + // Zero if the result is below the smallest normal, or the sum is. + e->CMP(r.temp, AssumeMakeOperand2(0x01800000)); + e->SetCC(CC_LO); + e->MOV(r.result, Operand2(0)); + e->SetCC(CC_AL); + e->CMP(r.sum, Operand2(4)); + e->SetCC(CC_LO); + e->MOV(r.result, Operand2(0)); + e->SetCC(CC_AL); + e->ORR(r.result, r.result, r.sign); + return special; +} + +static void ArmVfpuDotSpecialC(MIPSState *mips) { + const float *args = (const float *)mips->vfpuDotArgs; + const float result = vfpu_dot_reference(args, args + 4); + memcpy(&mips->vfpuDotResult, &result, sizeof(result)); +} + +// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. Called with the +// stack aligned to 8, and preserves everything that a call may clobber, but Q8-Q15. +inline const u8 *GenerateArmVfpuDotSpecial(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx) { + using namespace ArmGen; + const u8 *start = e->AlignCode16(); + e->PUSH(8, R0, R1, R2, R3, R4, R5, R12, R_LR); + e->VPUSH(D0, 8); + e->QuickCallFunctionR(R12, &ArmVfpuDotSpecialC, ctx); + e->VPOP(D0, 8); + e->POP(8, R0, R1, R2, R3, R4, R5, R12, R_PC); + return start; +} + +// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of Q8 and Q9, into R3, +// with special (from GenerateArmVfpuDotSpecial()) for inf and NaN. It clobbers Q0, Q8-Q15, R0-R3, R12, +// LR and the flags, and saves only R4: the call site flushes whatever of the others holds guest +// state, which in most blocks is nothing. Q8-Q15 are never allocated. +inline const u8 *GenerateArmVfpuDotCall(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx, const u8 *special) { + using namespace ArmGen; + const u8 *start = e->AlignCode16(); + e->PUSH(2, R4, R_LR); + const ArmVfpuDotRegs regs{ R0, R1, R2, R3, R12, R4 }; + FixupBranch specialBranch = EmitArmVfpuDot(e, regs); + e->POP(2, R4, R_PC); + + // The operands are still in Q8 and Q9. + e->SetJumpTarget(specialBranch); + e->ADDI2R(R0, ctx, offsetof(MIPSState, vfpuDotArgs), R1); + for (int i = 0; i < 4; i++) + e->VSTR((ARMReg)(D16 + i), R0, i * 8); + e->BL(special); + e->LDR(R3, ctx, offsetof(MIPSState, vfpuDotResult)); + e->POP(2, R4, R_PC); + return start; +} + +} // namespace MIPSComp diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index 82f9cd5f09..59badf30a4 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -281,6 +281,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index dffad90dd9..9d0f9ce361 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -600,6 +600,7 @@ + From 6207ef9ec71be157cce7d5ad19dba4f460267254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 20:28:52 +0200 Subject: [PATCH 15/17] x86-64 IR JIT: Call the dot routine with the operands in registers The operands go in xmm0 and xmm1 and the result comes back in edi. The routine saves nothing: the call site flushes whatever guest state is in the registers it clobbers, the xmms allocated last and rax, rcx, rdx, rsi and rdi, the last two never allocated outside Windows. 32-bit x86 keeps going through the IR interpreter. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/x86/X64IRCompVec.cpp | 40 +++++++- Core/MIPS/x86/X64IRRegCache.cpp | 7 ++ Core/MIPS/x86/X64IRRegCache.h | 4 + Core/MIPS/x86/X64VfpuDot.h | 176 +++++++++++++++----------------- 4 files changed, 127 insertions(+), 100 deletions(-) diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index c55ac7781f..a25a699f13 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -275,17 +275,47 @@ void X64JitBackend::CompIR_VecHoriz(IRInst inst) { } case IROp::Vec4VfpuDot: - // Through memory and the shared routine, which preserves every register. + { +#if PPSSPP_ARCH(AMD64) if (!vfpuDot_ || Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { CompIR_Generic(inst); break; } - regs_.Map(inst); - MOVUPS(MDisp(CTXREG, offsetof(MIPSState, vfpuDotArgs)), regs_.FX(inst.src1)); - MOVUPS(MDisp(CTXREG, offsetof(MIPSState, vfpuDotArgs) + 16), regs_.FX(inst.src2)); + // The operands go to the shared routine in xmm0 and xmm1, and the result comes back in edi. + // Whatever guest state is in the registers it clobbers gets flushed first: these come last in + // the allocation order, so usually that's nothing. Then again after the operands are moved, + // in case they were mapped into one of them. + auto flushClobbered = [&]() { + regs_.FlushHostRegs({ RCX, RDX, RSI, RDI }, { XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM14, XMM15 }); + }; + flushClobbered(); + regs_.SpillLockFPR(inst.src1, inst.src2); + const X64Reg x = regs_.MapVec4(inst.src1); + const X64Reg y = regs_.MapVec4(inst.src2); + if (x == XMM1 && y == XMM0) { + MOVAPS(XMM2, R(XMM0)); + MOVAPS(XMM0, R(XMM1)); + MOVAPS(XMM1, R(XMM2)); + } else if (y == XMM0) { + MOVAPS(XMM1, R(y)); + MOVAPS(XMM0, R(x)); + } else { + if (x != XMM0) + MOVAPS(XMM0, R(x)); + if (y != XMM1) + MOVAPS(XMM1, R(y)); + } + regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); + flushClobbered(); CALL(vfpuDot_); - MOVSS(regs_.FX(inst.dest), MDisp(CTXREG, offsetof(MIPSState, vfpuDotResult))); + regs_.MapFPR(inst.dest, MIPSMap::NOINIT); + MOVD_xmm(regs_.FX(inst.dest), R(EDI)); +#else + // 32-bit x86 has too few registers for the shared routine. + CompIR_Generic(inst); +#endif break; + } default: INVALIDOP; diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index f3644e2778..a04ee29773 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -119,6 +119,13 @@ const int *X64IRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c } } +void X64IRRegCache::FlushHostRegs(std::initializer_list gprs, std::initializer_list xmms) { + for (X64Reg r : gprs) + FlushNativeReg(GPRToNativeReg(r)); + for (X64Reg r : xmms) + FlushNativeReg(XMMToNativeReg(r)); +} + void X64IRRegCache::FlushBeforeCall() { // These registers are not preserved by function calls. #if PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/X64IRRegCache.h b/Core/MIPS/x86/X64IRRegCache.h index 8a21f563d8..16ea14cf2b 100644 --- a/Core/MIPS/x86/X64IRRegCache.h +++ b/Core/MIPS/x86/X64IRRegCache.h @@ -18,6 +18,8 @@ #pragma once #include "ppsspp_config.h" + +#include #if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) #include "Common/x64Emitter.h" @@ -95,6 +97,8 @@ public: // Note: may change the high lanes of single-register XMMs. void FlushAll(bool gprs = true, bool fprs = true) override; void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list gprs, std::initializer_list xmms); Gen::X64Reg GetAndLockTempGPR(); Gen::X64Reg GetAndLockTempFPR(); diff --git a/Core/MIPS/x86/X64VfpuDot.h b/Core/MIPS/x86/X64VfpuDot.h index ce4778f5f7..064957aef3 100644 --- a/Core/MIPS/x86/X64VfpuDot.h +++ b/Core/MIPS/x86/X64VfpuDot.h @@ -31,16 +31,14 @@ namespace MIPSComp { -// Generates the VFPU dot product routine the x86-64 JITs CALL: vfpu_dot() of the two Vec4s in -// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It's -// vfpu_dot_sse2(), SSE2 only, and leaves inf and NaN to vfpu_dot_reference(). Every register is -// preserved except the flags. +// Generates the VFPU dot product routine the x86-64 IR JIT CALLs: vfpu_dot() of xmm0 and xmm1, into +// edi. ctx holds the MIPSState pointer. It's vfpu_dot_sse2(), SSE2 only, and leaves inf and NaN to +// vfpu_dot_reference(). It clobbers xmm0-xmm5, xmm14, xmm15, rax, rcx, rdx, rsi, rdi and the flags, +// and saves nothing: the call site flushes whatever of those holds guest state. The xmms are the last +// ones allocated, and rsi and rdi are never allocated outside Windows. inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { using namespace Gen; const int args = (int)offsetof(MIPSState, vfpuDotArgs); - const int result = (int)offsetof(MIPSState, vfpuDotResult); - static const X64Reg saved[] = { RAX, RCX, RDX, RSI, RDI, R8, R10, R11 }; - const int numXmm = 8; // The constants, RIP-relative. auto constant4 = [&](u32 a, u32 b, u32 c, u32 d) { @@ -61,40 +59,33 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { const u8 *scaleBase = splat(127 + 28); const u8 *start = e->AlignCode16(); - for (X64Reg r : saved) - e->PUSH(r); - e->SUB(64, R(RSP), Imm32(16 * numXmm)); - for (int i = 0; i < numXmm; i++) - e->MOVUPS(MDisp(RSP, i * 16), (X64Reg)(XMM0 + i)); // xmm0 = x, xmm1 = y. Exponents: a zero or denormal operand makes the product zero (xmm4 = // flush), and inf or NaN gets 0x7FF, above any real exponent sum, so one maximum finds the // alignment and the specials (xmm5 = e). They all fit in 15 bits, so the 16-bit max does. - e->MOVUPS(XMM0, MDisp(ctx, args)); - e->MOVUPS(XMM1, MDisp(ctx, args + 16)); - e->PXOR(XMM7, R(XMM7)); + e->PXOR(XMM15, R(XMM15)); e->MOVDQA(XMM2, R(XMM0)); e->PAND(XMM2, M(expBits)); e->MOVDQA(XMM3, R(XMM1)); e->PAND(XMM3, M(expBits)); e->MOVDQA(XMM4, R(XMM2)); - e->PCMPEQD(XMM4, R(XMM7)); + e->PCMPEQD(XMM4, R(XMM15)); e->MOVDQA(XMM5, R(XMM3)); - e->PCMPEQD(XMM5, R(XMM7)); + e->PCMPEQD(XMM5, R(XMM15)); e->POR(XMM4, R(XMM5)); - e->MOVDQA(XMM6, R(XMM2)); - e->PCMPEQD(XMM6, M(expBits)); + e->MOVDQA(XMM14, R(XMM2)); + e->PCMPEQD(XMM14, M(expBits)); e->MOVDQA(XMM5, R(XMM3)); e->PCMPEQD(XMM5, M(expBits)); - e->POR(XMM6, R(XMM5)); - e->PSRLD(XMM6, 21); + e->POR(XMM14, R(XMM5)); + e->PSRLD(XMM14, 21); e->MOVDQA(XMM5, R(XMM2)); e->PADDD(XMM5, R(XMM3)); e->PSRLD(XMM5, 23); e->MOVDQA(XMM2, R(XMM4)); e->PANDN(XMM2, R(XMM5)); e->MOVDQA(XMM5, R(XMM2)); - e->POR(XMM5, R(XMM6)); + e->POR(XMM5, R(XMM14)); e->PSHUFD(XMM2, R(XMM5), 0x4E); e->PMAXSW(XMM2, R(XMM5)); e->PSHUFD(XMM3, R(XMM2), 0xB1); @@ -104,33 +95,35 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { FixupBranch special = e->J_CC(CC_AE, true); // 24x24-bit products, even and odd lanes separately, kept to 2 extra bits with round-to-odd - // (xmm3 = p). + // (xmm4 = p). xmm0 = the signs, while x and y are still around. e->MOVDQA(XMM3, R(XMM0)); e->PAND(XMM3, M(mantMask)); e->POR(XMM3, M(implicitBit)); - e->MOVDQA(XMM6, R(XMM1)); - e->PAND(XMM6, M(mantMask)); - e->POR(XMM6, M(implicitBit)); - e->MOVDQA(XMM7, R(XMM3)); - e->PMULUDQ(XMM7, R(XMM6)); // even products + e->MOVDQA(XMM14, R(XMM1)); + e->PAND(XMM14, M(mantMask)); + e->POR(XMM14, M(implicitBit)); + e->PXOR(XMM0, R(XMM1)); + e->PSRAD(XMM0, 31); + e->MOVDQA(XMM15, R(XMM3)); + e->PMULUDQ(XMM15, R(XMM14)); // even products e->PSRLQ(XMM3, 32); - e->PSRLQ(XMM6, 32); - e->PMULUDQ(XMM3, R(XMM6)); // odd products - e->MOVDQA(XMM6, R(XMM7)); - e->PSRLQ(XMM6, 21); - e->PAND(XMM7, M(lowMask)); + e->PSRLQ(XMM14, 32); + e->PMULUDQ(XMM3, R(XMM14)); // odd products + e->MOVDQA(XMM14, R(XMM15)); + e->PSRLQ(XMM14, 21); + e->PAND(XMM15, M(lowMask)); e->MOVDQA(XMM1, R(XMM3)); e->PSRLQ(XMM1, 21); e->PSLLQ(XMM1, 32); - e->POR(XMM6, R(XMM1)); // kept + e->POR(XMM14, R(XMM1)); // kept e->PAND(XMM3, M(lowMask)); e->PSLLQ(XMM3, 32); - e->POR(XMM3, R(XMM7)); // dropped - e->PXOR(XMM7, R(XMM7)); - e->PCMPEQD(XMM3, R(XMM7)); + e->POR(XMM3, R(XMM15)); // dropped + e->PXOR(XMM15, R(XMM15)); + e->PCMPEQD(XMM3, R(XMM15)); e->PANDN(XMM3, M(ones)); // sticky - e->POR(XMM3, R(XMM6)); - e->PANDN(XMM4, R(XMM3)); // p, in xmm4 + e->POR(XMM3, R(XMM14)); + e->PANDN(XMM4, R(XMM3)); // Align to the largest exponent by truncation. SSE2 has no per-lane shift, so p >> d is done // as (p * 2^(28 - d)) >> 28, with the power of two built as float bits and converted by @@ -153,80 +146,70 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { e->POR(XMM4, R(XMM5)); // Apply the signs and sum exactly. - e->MOVUPS(XMM1, MDisp(ctx, args + 16)); - e->PXOR(XMM0, R(XMM1)); - e->PSRAD(XMM0, 31); e->PXOR(XMM4, R(XMM0)); e->PSUBD(XMM4, R(XMM0)); e->PSHUFD(XMM0, R(XMM4), 0x4E); e->PADDD(XMM4, R(XMM0)); e->PSHUFD(XMM0, R(XMM4), 0xB1); e->PADDD(XMM4, R(XMM0)); - e->MOVD_xmm(R(R11), XMM4); - e->MOV(32, R(R10), R(EAX)); // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. - // ESI: the sign, EDI: the significand m, EDX: its leading zeros as 64 bits. - e->MOV(32, R(ESI), R(R11)); - e->AND(32, R(ESI), Imm32(0x80000000)); - e->MOV(32, R(EAX), R(R11)); + // esi = the sum, then its sign. edi = the largest exponent sum, then the result. edx = the + // significand m. ecx = 62 - its top bit, the shift that puts that at bit 62. + e->MOVD_xmm(R(ESI), XMM4); + e->MOV(32, R(EDI), R(EAX)); + e->MOV(32, R(EAX), R(ESI)); e->CDQ(); e->XOR(32, R(EAX), R(EDX)); e->SUB(32, R(EAX), R(EDX)); + e->AND(32, R(ESI), Imm32(0x80000000)); e->SHR(32, R(EAX), Imm8(2)); - e->MOV(32, R(EDI), R(EAX)); + e->MOV(32, R(EDX), R(EAX)); e->MOV(32, R(ECX), R(EAX)); e->OR(32, R(ECX), Imm32(1)); e->BSR(32, ECX, R(ECX)); - e->MOV(32, R(EDX), Imm32(63)); - e->SUB(32, R(EDX), R(ECX)); - e->LEA(32, ECX, MDisp(EDX, -1)); - e->MOV(32, R(EAX), R(EDI)); + e->NEG(32, R(ECX)); + e->ADD(32, R(ECX), Imm32(62)); e->SHL(64, R(RAX), R(CL)); - e->MOV(64, R(R8), R(RAX)); - e->SHR(64, R(R8), Imm8(39)); - e->AND(32, R(R8), Imm32(1)); - e->ADD(64, R(RAX), R(R8)); - e->MOV(64, R(R8), Imm64((1ULL << 38) - 1)); - e->ADD(64, R(RAX), R(R8)); + e->SUB(32, R(EDI), R(ECX)); + e->SUB(32, R(EDI), Imm32(89)); + e->MOV(64, R(RCX), R(RAX)); + e->SHR(64, R(RCX), Imm8(39)); + e->AND(32, R(ECX), Imm32(1)); + e->ADD(64, R(RAX), R(RCX)); + e->MOV(64, R(RCX), Imm64((1ULL << 38) - 1)); + e->ADD(64, R(RAX), R(RCX)); e->SHR(64, R(RAX), Imm8(39)); - e->MOV(32, R(R8), R(R10)); - e->SUB(32, R(R8), R(EDX)); - e->SUB(32, R(R8), Imm32(88)); - e->MOVSX(64, 32, R8, R(R8)); - e->SHL(64, R(R8), Imm8(23)); - e->ADD(64, R(R8), R(RAX)); - e->MOV(64, R(RAX), Imm64(0x7F800000)); - e->CMP(64, R(R8), R(RAX)); - e->CMOVcc(64, R8, R(RAX), CC_GE); - e->MOV(64, R(RAX), Imm64(0x00800000)); + e->MOVSX(64, 32, RDI, R(EDI)); + e->SHL(64, R(RDI), Imm8(23)); + e->ADD(64, R(RDI), R(RAX)); + e->MOV(32, R(EAX), Imm32(0x7F800000)); + e->CMP(64, R(RDI), R(RAX)); + e->CMOVcc(64, RDI, R(RAX), CC_GE); e->XOR(32, R(ECX), R(ECX)); - e->CMP(64, R(R8), R(RAX)); - e->CMOVcc(64, R8, R(RCX), CC_L); - e->TEST(32, R(EDI), R(EDI)); - e->CMOVcc(64, R8, R(RCX), CC_Z); - e->OR(32, R(R8), R(ESI)); - e->MOV(32, MDisp(ctx, result), R(R8)); - - const u8 *epilogue = e->GetCodePointer(); - for (int i = 0; i < numXmm; i++) - e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, i * 16)); - e->ADD(64, R(RSP), Imm32(16 * numXmm)); - for (int i = (int)(sizeof(saved) / sizeof(saved[0])) - 1; i >= 0; i--) - e->POP(saved[i]); + e->CMP(64, R(RDI), Imm32(0x00800000)); + e->CMOVcc(32, EDI, R(ECX), CC_L); + e->TEST(32, R(EDX), R(EDX)); + e->CMOVcc(32, EDI, R(ECX), CC_Z); + e->OR(32, R(EDI), R(ESI)); e->RET(); - // Inf or NaN: the C version, with the other registers a call may clobber saved too, on an - // aligned stack. + // Inf or NaN: the C version, with x and y still in xmm0 and xmm1. The call can clobber registers + // that the call site didn't flush, so those are saved, around an aligned stack. e->SetJumpTarget(special); + e->MOVUPS(MDisp(ctx, args), XMM0); + e->MOVUPS(MDisp(ctx, args + 16), XMM1); + e->PUSH(R8); e->PUSH(R9); - e->PUSH(RBX); - e->MOV(64, R(RBX), R(RSP)); + e->PUSH(R10); + e->PUSH(R11); + e->PUSH(RBP); + e->MOV(64, R(RBP), R(RSP)); e->AND(64, R(RSP), Imm32(-16)); - // The upper XMMs, and the shadow space Windows wants. + // xmm6-xmm13, and the shadow space Windows wants. e->SUB(64, R(RSP), Imm32(8 * 16 + 32)); - for (int i = 8; i < 16; i++) - e->MOVUPS(MDisp(RSP, 32 + (i - 8) * 16), (X64Reg)(XMM0 + i)); + for (int i = 6; i < 14; i++) + e->MOVUPS(MDisp(RSP, 32 + (i - 6) * 16), (X64Reg)(XMM0 + i)); #ifdef _WIN32 e->LEA(64, RCX, MDisp(ctx, args)); e->LEA(64, RDX, MDisp(ctx, args + 16)); @@ -236,13 +219,16 @@ inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { #endif e->MOV(64, R(RAX), ImmPtr((const void *)&vfpu_dot_reference)); e->CALLptr(R(RAX)); - e->MOVSS(MDisp(ctx, result), XMM0); - for (int i = 8; i < 16; i++) - e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, 32 + (i - 8) * 16)); - e->MOV(64, R(RSP), R(RBX)); - e->POP(RBX); + e->MOVD_xmm(R(EDI), XMM0); + for (int i = 6; i < 14; i++) + e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, 32 + (i - 6) * 16)); + e->MOV(64, R(RSP), R(RBP)); + e->POP(RBP); + e->POP(R11); + e->POP(R10); e->POP(R9); - e->JMP(epilogue, true); + e->POP(R8); + e->RET(); return start; } From 136e6ca927ffd03f864a0156a62f1625eec2c180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 21:45:18 +0200 Subject: [PATCH 16/17] IR: Include prefix constants past the vector size in the accurate dot products On the PSP, lanes past the vector size where both the S and T prefixes are constants add their product to vdot, and an S constant there is added in by vfad and vavg (cpu/vfpu/prefix_lanes). Without AccurateVdot, the IR leaves those to the interpreter; the accurate dot products now include them. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/IR/IRCompVFPU.cpp | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index 7a270c6423..016a1ebc81 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -829,10 +829,13 @@ namespace MIPSComp { int n = GetNumVectorElements(sz); // On the PSP, an S constant past n is added in too (cpu/vfpu/prefix_lanes), which - // IsPrefixWithinSize() lets through. Rare enough for the interpreter. - for (int i = n; i < 4; i++) { - if (VFPUPrefixIsConstant(js.prefixS, i)) - DISABLE; + // IsPrefixWithinSize() lets through. The accurate dot product includes it, the rest leave it + // to the interpreter, as it's rare. + if (!UseAccurateVdot()) { + for (int i = n; i < 4; i++) { + if (VFPUPrefixIsConstant(js.prefixS, i)) + DISABLE; + } } u8 sregs[4], dregs[1]; @@ -847,7 +850,8 @@ namespace MIPSComp { const float c = vavg ? vavgConst[n - 1] : 1.0f; DotLane sl[4], tl[4]; for (int i = 0; i < 4; i++) { - sl[i] = i < n ? DotReg(sregs[i]) : DotConst(0.0f); + const bool constantLane = i >= n && VFPUPrefixIsConstant(js.prefixS, i); + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane ? VFPUPrefixConstant(js.prefixS, i) : 0.0f); tl[i] = DotConst(c); } u8 sv, tv; @@ -894,11 +898,16 @@ namespace MIPSComp { int n = GetNumVectorElements(sz); // On the PSP, lanes past n where both prefixes are constants add their product, and the rest - // read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through, and they're - // rare enough for the interpreter. - for (int i = n; i < 4; i++) { - if (VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i)) - DISABLE; + // read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through. The accurate + // dot product includes them, the rest leave them to the interpreter, as they're rare. + auto constantLane = [&](int i) { + return i >= n && VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i); + }; + if (!UseAccurateVdot()) { + for (int i = n; i < 4; i++) { + if (constantLane(i)) + DISABLE; + } } // TODO: Force read one of them into regs? probably not. @@ -908,11 +917,10 @@ namespace MIPSComp { GetVectorRegsPrefixD(dregs, V_Single, vd); if (UseAccurateVdot()) { - // The prefixes stay within n lanes, so the rest are zero, as in the interpreter. DotLane sl[4], tl[4]; for (int i = 0; i < 4; i++) { - sl[i] = i < n ? DotReg(sregs[i]) : DotConst(0.0f); - tl[i] = i < n ? DotReg(tregs[i]) : DotConst(0.0f); + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixS, i) : 0.0f); + tl[i] = i < n ? DotReg(tregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixT, i) : 0.0f); } u8 sv, tv; if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) From 6757fdc12886d75527ca280054009b8cf4448c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 14 Oct 2026 23:10:59 +0200 Subject: [PATCH 17/17] x86-64 IR JIT: Address the dot routine's operands from CTXREG's offset CTXREG points at mips->f[0] in the x86-64 IR JIT, 128 bytes into MIPSState, but the dot routine's inf and NaN path stored its operands at offsetof(MIPSState, vfpuDotArgs) from it. It read them back from the same place, so the results were right, but it wrote over invalidateAll_ and 16 bytes past the end of mipsr4k. On Alpine that crashed the unit tests. Co-Authored-By: Claude Opus 5.5 --- Core/MIPS/x86/X64VfpuDot.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/MIPS/x86/X64VfpuDot.h b/Core/MIPS/x86/X64VfpuDot.h index 064957aef3..9f214a9662 100644 --- a/Core/MIPS/x86/X64VfpuDot.h +++ b/Core/MIPS/x86/X64VfpuDot.h @@ -32,13 +32,13 @@ namespace MIPSComp { // Generates the VFPU dot product routine the x86-64 IR JIT CALLs: vfpu_dot() of xmm0 and xmm1, into -// edi. ctx holds the MIPSState pointer. It's vfpu_dot_sse2(), SSE2 only, and leaves inf and NaN to -// vfpu_dot_reference(). It clobbers xmm0-xmm5, xmm14, xmm15, rax, rcx, rdx, rsi, rdi and the flags, -// and saves nothing: the call site flushes whatever of those holds guest state. The xmms are the last -// ones allocated, and rsi and rdi are never allocated outside Windows. +// edi. ctx points at mips->f[0], as CTXREG does. It's vfpu_dot_sse2(), SSE2 only, and leaves inf +// and NaN to vfpu_dot_reference(). It clobbers xmm0-xmm5, xmm14, xmm15, rax, rcx, rdx, rsi, rdi and +// the flags, and saves nothing: the call site flushes whatever of those holds guest state. The xmms +// are the last ones allocated, and rsi and rdi are never allocated outside Windows. inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { using namespace Gen; - const int args = (int)offsetof(MIPSState, vfpuDotArgs); + const int args = (int)(offsetof(MIPSState, vfpuDotArgs) - offsetof(MIPSState, f)); // The constants, RIP-relative. auto constant4 = [&](u32 a, u32 b, u32 c, u32 d) {