diff --git a/Common/Arm64Emitter.cpp b/Common/Arm64Emitter.cpp index fbab83833e..931cc23b26 100644 --- a/Common/Arm64Emitter.cpp +++ b/Common/Arm64Emitter.cpp @@ -3080,6 +3080,10 @@ void ARM64FloatEmitter::AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(0, 0, 3, Rd, Rn, Rm); } +void ARM64FloatEmitter::BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(0, 1, 3, Rd, Rn, Rm); +} void ARM64FloatEmitter::EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(1, 0, 3, Rd, Rn, Rm); @@ -3196,6 +3200,37 @@ void ARM64FloatEmitter::UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(1, EncodeSize(size), 0xC, Rd, Rn, Rm); } +void ARM64FloatEmitter::ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(0, EncodeSize(size), 0x10, Rd, Rn, Rm); +} +void ARM64FloatEmitter::SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x10, Rd, Rn, Rm); +} +void ARM64FloatEmitter::UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x14, Rd, Rn, Rm); +} +void ARM64FloatEmitter::USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + EmitThreeSame(1, EncodeSize(size), 0x8, Rd, Rn, Rm); +} +void ARM64FloatEmitter::UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + _assert_msg_(IsQuad(Rd) && !IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__); + Write32((1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} +void ARM64FloatEmitter::UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) +{ + _assert_msg_(IsQuad(Rd) && IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__); + Write32((1 << 30) | (1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} +void ARM64FloatEmitter::ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn) +{ + _assert_msg_(size < 64 && (size < 32 || IsQuad(Rn)), "%s: bad size", __FUNCTION__); + Write32((IsQuad(Rn) << 30) | (0xE << 24) | (EncodeSize(size) << 22) | (0x18 << 17) | (0x1B << 12) | (2 << 10) | (DecodeReg(Rn) << 5) | DecodeReg(Rd)); +} void ARM64FloatEmitter::SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm) { EmitThreeSame(0, EncodeSize(size), 0xD, Rd, Rn, Rm); @@ -3846,6 +3881,12 @@ void ARM64FloatEmitter::SHL(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { EmitShiftImm(IsQuad(Rd), false, imm >> 3, imm & 7, 0xA, Rd, Rn); } +void ARM64FloatEmitter::USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { + _assert_msg_(shift > 0 && shift <= dest_size, "%s shift amount out of range!", __FUNCTION__); + u32 imm = EncodeImmShiftRight(dest_size, shift); + EmitShiftImm(IsQuad(Rd), true, imm >> 3, imm & 7, 0x2, Rd, Rn); +} + void ARM64FloatEmitter::USHR(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) { _assert_msg_(shift < dest_size, "%s shift amount must less than the element size!", __FUNCTION__); u32 imm = EncodeImmShiftRight(dest_size, shift); diff --git a/Common/Arm64Emitter.h b/Common/Arm64Emitter.h index 19fef73745..28b0262c25 100644 --- a/Common/Arm64Emitter.h +++ b/Common/Arm64Emitter.h @@ -854,6 +854,8 @@ public: // Vector void AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Rn & ~Rm. + void BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void BSL(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void BIT(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); @@ -888,6 +890,17 @@ public: void UMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Integer vector add and subtract. + void ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Widening multiplies: Rd (a quad) gets the products of the lower (UMULL) or upper (UMULL2) + // halves of Rn and Rm, src_size bits each. + void UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + void UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); + // Sum of all lanes, into a scalar of the lane size. + void ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn); void SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); void SMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm); @@ -995,6 +1008,8 @@ public: void SHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SHLL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SHRN(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); + // Rd += Rn >> shift, per lane. + void USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); void SHRN2(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift); void SXTL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); void SXTL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn); diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 4e691f1a7b..34e9f16f7c 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -36,6 +36,7 @@ list(APPEND CoreExtra MIPS/ARM/ArmIRRegCache.cpp MIPS/ARM/ArmIRJit.h MIPS/ARM/ArmIRRegCache.h + MIPS/ARM/ArmVfpuDot.h ) list(APPEND CoreExtra @@ -50,6 +51,7 @@ list(APPEND CoreExtra MIPS/ARM64/Arm64IRJit.h MIPS/ARM64/Arm64IRRegCache.cpp MIPS/ARM64/Arm64IRRegCache.h + MIPS/ARM64/Arm64VfpuDot.h Util/DisArm64.cpp ) @@ -65,6 +67,7 @@ list(APPEND CoreExtra MIPS/x86/X64IRJit.h MIPS/x86/X64IRRegCache.cpp MIPS/x86/X64IRRegCache.h + MIPS/x86/X64VfpuDot.h ) list(APPEND CoreExtra diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 25dee0f601..de55ed159f 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -199,6 +199,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode", CmdLineMode::Headless}, {POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode", CmdLineMode::Headless}, {POFF(oldAtrac), CmdParamType::Bool, "old-atrac", '\0', "Use old ATRAC decoder"}, + {POFF(accurateVdot), CmdParamType::Bool, "accurate-vdot", '\0', "Compute VFPU dot products bit-exactly like the PSP (default; =false for the faster approximation)"}, {POFF(memStickInserted), CmdParamType::BoolInverse, "no-memstick", '\0', "Run with the memory stick removed"}, {POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application}, {POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless}, @@ -610,6 +611,10 @@ void CommandLineOptions::ApplyToConfig() const { g_Config.bUseOldAtrac = oldAtrac.value(); g_Config.DoNotSaveSetting(&g_Config.bUseOldAtrac); } + if (accurateVdot.has_value()) { + g_Config.bAccurateVdot = accurateVdot.value(); + g_Config.DoNotSaveSetting(&g_Config.bAccurateVdot); + } if (root.has_value()) { // No DoNotSaveSetting() here - mountRoot isn't an ordinary setting and never gets diff --git a/Core/CmdLine.h b/Core/CmdLine.h index c2a3814b22..66a0f9e9b4 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -166,6 +166,7 @@ struct CommandLineOptions { bool optionS = false; // a legacy option std::optional oldAtrac; + std::optional accurateVdot; // Headless options that may also be mildly useful in application mode std::optional resolutionScale; diff --git a/Core/Compatibility.cpp b/Core/Compatibility.cpp index 3d0d254129..c8159cc13c 100644 --- a/Core/Compatibility.cpp +++ b/Core/Compatibility.cpp @@ -114,7 +114,7 @@ void Compatibility::CheckSettings(IniFile &iniFile, const std::string &gameID) { CheckSetting(iniFile, gameID, "GoWFramerateHack60", &flags_.GoWFramerateHack60); CheckSetting(iniFile, gameID, "FramerateHack30", &flags_.FramerateHack30); CheckSetting(iniFile, gameID, "HideISOFiles", &flags_.HideISOFiles); - CheckSetting(iniFile, gameID, "MoreAccurateVMMUL", &flags_.MoreAccurateVMMUL); + CheckSetting(iniFile, gameID, "ForceAccurateVdot", &flags_.ForceAccurateVdot); CheckSetting(iniFile, gameID, "ForceSoftwareRenderer", &flags_.ForceSoftwareRenderer); CheckSetting(iniFile, gameID, "DarkStalkersPresentHack", &flags_.DarkStalkersPresentHack); CheckSetting(iniFile, gameID, "ReportSmallMemstick", &flags_.ReportSmallMemstick); diff --git a/Core/Compatibility.h b/Core/Compatibility.h index 7eee4e30ec..25c0957792 100644 --- a/Core/Compatibility.h +++ b/Core/Compatibility.h @@ -69,7 +69,7 @@ struct CompatFlags { bool GoWFramerateHack60; bool FramerateHack30; bool HideISOFiles; - bool MoreAccurateVMMUL; + bool ForceAccurateVdot; bool ForceSoftwareRenderer; bool DarkStalkersPresentHack; bool ReportSmallMemstick; diff --git a/Core/Config.cpp b/Core/Config.cpp index b8e59fd76c..5961e2d207 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -436,6 +436,7 @@ static const ConfigSetting cpuSettings[] = { ConfigSetting("IOTimingMethod", SETTING(g_Config, iIOTimingMethod), IOTIMING_FAST, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("FastMemoryAccess", SETTING(g_Config, bFastMemory), true, CfgFlag::PER_GAME), ConfigSetting("FunctionReplacements", SETTING(g_Config, bFuncReplacements), true, CfgFlag::PER_GAME | CfgFlag::REPORT), + ConfigSetting("AccurateVdot", SETTING(g_Config, bAccurateVdot), true, CfgFlag::PER_GAME | CfgFlag::REPORT), ConfigSetting("HideSlowWarnings", SETTING(g_Config, bHideSlowWarnings), false, CfgFlag::DEFAULT), ConfigSetting("HideStateWarnings", SETTING(g_Config, bHideStateWarnings), false, CfgFlag::DEFAULT), ConfigSetting("JitDisableFlags", SETTING(g_Config, uJitDisableFlags), (uint32_t)0, CfgFlag::PER_GAME), diff --git a/Core/Config.h b/Core/Config.h index 9458f6274a..5a2bcd0d62 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -235,6 +235,8 @@ public: bool bWow64WarningDismissed; bool bForceLagSync; bool bFuncReplacements; + // Compute the VFPU's dot products (vdot, vmmul, vtfm and the rest) bit-exactly like the PSP. + bool bAccurateVdot; bool bHideSlowWarnings; bool bHideStateWarnings; uint32_t uJitDisableFlags; diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 163be032c6..b6a47a8d6f 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -870,6 +870,7 @@ + @@ -1018,6 +1019,7 @@ + @@ -1063,6 +1065,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 9e17d9263a..f293f6ac42 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -1725,6 +1725,9 @@ MIPS\ARM + + MIPS\ARM + Font @@ -2157,6 +2160,9 @@ MIPS\x86 + + MIPS\x86 + MIPS\x86 @@ -2166,6 +2172,9 @@ MIPS\ARM64 + + MIPS\ARM64 + Util diff --git a/Core/MIPS/ARM/ArmIRAsm.cpp b/Core/MIPS/ARM/ArmIRAsm.cpp index d8755a52bd..e4a2d6c636 100644 --- a/Core/MIPS/ARM/ArmIRAsm.cpp +++ b/Core/MIPS/ARM/ArmIRAsm.cpp @@ -21,6 +21,7 @@ #include "Common/Log.h" #include "Common/MemoryUtil.h" #include "Core/CoreTiming.h" +#include "Core/MIPS/ARM/ArmVfpuDot.h" #include "Core/MemMap.h" #include "Core/MIPS/ARM/ArmIRJit.h" #include "Core/MIPS/ARM/ArmIRRegCache.h" @@ -100,6 +101,8 @@ void ArmJitBackend::GenerateFixedCode(MIPSState *mipsState) { B(R_LR); } + vfpuDot_ = GenerateArmVfpuDotCall(this, CTXREG, GenerateArmVfpuDotSpecial(this, CTXREG)); + hooks_.enterDispatcher = (IRNativeFuncNoArg)AlignCode16(); // Nine registers, so we're misaligned by 4 for calls, which the SUB fixes. diff --git a/Core/MIPS/ARM/ArmIRCompVec.cpp b/Core/MIPS/ARM/ArmIRCompVec.cpp index fe98a56139..42bb52c7c8 100644 --- a/Core/MIPS/ARM/ArmIRCompVec.cpp +++ b/Core/MIPS/ARM/ArmIRCompVec.cpp @@ -18,8 +18,10 @@ #include "ppsspp_config.h" #if PPSSPP_ARCH(ARM) +#include "Common/CPUDetect.h" #include "Core/MIPS/ARM/ArmIRJit.h" #include "Core/MIPS/ARM/ArmIRRegCache.h" +#include "Core/MIPS/ARM/ArmVfpuDot.h" // This file contains compilation for Vec4 operations. // @@ -170,12 +172,37 @@ void ArmJitBackend::CompIR_VecHoriz(IRInst inst) { VMOV(regs_.F(inst.dest), SCRATCHF1); break; + case IROp::Vec4VfpuDot: + if (cpu_info.bNEON) { + CompVfpuDot(inst); + } else { + CompIR_Generic(inst); + } + break; + default: INVALIDOP; break; } } +void ArmJitBackend::CompVfpuDot(IRInst inst) { + regs_.Map(inst); + // The lanes into Q8 and Q9, through S0-S3 (Q0), since Q8-Q15 have no S registers. + for (int i = 0; i < 4; i++) + VMOV((ARMReg)(S0 + i), regs_.F(inst.src1 + i)); + VMOV_neon(Q8, Q0); + for (int i = 0; i < 4; i++) + VMOV((ARMReg)(S0 + i), regs_.F(inst.src2 + i)); + VMOV_neon(Q9, Q0); + + // The shared routine clobbers these too, so whatever guest state is there gets flushed. They come + // late in the allocation order, so usually that's nothing. + regs_.FlushHostRegs({ R2, R3, R12, R14 }); + BL(vfpuDot_); + VMOV(regs_.F(inst.dest), R3); +} + void ArmJitBackend::CompIR_VecPack(IRInst inst) { CONDITIONAL_DISABLE; diff --git a/Core/MIPS/ARM/ArmIRJit.cpp b/Core/MIPS/ARM/ArmIRJit.cpp index 605c39a180..a6f961e7f9 100644 --- a/Core/MIPS/ARM/ArmIRJit.cpp +++ b/Core/MIPS/ARM/ArmIRJit.cpp @@ -63,7 +63,12 @@ bool ArmJitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) { return false; IRBlock *block = irBlockCache->GetBlock(block_num); - BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32)); + // An accurate dot product gathers its lanes and can flush a few registers around its call. + size_t sizeEstimate = 0; + const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block); + for (int i = 0; i < block->GetNumIRInstructions(); ++i) + sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32; + BeginWrite(std::min(GetSpaceLeft(), sizeEstimate)); u32 startPC = block->GetOriginalStart(); bool wroteCheckedOffset = false; diff --git a/Core/MIPS/ARM/ArmIRJit.h b/Core/MIPS/ARM/ArmIRJit.h index 2c9951990e..56d08e6cf5 100644 --- a/Core/MIPS/ARM/ArmIRJit.h +++ b/Core/MIPS/ARM/ArmIRJit.h @@ -116,6 +116,8 @@ private: void CompFloatToIntWithRounding(ArmGen::ARMReg dest, ArmGen::ARMReg src, int rmode); // Puts the masked address in SCRATCH1, as an offset from MEMBASEREG. Destroys SCRATCH2. void PrepareSrc1Address(IRInst inst); + // Vec4VfpuDot with NEON (see ArmVfpuDot.h). + void CompVfpuDot(IRInst inst); JitOptions &jo; ArmIRRegCache regs_; @@ -127,6 +129,8 @@ private: const u8 *dispatcherNoCheck_ = nullptr; const u8 *restoreRoundingMode_ = nullptr; const u8 *applyRoundingMode_ = nullptr; + // See GenerateArmVfpuDotCall(). + const u8 *vfpuDot_ = nullptr; int jitStartOffset_ = 0; int compilingBlockNum_ = -1; diff --git a/Core/MIPS/ARM/ArmIRRegCache.cpp b/Core/MIPS/ARM/ArmIRRegCache.cpp index 9e2f3a31cf..4ce356e94e 100644 --- a/Core/MIPS/ARM/ArmIRRegCache.cpp +++ b/Core/MIPS/ARM/ArmIRRegCache.cpp @@ -71,6 +71,11 @@ const int *ArmIRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c } } +void ArmIRRegCache::FlushHostRegs(std::initializer_list regs) { + for (ARMReg r : regs) + FlushNativeReg((IRNativeReg)r); +} + void ArmIRRegCache::FlushBeforeCall() { // These are not preserved by calls. for (int r : { R2, R3, R12, R14 }) diff --git a/Core/MIPS/ARM/ArmIRRegCache.h b/Core/MIPS/ARM/ArmIRRegCache.h index 5508dd5ae2..e2cf8dda8f 100644 --- a/Core/MIPS/ARM/ArmIRRegCache.h +++ b/Core/MIPS/ARM/ArmIRRegCache.h @@ -18,6 +18,8 @@ #pragma once #include "ppsspp_config.h" + +#include #if PPSSPP_ARCH(ARM) #include "Common/ArmEmitter.h" @@ -58,6 +60,8 @@ public: ArmGen::ARMReg MapWithFPRTemp(const IRInst &inst); void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list regs); void FlushAll(bool gprs = true, bool fprs = true) override; ArmGen::ARMReg GetAndLockTempGPR(); diff --git a/Core/MIPS/ARM/ArmVfpuDot.h b/Core/MIPS/ARM/ArmVfpuDot.h new file mode 100644 index 0000000000..3a163d493c --- /dev/null +++ b/Core/MIPS/ARM/ArmVfpuDot.h @@ -0,0 +1,199 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/ArmEmitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// The core registers EmitArmVfpuDot() works in, all different. The result ends up in result. +struct ArmVfpuDotRegs { + ArmGen::ARMReg sum; + ArmGen::ARMReg ehi; + ArmGen::ARMReg sign; + ArmGen::ARMReg result; + ArmGen::ARMReg lz; + ArmGen::ARMReg temp; +}; + +// Emits vfpu_dot() of the Vec4s in Q8 (x) and Q9 (y): vfpu_dot_neon(), in ARMv7 NEON after what clang +// makes of it, with Q0 and Q8-Q15 as temps. The end is in 32-bit integers rather than +// vfpu_dot_finish()'s 64-bit ones (checked bit-exact against it): the significand goes to bit 30 +// instead of 62, the exponent is clamped to [-2, 254], and the zero test is on the result plus 2^24, so +// that it and the inf test are unsigned compares. Inf and NaN aren't handled: the returned branch is +// taken for them, with Q8 and Q9 intact. Clobbers the flags. +inline ArmGen::FixupBranch EmitArmVfpuDot(ArmGen::ARMXEmitter *e, const ArmVfpuDotRegs &r) { + using namespace ArmGen; + const ARMReg x = Q8, y = Q9; + + // Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF added, + // above any real exponent sum, so one maximum finds the alignment and the specials (q11 = e, + // q13 = normal). + e->MOVI2R(r.ehi, 0x7F800000); + e->VDUP(I_32, Q10, r.ehi); + e->VAND(Q11, x, Q10); + e->VAND(Q12, y, Q10); + e->VTST(I_32, Q13, x, Q10); + e->VTST(I_32, Q14, y, Q10); + e->VAND(Q13, Q13, Q14); + e->VMAX(I_32 | I_UNSIGNED, Q14, Q11, Q12); + e->VCEQ(I_32, Q14, Q14, Q10); + e->VADD(I_32, Q11, Q11, Q12); + e->VSHR(I_32 | I_UNSIGNED, Q11, Q11, 23); + e->VAND(Q11, Q11, Q13); + e->VMOV_imm(I_32, Q12, VIMM__x1__x1, 0x07); // 0x7FF + e->VAND(Q14, Q14, Q12); + e->VADD(I_32, Q11, Q11, Q14); + e->VPMAX(I_32 | I_UNSIGNED, D24, D22, D23); + e->VPMAX(I_32 | I_UNSIGNED, D24, D24, D24); + e->VMOV_neon(I_32, r.ehi, D24, 0); + // Real exponent sums are at most 508. + e->CMP(r.ehi, AssumeMakeOperand2(0x7F0)); + FixupBranch special = e->B_CC(CC_HS); + + // q0 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q10 = p). + e->VEOR(Q0, x, y); + e->VSHR(I_32 | I_SIGNED, Q0, Q0, 31); + e->VMOV_imm(I_32, Q10, VIMM_x11_x11, 0x7F); // 0x7FFFFF + e->VAND(Q15, x, Q10); + e->VORR_imm(I_32, Q15, VIMM_x___x__, 0x80); + e->VAND(Q14, y, Q10); + e->VORR_imm(I_32, Q14, VIMM_x___x__, 0x80); + e->VMULL(I_32 | I_UNSIGNED, Q8, D30, D28); + e->VMULL(I_32 | I_UNSIGNED, Q9, D31, D29); + e->VSHRN(I_64, D20, Q8, 21); + e->VSHRN(I_64, D21, Q9, 21); + e->VMOVN(I_64, D28, Q8); + e->VMOVN(I_64, D29, Q9); + e->VMOV_imm(I_32, Q15, VIMM_x11_x11, 0x1F); // (1 << 21) - 1 + e->VAND(Q14, Q14, Q15); + e->VMOV_imm(I_32, Q15, VIMM___x___x, 1); + e->VMIN(I_32 | I_UNSIGNED, Q14, Q14, Q15); + e->VORR(Q10, Q10, Q14); + e->VAND(Q10, Q10, Q13); + + // Align to the largest exponent by truncation, apply the signs, and sum exactly. + e->VDUP(I_32, Q12, D24, 0); + e->VSUB(I_32, Q11, Q11, Q12); + e->VMVN_imm(I_32, Q15, VIMM___x___x, 27); // -28 + e->VMAX(I_32 | I_SIGNED, Q11, Q11, Q15); + e->VSHL(I_32 | I_UNSIGNED, Q10, Q10, Q11); + e->VEOR(Q10, Q10, Q0); + e->VSUB(I_32, Q10, Q10, Q0); + e->VADD(I_32, D20, D20, D21); + e->VPADD(I_32, D20, D20, D20); + e->VMOV_neon(I_32, r.sum, D20, 0); + + // The end of vfpu_dot(): ehi = the largest exponent sum, sum = the sum, still scaled by 4. + e->AND(r.sign, r.sum, AssumeMakeOperand2(0x80000000)); + e->CMP(r.sum, Operand2(0)); + e->SetCC(CC_LT); + e->RSB(r.sum, r.sum, Operand2(0)); + e->SetCC(CC_AL); + e->MOV(r.result, Operand2(r.sum, ST_LSR, 2)); + e->ORR(r.lz, r.result, Operand2(1)); + e->CLZ(r.lz, r.lz); + // Round to 24 bits, nearest even, with the top bit at bit 30. + e->SUB(r.temp, r.lz, Operand2(1)); + e->MOV(r.result, Operand2(r.result, ST_LSL, r.temp)); + e->UBFX(r.temp, r.result, 7, 1); + e->ADD(r.result, r.result, r.temp); + e->ADD(r.result, r.result, Operand2(0x3F)); + e->MOV(r.result, Operand2(r.result, ST_LSR, 7)); + // The exponent, clamped to [-2, 254], which doesn't change the result. + e->SUB(r.ehi, r.ehi, r.lz); + e->SUB(r.ehi, r.ehi, Operand2(120)); + e->CMN(r.ehi, Operand2(2)); + e->SetCC(CC_LT); + e->MVN(r.ehi, Operand2(1)); + e->SetCC(CC_AL); + e->CMP(r.ehi, Operand2(254)); + e->SetCC(CC_GT); + e->MOV(r.ehi, Operand2(254)); + e->SetCC(CC_AL); + // Adding the significand, implicit bit and all, lets a carry from the rounding bump the exponent. + e->ADD(r.result, r.result, Operand2(r.ehi, ST_LSL, 23)); + e->ADD(r.temp, r.result, AssumeMakeOperand2(0x01000000)); + // 0x7F800000 isn't an immediate, so compare the exponent field. + e->MOV(r.lz, Operand2(r.result, ST_LSR, 23)); + e->CMP(r.lz, Operand2(0xFF)); + e->SetCC(CC_HS); + e->MOVI2R(r.result, 0x7F800000); + e->SetCC(CC_AL); + // Zero if the result is below the smallest normal, or the sum is. + e->CMP(r.temp, AssumeMakeOperand2(0x01800000)); + e->SetCC(CC_LO); + e->MOV(r.result, Operand2(0)); + e->SetCC(CC_AL); + e->CMP(r.sum, Operand2(4)); + e->SetCC(CC_LO); + e->MOV(r.result, Operand2(0)); + e->SetCC(CC_AL); + e->ORR(r.result, r.result, r.sign); + return special; +} + +static void ArmVfpuDotSpecialC(MIPSState *mips) { + const float *args = (const float *)mips->vfpuDotArgs; + const float result = vfpu_dot_reference(args, args + 4); + memcpy(&mips->vfpuDotResult, &result, sizeof(result)); +} + +// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. Called with the +// stack aligned to 8, and preserves everything that a call may clobber, but Q8-Q15. +inline const u8 *GenerateArmVfpuDotSpecial(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx) { + using namespace ArmGen; + const u8 *start = e->AlignCode16(); + e->PUSH(8, R0, R1, R2, R3, R4, R5, R12, R_LR); + e->VPUSH(D0, 8); + e->QuickCallFunctionR(R12, &ArmVfpuDotSpecialC, ctx); + e->VPOP(D0, 8); + e->POP(8, R0, R1, R2, R3, R4, R5, R12, R_PC); + return start; +} + +// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of Q8 and Q9, into R3, +// with special (from GenerateArmVfpuDotSpecial()) for inf and NaN. It clobbers Q0, Q8-Q15, R0-R3, R12, +// LR and the flags, and saves only R4: the call site flushes whatever of the others holds guest +// state, which in most blocks is nothing. Q8-Q15 are never allocated. +inline const u8 *GenerateArmVfpuDotCall(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx, const u8 *special) { + using namespace ArmGen; + const u8 *start = e->AlignCode16(); + e->PUSH(2, R4, R_LR); + const ArmVfpuDotRegs regs{ R0, R1, R2, R3, R12, R4 }; + FixupBranch specialBranch = EmitArmVfpuDot(e, regs); + e->POP(2, R4, R_PC); + + // The operands are still in Q8 and Q9. + e->SetJumpTarget(specialBranch); + e->ADDI2R(R0, ctx, offsetof(MIPSState, vfpuDotArgs), R1); + for (int i = 0; i < 4; i++) + e->VSTR((ARMReg)(D16 + i), R0, i * 8); + e->BL(special); + e->LDR(R3, ctx, offsetof(MIPSState, vfpuDotResult)); + e->POP(2, R4, R_PC); + return start; +} + +} // namespace MIPSComp diff --git a/Core/MIPS/ARM64/Arm64IRAsm.cpp b/Core/MIPS/ARM64/Arm64IRAsm.cpp index 07296dbe11..a4c6aa0ec4 100644 --- a/Core/MIPS/ARM64/Arm64IRAsm.cpp +++ b/Core/MIPS/ARM64/Arm64IRAsm.cpp @@ -22,6 +22,7 @@ #include "Common/Log.h" #include "Core/CoreTiming.h" #include "Core/MemMap.h" +#include "Core/MIPS/ARM64/Arm64VfpuDot.h" #include "Core/MIPS/ARM64/Arm64IRJit.h" #include "Core/MIPS/ARM64/Arm64IRRegCache.h" #include "Core/MIPS/JitCommon/JitCommon.h" @@ -264,6 +265,9 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) { RET(); } + vfpuDotSpecial_ = GenerateArm64VfpuDotSpecial(this, &fp_, CTXREG); + vfpuDot_ = GenerateArm64VfpuDotCall(this, &fp_, CTXREG, vfpuDotSpecial_); + // Leave this at the end, add more stuff above. if (enableDisasm) { std::vector lines = DisassembleArm64(disasmStart, (int)(GetCodePtr() - disasmStart)); diff --git a/Core/MIPS/ARM64/Arm64IRCompVec.cpp b/Core/MIPS/ARM64/Arm64IRCompVec.cpp index 45924a088e..ca308caa33 100644 --- a/Core/MIPS/ARM64/Arm64IRCompVec.cpp +++ b/Core/MIPS/ARM64/Arm64IRCompVec.cpp @@ -24,6 +24,7 @@ #include "Core/MemMap.h" #include "Core/MIPS/ARM64/Arm64IRJit.h" #include "Core/MIPS/ARM64/Arm64IRRegCache.h" +#include "Core/MIPS/ARM64/Arm64VfpuDot.h" // This file contains compilation for vector instructions. // @@ -599,6 +600,32 @@ void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + { + // The operands go to the shared routine in q0 and q1, and the result comes back in w1. The + // routine clobbers q4-q6 and x0-x3 too, so whatever guest state is there gets flushed: these + // come late in the allocation order, so usually that's nothing. + regs_.SpillLockFPR(inst.src1, inst.src2); + regs_.MapVec4(inst.src1); + regs_.MapVec4(inst.src2); + fp_.MOV(Q0, regs_.FQ(inst.src1)); + fp_.MOV(Q1, regs_.FQ(inst.src2)); + regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); + regs_.FlushHostRegs({ Q4, Q5, Q6, W0, W1, W2, W3 }); + BL(vfpuDot_); + fp_.FMOV(SCRATCHF1, W1); + + if (Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { + // The dest lane is in a mapped Vec4. + regs_.MapVec4(inst.dest & ~3, MIPSMap::DIRTY); + fp_.INS(32, regs_.FQ(inst.dest & ~3), inst.dest & 3, EncodeRegToQuad(SCRATCHF1), 0); + } else { + regs_.MapFPR(inst.dest, MIPSMap::NOINIT); + fp_.FMOV(regs_.F(inst.dest), SCRATCHF1); + } + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/ARM64/Arm64IRJit.cpp b/Core/MIPS/ARM64/Arm64IRJit.cpp index 56b0adbc93..ca4136c020 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.cpp +++ b/Core/MIPS/ARM64/Arm64IRJit.cpp @@ -71,7 +71,12 @@ bool Arm64JitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) { return false; IRBlock *block = irBlockCache->GetBlock(block_num); - BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32)); + // An accurate dot product can flush several registers around its call. + size_t sizeEstimate = 0; + const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block); + for (int i = 0; i < block->GetNumIRInstructions(); ++i) + sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32; + BeginWrite(std::min(GetSpaceLeft(), sizeEstimate)); u32 startPC = block->GetOriginalStart(); bool wroteCheckedOffset = false; diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index d6efefd516..20d428ae26 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -145,6 +145,10 @@ private: // Indexed by FPCR FZ:RN bits for convenience. Uses SCRATCH2. const u8 *convertS0ToSCRATCH1_[8]; + // See GenerateArm64VfpuDotSpecial(). + const u8 *vfpuDotSpecial_ = nullptr; + // See GenerateArm64VfpuDotCall(). + const u8 *vfpuDot_ = nullptr; // Note: mutable state used at runtime. const u8 *currentRoundingFunc_ = nullptr; diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.cpp b/Core/MIPS/ARM64/Arm64IRRegCache.cpp index f7bea20c00..556fa4ea39 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.cpp +++ b/Core/MIPS/ARM64/Arm64IRRegCache.cpp @@ -147,6 +147,15 @@ void Arm64IRRegCache::EmitSaveStaticRegisters() { } } +void Arm64IRRegCache::FlushHostRegs(std::initializer_list regs) { + for (ARM64Reg r : regs) { + if (IsVector(r)) + FlushNativeReg(VFPToNativeReg(EncodeRegToSingle(r))); + else + FlushNativeReg(GPRToNativeReg(DecodeReg(r))); + } +} + void Arm64IRRegCache::FlushBeforeCall() { // These registers are not preserved by function calls. auto isGPRSaved = [&](IRNativeReg nreg) { diff --git a/Core/MIPS/ARM64/Arm64IRRegCache.h b/Core/MIPS/ARM64/Arm64IRRegCache.h index f3f0b58b6f..d444913d97 100644 --- a/Core/MIPS/ARM64/Arm64IRRegCache.h +++ b/Core/MIPS/ARM64/Arm64IRRegCache.h @@ -18,6 +18,9 @@ #pragma once #include "ppsspp_config.h" + +#include + // In other words, PPSSPP_ARCH(ARM64) || DISASM_ALL. #if PPSSPP_ARCH(ARM64) || (PPSSPP_PLATFORM(WINDOWS) && !defined(__LIBRETRO__)) @@ -65,6 +68,8 @@ public: Arm64Gen::ARM64Reg MapWithFPRTemp(const IRInst &inst); void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list regs); void FlushAll(bool gprs = true, bool fprs = true) override; Arm64Gen::ARM64Reg GetAndLockTempGPR(); diff --git a/Core/MIPS/ARM64/Arm64VfpuDot.h b/Core/MIPS/ARM64/Arm64VfpuDot.h new file mode 100644 index 0000000000..ce8d64601f --- /dev/null +++ b/Core/MIPS/ARM64/Arm64VfpuDot.h @@ -0,0 +1,193 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/Arm64Emitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// The registers EmitArm64VfpuDot() works in. x and y are only read. q are seven vector temps and w +// six GPR temps, none of them x or y, except that q[0] and q[1] may be x and y themselves. +struct Arm64VfpuDotRegs { + Arm64Gen::ARM64Reg x; + Arm64Gen::ARM64Reg y; + Arm64Gen::ARM64Reg q[7]; + Arm64Gen::ARM64Reg w[6]; +}; + +// Emits vfpu_dot() of the Vec4s x and y: vfpu_dot_neon(), scheduled after what clang makes of it. +// The result ends up in w[3]. Inf and NaN aren't handled: the returned branch is taken for them, +// and the caller sends those to GenerateArm64VfpuDotSpecial()'s routine. Clobbers the flags. +inline Arm64Gen::FixupBranch EmitArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, const Arm64VfpuDotRegs &r) { + using namespace Arm64Gen; + const ARM64Reg x = EncodeRegToQuad(r.x); + const ARM64Reg y = EncodeRegToQuad(r.y); + ARM64Reg q[7]; + for (int i = 0; i < 7; i++) + q[i] = EncodeRegToQuad(r.q[i]); + const ARM64Reg emax = DecodeReg(r.w[0]); + const ARM64Reg sum = DecodeReg(r.w[1]); + const ARM64Reg sign = DecodeReg(r.w[2]); + const ARM64Reg m = DecodeReg(r.w[3]); + const ARM64Reg t2 = DecodeReg(r.w[4]); + const ARM64Reg t3 = DecodeReg(r.w[5]); + + // Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF + // added, above any real exponent sum, so one maximum finds the alignment and the specials + // (q3 = e, q5 = normal). + e->MOVI2R(emax, 0x7F800000); + fp->DUP(32, q[2], emax); + fp->AND(q[3], x, q[2]); + fp->AND(q[4], y, q[2]); + fp->CMTST(32, q[5], x, q[2]); + fp->CMEQ(32, q[6], q[4]); + fp->BIC(q[5], q[5], q[6]); + fp->UMAX(32, q[6], q[3], q[4]); + fp->CMEQ(32, q[6], q[6], q[2]); + fp->ADD(32, q[3], q[3], q[4]); + fp->USHR(32, q[3], q[3], 23); + fp->AND(q[3], q[3], q[5]); + fp->MOVI(32, q[2], 0x07, 8, true); // 0x7FF + fp->AND(q[6], q[6], q[2]); + fp->ADD(32, q[3], q[3], q[6]); + fp->UMAXP(32, q[2], q[3], q[3]); + fp->UMAXP(32, q[2], q[2], q[2]); + fp->UMOV(32, emax, q[2], 0); + e->CMPI2R(emax, 0x7FF); + FixupBranch special = e->B(CC_HS); + + // q4 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q1 = p). + fp->EOR(q[4], x, y); + fp->SSHR(32, q[4], q[4], 31); + fp->MOVI(32, q[6], 0x7F, 16, true); // 0x7FFFFF + fp->AND(q[0], x, q[6]); + fp->ORR(32, q[0], 0x80, 16); + fp->AND(q[1], y, q[6]); + fp->ORR(32, q[1], 0x80, 16); + fp->UMULL(32, q[6], EncodeRegToDouble(q[0]), EncodeRegToDouble(q[1])); + fp->UMULL2(32, q[0], q[0], q[1]); + fp->SHRN(32, EncodeRegToDouble(q[1]), q[6], 21); + fp->SHRN2(32, q[1], q[0], 21); + fp->UZP1(32, q[6], q[6], q[0]); + fp->MOVI(32, q[0], 0x1F, 16, true); // (1 << 21) - 1 + fp->AND(q[6], q[6], q[0]); + fp->MOVI(32, q[0], 1); + fp->UMIN(32, q[6], q[6], q[0]); + fp->ORR(q[1], q[1], q[6]); + fp->AND(q[1], q[1], q[5]); + + // Align to the largest exponent by truncation, apply the signs, and sum exactly. + fp->DUP(32, q[2], q[2], 0); + fp->SUB(32, q[3], q[3], q[2]); + fp->MVNI(32, q[2], 27); // -28 + fp->SMAX(32, q[3], q[3], q[2]); + fp->USHL(32, q[1], q[1], q[3]); + fp->EOR(q[1], q[1], q[4]); + fp->SUB(32, q[1], q[1], q[4]); + fp->ADDV(32, EncodeRegToSingle(q[1]), q[1]); + fp->UMOV(32, sum, q[1], 0); + + // vfpu_dot_finish(): emax = the largest exponent sum, sum = the sum. + e->ANDI2R(sign, sum, 0x80000000); + e->CMPI2R(sum, 0); + e->CSNEG(sum, sum, sum, CC_GE); + e->LSR(m, sum, 2); + e->ORRI2R(t2, m, 1); + e->CLZ(t2, t2); + e->ADDI2R(t3, t2, 31); + e->LSLV(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->UBFX(EncodeRegTo64(t3), EncodeRegTo64(m), 39, 1); + e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->MOVI2R(EncodeRegTo64(t3), (1ULL << 38) - 1); + e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3)); + e->LSR(EncodeRegTo64(m), EncodeRegTo64(m), 39); + e->SUB(emax, emax, t2); + e->SUBI2R(emax, emax, 120); + e->SXTW(EncodeRegTo64(emax), emax); + e->ADD(EncodeRegTo64(emax), EncodeRegTo64(m), EncodeRegTo64(emax), ArithOption(EncodeRegTo64(emax), ST_LSL, 23)); + e->MOVI2R(m, 0x7F7FFFFF); + e->CMP(EncodeRegTo64(emax), EncodeRegTo64(m)); + e->MOVI2R(m, 0x7F800000); + e->CSEL(m, m, emax, CC_GT); + // Zero if the result is below the smallest normal, or the sum (still scaled by 4) is. + e->MOVI2R(t2, 0x00800000); + e->CMP(EncodeRegTo64(emax), EncodeRegTo64(t2)); + e->CCMP(sum, 4, 0, CC_GE); + e->CSEL(m, WZR, m, CC_LO); + e->ORR(m, m, sign); + return special; +} + +// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in +// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It clobbers +// q0-q3, x16, x17, x30 and the flags, which the IR JIT keeps free of guest state, and preserves +// everything else. +inline const u8 *GenerateArm64VfpuDotSpecial(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) { + using namespace Arm64Gen; + const s32 args = (s32)offsetof(MIPSState, vfpuDotArgs); + const s32 result = (s32)offsetof(MIPSState, vfpuDotResult); + + const u8 *start = e->AlignCode16(); + for (int i = 0; i < 16; i += 2) + e->STP(INDEX_PRE, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, -16); + e->STP(INDEX_PRE, X18, X30, SP, -16); + for (int i = 4; i < 32; i += 2) + fp->STP(128, INDEX_PRE, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, -32); + e->ADDI2R(X0, ctx, args); + e->ADDI2R(X1, ctx, args + 16); + e->MOVP2R(X16, &vfpu_dot_reference); + e->BLR(X16); + fp->STR(32, INDEX_UNSIGNED, S0, ctx, result); + for (int i = 30; i >= 4; i -= 2) + fp->LDP(128, INDEX_POST, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, 32); + e->LDP(INDEX_POST, X18, X30, SP, 16); + for (int i = 14; i >= 0; i -= 2) + e->LDP(INDEX_POST, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, 16); + e->RET(); + return start; +} + +// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of q0 and q1, into w1, +// with special (from GenerateArm64VfpuDotSpecial()) for inf and NaN. It clobbers q0-q6, x0-x3, x16, +// x17, x30 and the flags, and saves nothing: the call site flushes whatever of those holds guest +// state, which in most blocks is nothing. +inline const u8 *GenerateArm64VfpuDotCall(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx, const u8 *special) { + using namespace Arm64Gen; + const u8 *start = e->AlignCode16(); + const Arm64VfpuDotRegs regs{ Q0, Q1, { Q0, Q1, Q2, Q3, Q4, Q5, Q6 }, { W16, W17, W0, W1, W2, W3 } }; + FixupBranch specialBranch = EmitArm64VfpuDot(e, fp, regs); + e->RET(); + + // The operands are still in q0 and q1. + e->SetJumpTarget(specialBranch); + e->ADDI2R(X16, ctx, offsetof(MIPSState, vfpuDotArgs)); + fp->STP(128, INDEX_SIGNED, Q0, Q1, X16, 0); + e->STR(INDEX_PRE, X30, SP, -16); + e->BL(special); + e->LDR(INDEX_POST, X30, SP, 16); + e->LDR(INDEX_UNSIGNED, W1, ctx, offsetof(MIPSState, vfpuDotResult)); + e->RET(); + return start; +} + +} // namespace MIPSComp diff --git a/Core/MIPS/IR/IRCompVFPU.cpp b/Core/MIPS/IR/IRCompVFPU.cpp index ac8003b9ec..016a1ebc81 100644 --- a/Core/MIPS/IR/IRCompVFPU.cpp +++ b/Core/MIPS/IR/IRCompVFPU.cpp @@ -150,6 +150,89 @@ namespace MIPSComp { return true; } + bool IRFrontend::UseAccurateVdot() const { + return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot; + } + + // The lanes are already a Vec4 reg, used as is. + static bool DotLanesAreVec4(const IRFrontend::DotLane lanes[4]) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg < 0 || lanes[i].negate || lanes[i].reg != lanes[0].reg + i) + return false; + } + return (lanes[0].reg & 3) == 0; + } + + static bool DotLanesRead(const IRFrontend::DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg >= vec && lanes[i].reg < vec + 4) + return true; + } + return false; + } + + // Building lane i into vec + i only clobbers what a later lane still has to read. + static bool DotLanesBuildableIn(const IRFrontend::DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + for (int j = i + 1; j < 4; j++) { + if (lanes[j].reg == vec + i) + return false; + } + } + return true; + } + + bool IRFrontend::BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec) { + const bool sReady = DotLanesAreVec4(s), tReady = DotLanesAreVec4(t); + // The D temp holds the destination when a D prefix masks lanes. + const int temps[] = { IRVTEMP_0, js.prefixD == 0 ? IRVTEMP_PFX_D : -1, IRVTEMP_PFX_T, IRVTEMP_PFX_S }; + + int sTemp = -1, tTemp = -1; + if (!sReady) { + for (int temp : temps) { + if (temp >= 0 && temp != avoid && !DotLanesRead(t, temp) && DotLanesBuildableIn(s, temp)) { + sTemp = temp; + break; + } + } + if (sTemp < 0) + return false; + } + if (!tReady) { + for (int temp : temps) { + // Built after s, so it mustn't overwrite what s is (built or not). + const int sVecReg = sReady ? s[0].reg : sTemp; + const bool hitsS = sVecReg >= temp && sVecReg < temp + 4; + if (temp >= 0 && temp != avoid && temp != sTemp && !hitsS && DotLanesBuildableIn(t, temp)) { + tTemp = temp; + break; + } + } + if (tTemp < 0) + return false; + } + + auto build = [&](const DotLane lanes[4], int vec) { + for (int i = 0; i < 4; i++) { + if (lanes[i].reg < 0) { + ir.WriteSetConstantFloat(vec + i, lanes[i].constant); + } else if (lanes[i].negate) { + ir.Write(IROp::FNeg, vec + i, lanes[i].reg); + } else if (lanes[i].reg != vec + i) { + ir.Write(IROp::FMov, vec + i, lanes[i].reg); + } + } + }; + // s's temp holds no lane of t, and t's temp isn't where s is, so neither clobbers the other. + if (!sReady) + build(s, sTemp); + if (!tReady) + build(t, tTemp); + *sVec = sReady ? s[0].reg : sTemp; + *tVec = tReady ? t[0].reg : tTemp; + return true; + } + // True if the prefix only touches lanes the op has. A position past the size may only be the // identity, and a position within it may not name a lane past it (which zeroes the result lane // on hardware, see cpu/vfpu/prefix_ctrl - the interpreter handles that). @@ -698,6 +781,21 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, vt); GetVectorRegsPrefixD(dregs, V_Single, vd); + if (UseAccurateVdot()) { + // s's last lane is 1, so t's is added in. + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + sl[i] = i < n - 1 ? DotReg(sregs[i]) : DotConst(i == n - 1 ? 1.0f : 0.0f); + tl[i] = i < n ? DotReg(tregs[i]) : DotConst(0.0f); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, vd); + return; + } + ir.Write(IROp::FMul, IRVTEMP_0, sregs[0], tregs[0]); for (int i = 1; i < n; i++) { @@ -731,16 +829,39 @@ namespace MIPSComp { int n = GetNumVectorElements(sz); // On the PSP, an S constant past n is added in too (cpu/vfpu/prefix_lanes), which - // IsPrefixWithinSize() lets through. Rare enough for the interpreter. - for (int i = n; i < 4; i++) { - if (VFPUPrefixIsConstant(js.prefixS, i)) - DISABLE; + // IsPrefixWithinSize() lets through. The accurate dot product includes it, the rest leave it + // to the interpreter, as it's rare. + if (!UseAccurateVdot()) { + for (int i = n; i < 4; i++) { + if (VFPUPrefixIsConstant(js.prefixS, i)) + DISABLE; + } } u8 sregs[4], dregs[1]; GetVectorRegsPrefixS(sregs, sz, _VS); GetVectorRegsPrefixD(dregs, V_Single, _VD); + if (UseAccurateVdot()) { + // Dot products with constants from the T prefix in every lane: ones for vfad, 1/n for + // vavg (0 for vavg.s), as in the interpreter. + const bool vavg = ((op >> 16) & 31) == 7; + static const float vavgConst[4] = { 0.0f, 0.5f, 1.0f / 3.0f, 0.25f }; + const float c = vavg ? vavgConst[n - 1] : 1.0f; + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + const bool constantLane = i >= n && VFPUPrefixIsConstant(js.prefixS, i); + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane ? VFPUPrefixConstant(js.prefixS, i) : 0.0f); + tl[i] = DotConst(c); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, _VD); + return; + } + // We have to start at +0.000 in case any values are -0.000. ir.WriteSetConstantFloat(IRVTEMP_0, 0.0f); for (int i = 0; i < n; ++i) { @@ -777,11 +898,16 @@ namespace MIPSComp { int n = GetNumVectorElements(sz); // On the PSP, lanes past n where both prefixes are constants add their product, and the rest - // read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through, and they're - // rare enough for the interpreter. - for (int i = n; i < 4; i++) { - if (VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i)) - DISABLE; + // read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through. The accurate + // dot product includes them, the rest leave them to the interpreter, as they're rare. + auto constantLane = [&](int i) { + return i >= n && VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i); + }; + if (!UseAccurateVdot()) { + for (int i = n; i < 4; i++) { + if (constantLane(i)) + DISABLE; + } } // TODO: Force read one of them into regs? probably not. @@ -790,6 +916,20 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, vt); GetVectorRegsPrefixD(dregs, V_Single, vd); + if (UseAccurateVdot()) { + DotLane sl[4], tl[4]; + for (int i = 0; i < 4; i++) { + sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixS, i) : 0.0f); + tl[i] = i < n ? DotReg(tregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixT, i) : 0.0f); + } + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, vd); + return; + } + if (IsVec4(sz, sregs) && IsVec4(sz, tregs)) { if (IsOverlapSafe(dregs[0], n, sregs, n, tregs)) { ir.Write(IROp::Vec4Dot, dregs[0], sregs[0], tregs[0]); @@ -1615,12 +1755,6 @@ namespace MIPSComp { DISABLE; } - if (PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { - // Fall back to interpreter, which has the accurate implementation. - // Later we might do something more optimized here. - DISABLE; - } - // Matrix multiply (weird prefixes) // D[0 .. N, 0 .. M] = S[0 .. N, 0 .. M]' * T[0 .. N, 0 .. M] // Note: Behaves as if it's implemented through a series of vdots. @@ -1635,6 +1769,35 @@ namespace MIPSComp { MatrixOverlapType soverlap = GetMatrixOverlap(vs, vd, sz); MatrixOverlapType toverlap = GetMatrixOverlap(vt, vd, sz); + if (UseAccurateVdot()) { + // d[a][b] = s[b] dot t[a], each a row of n lanes and zeros, as in the interpreter. With + // no overlap, the results can go straight to d. + if (soverlap || toverlap) + DISABLE; + u8 sregs[16], tregs[16], dregs[16]; + GetMatrixRegs(sregs, sz, vs); + GetMatrixRegs(tregs, sz, vt); + GetMatrixRegs(dregs, sz, vd); + for (int a = 0; a < n; a++) { + DotLane tl[4]; + for (int c = 0; c < 4; c++) + tl[c] = c < n ? DotReg(tregs[a * 4 + c]) : DotConst(0.0f); + for (int b = 0; b < n; b++) { + DotLane sl[4]; + for (int c = 0; c < 4; c++) + sl[c] = c < n ? DotReg(sregs[b * 4 + c]) : DotConst(0.0f); + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + // Built once per row of t. + for (int c = 0; c < 4; c++) + tl[c] = DotReg(tv + c); + ir.Write(IROp::Vec4VfpuDot, dregs[a * 4 + b], sv, tv); + } + } + return; + } + // A very common arrangment. Rearrange to something we can handle. // A 1x1 matrix has no transposed form, the bit then picks a different register. if (n > 1 && IsMatrixTransposed(vd)) { @@ -1737,6 +1900,30 @@ namespace MIPSComp { GetVectorRegs(tregs, sz, _VT); GetVectorRegs(dregs, sz, _VD); + if (UseAccurateVdot()) { + // A dot per row, against t with the lanes past it zero, and lane ins 1 for the + // homogenous form. The results go to the D temp first, since d may overlap s or t. + const int tn = homogenous ? n - 1 : n; + DotLane tl[4]; + for (int i = 0; i < 4; i++) + tl[i] = i < tn ? DotReg(tregs[i]) : DotConst(homogenous && i == ins ? 1.0f : 0.0f); + for (int i = 0; i < n; i++) { + DotLane sl[4]; + for (int k = 0; k < 4; k++) + sl[k] = k < n ? DotReg(sregs[i * 4 + k]) : DotConst(0.0f); + u8 sv, tv; + if (!BuildDotOperands(sl, tl, IRVTEMP_PFX_D, &sv, &tv)) + DISABLE; + // Built once, t is a Vec4 for the rest. + for (int k = 0; k < 4; k++) + tl[k] = DotReg(tv + k); + ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv); + } + for (int i = 0; i < n; i++) + ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i); + return; + } + // SIMD-optimized implementations - if sregs[0..3] is non-consecutive, it's transposed. if (msz == M_4x4 && IsMatrixTransposedVec4(msz, sregs)) { int s0 = IRVTEMP_0; @@ -1871,6 +2058,18 @@ namespace MIPSComp { GetVectorRegsPrefixT(tregs, sz, _VT); GetVectorRegsPrefixD(dregs, V_Single, _VD); + if (UseAccurateVdot()) { + // The T prefix's swizzle is forced to yx: s0 * t1 + (-s1) * t0. + const DotLane sl[4] = { DotReg(sregs[0]), DotReg(sregs[1], true), DotConst(0.0f), DotConst(0.0f) }; + const DotLane tl[4] = { DotReg(tregs[1]), DotReg(tregs[0]), DotConst(0.0f), DotConst(0.0f) }; + u8 sv, tv; + if (!BuildDotOperands(sl, tl, -1, &sv, &tv)) + DISABLE; + ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv); + ApplyPrefixD(dregs, V_Single, _VD); + return; + } + ir.Write(IROp::FMul, IRVTEMP_0, sregs[1], tregs[0]); ir.Write(IROp::FMul, dregs[0], sregs[0], tregs[1]); ir.Write(IROp::FSub, dregs[0], dregs[0], IRVTEMP_0); @@ -2086,6 +2285,49 @@ namespace MIPSComp { GetVectorRegs(tregs, sz, _VT); GetVectorRegs(dregs, sz, _VD); + if (UseAccurateVdot() && (sz == V_Triple || sz == V_Quad)) { + // The dots the interpreter does, against permuted and negated copies of t. The results + // go to the D temp first, since d may overlap s or t. + const DotLane zero = DotConst(0.0f); + auto tl = [&](int i, bool neg = false) { return i < n ? DotReg(tregs[i], neg) : zero; }; + DotLane tDots[4][4]; + int dots; + if (sz == V_Triple) { + const DotLane t0[4] = { zero, tl(2), tl(1, true), zero }; + const DotLane t1[4] = { tl(2, true), zero, tl(0), zero }; + const DotLane t2[4] = { tl(1), tl(0, true), zero, tl(2) }; + memcpy(tDots[0], t0, sizeof(t0)); + memcpy(tDots[1], t1, sizeof(t1)); + memcpy(tDots[2], t2, sizeof(t2)); + dots = 3; + } else { + const DotLane t0[4] = { tl(3), tl(2), tl(1, true), tl(0) }; + const DotLane t1[4] = { tl(2, true), tl(3), tl(0), tl(1) }; + const DotLane t2[4] = { tl(1), tl(0, true), tl(3), tl(2) }; + const DotLane t3[4] = { tl(0, true), tl(1, true), tl(2, true), tl(3) }; + memcpy(tDots[0], t0, sizeof(t0)); + memcpy(tDots[1], t1, sizeof(t1)); + memcpy(tDots[2], t2, sizeof(t2)); + memcpy(tDots[3], t3, sizeof(t3)); + dots = 4; + } + DotLane sl[4]; + for (int i = 0; i < 4; i++) + sl[i] = i < n ? DotReg(sregs[i]) : zero; + for (int i = 0; i < dots; i++) { + u8 sv, tv; + if (!BuildDotOperands(sl, tDots[i], IRVTEMP_PFX_D, &sv, &tv)) + DISABLE; + // Built once, s is a Vec4 for the rest. + for (int j = 0; j < 4; j++) + sl[j] = DotReg(sv + j); + ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv); + } + for (int i = 0; i < dots; i++) + ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i); + return; + } + if (sz == V_Triple) { u8 tempregs[4]{}; for (int i = 0; i < n; ++i) { diff --git a/Core/MIPS/IR/IRFrontend.h b/Core/MIPS/IR/IRFrontend.h index 2a4be17b8d..2cf80fd63c 100644 --- a/Core/MIPS/IR/IRFrontend.h +++ b/Core/MIPS/IR/IRFrontend.h @@ -99,6 +99,13 @@ public: opts = o; } + // One lane of a Vec4VfpuDot operand: a reg, maybe negated, or a constant (reg < 0). + struct DotLane { + int reg; + float constant; + bool negate; + }; + private: void RestoreRoundingMode(bool force = false); void ApplyRoundingMode(bool force = false); @@ -134,6 +141,14 @@ private: void GetVectorRegs(u8 regs[4], VectorSize N, int vectorReg); void GetMatrixRegs(u8 regs[16], MatrixSize N, int matrixReg); + static DotLane DotReg(int reg, bool negate = false) { return DotLane{ reg, 0.0f, negate }; } + static DotLane DotConst(float f) { return DotLane{ -1, f, false }; } + // Puts both operands of a Vec4VfpuDot in Vec4 regs, building them in free temps where they + // aren't one already. avoid is a Vec4 temp that must not be used (or -1). Returns false if + // there aren't enough free temps, and then nothing was written. + bool BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec); + bool UseAccurateVdot() const; + // State JitState js; IRWriter ir; diff --git a/Core/MIPS/IR/IRInst.cpp b/Core/MIPS/IR/IRInst.cpp index 9bf8fee367..2fd123fa8b 100644 --- a/Core/MIPS/IR/IRInst.cpp +++ b/Core/MIPS/IR/IRInst.cpp @@ -158,6 +158,7 @@ static const IRMeta irMeta[] = { { IROp::Vec4Mul, "Vec4Mul", "VVV" }, { IROp::Vec4Scale, "Vec4Scale", "VVF" }, { IROp::Vec4Dot, "Vec4Dot", "FVV" }, + { IROp::Vec4VfpuDot, "Vec4VfpuDot", "FVV" }, { IROp::Vec4Neg, "Vec4Neg", "VV" }, { IROp::Vec4Abs, "Vec4Abs", "VV" }, diff --git a/Core/MIPS/IR/IRInst.h b/Core/MIPS/IR/IRInst.h index 0f746246ae..32962e2ae1 100644 --- a/Core/MIPS/IR/IRInst.h +++ b/Core/MIPS/IR/IRInst.h @@ -187,6 +187,7 @@ enum class IROp : uint8_t { Vec4Div, Vec4Scale, Vec4Dot, + Vec4VfpuDot, // The VFPU's own dot product, bit-exact (vfpu_dot). Vec4Neg, Vec4Abs, diff --git a/Core/MIPS/IR/IRInterpreter.cpp b/Core/MIPS/IR/IRInterpreter.cpp index 6d67305c97..fa514dd566 100644 --- a/Core/MIPS/IR/IRInterpreter.cpp +++ b/Core/MIPS/IR/IRInterpreter.cpp @@ -171,6 +171,7 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { dispatch[(int)IROp::FCmpVfpuAggregate] = &&L_FCmpVfpuAggregate; dispatch[(int)IROp::FCmovVfpuCC] = &&L_FCmovVfpuCC; dispatch[(int)IROp::Vec4Dot] = &&L_Vec4Dot; + dispatch[(int)IROp::Vec4VfpuDot] = &&L_Vec4VfpuDot; dispatch[(int)IROp::FSin] = &&L_FSin; dispatch[(int)IROp::FCos] = &&L_FCos; dispatch[(int)IROp::FRSqrt] = &&L_FRSqrt; @@ -798,6 +799,10 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) { IR_NEXT; } + IR_CASE(Vec4VfpuDot) + mips->f[inst->dest] = vfpu_dot(&mips->f[inst->src1], &mips->f[inst->src2]); + IR_NEXT; + IR_CASE(FSin) mips->f[inst->dest] = vfpu_sin(mips->f[inst->src1]); IR_NEXT; diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index 7df5abf18a..484ed8a313 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -421,9 +421,11 @@ void IRNativeBackend::CompileIRInst(IRInst inst) { break; case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: CompIR_VecHoriz(inst); break; + case IROp::Vec2Unpack16To31: case IROp::Vec2Unpack16To32: case IROp::Vec4Unpack8To32: diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index 1961892748..699cc729ad 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -847,6 +847,7 @@ bool PropagateConstants(const IRWriter &in, IRWriter &out, const IROptions &opts case IROp::Vec4Mul: case IROp::Vec4Div: case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: case IROp::Vec4Scale: case IROp::Vec4Shuffle: case IROp::Vec4Blend: @@ -2185,6 +2186,7 @@ bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts) { break; case IROp::Vec4Dot: + case IROp::Vec4VfpuDot: if (overlapped(inst.dest, 1, inst.src1, 4, inst.src2, 4) && findAvailTempVec4()) { out.Write(inst.op, temp, inst.src1, inst.src2, inst.constant); if (usedLaterAsVec4(inst.dest & ~3)) { diff --git a/Core/MIPS/InterpreterVFPU.cpp b/Core/MIPS/InterpreterVFPU.cpp index 481577b47d..838025d1be 100644 --- a/Core/MIPS/InterpreterVFPU.cpp +++ b/Core/MIPS/InterpreterVFPU.cpp @@ -28,6 +28,7 @@ #include "Core/Core.h" #include "Core/MemMap.h" #include "Core/Reporting.h" +#include "Core/Config.h" #include "Core/System.h" #include "Core/MIPS/MIPS.h" @@ -74,7 +75,13 @@ #define M_SQRT1_2 0.707106781186547524401f #endif -static const bool USE_VFPU_DOT = false; +// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3). +static const bool USE_VFPU_RESULT_FIXUP = false; + +// The dot products (vdot, vmmul, vtfm and the rest) through vfpu_dot(), bit-exact with the PSP. +static inline bool AccurateVdot() { + return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot; +} union FloatBits { float f[4]; @@ -507,8 +514,7 @@ namespace MIPSInt ReadMatrix(mips, s, sz, vs); ReadMatrix(mips, t, sz, vt); - // TODO: Always use the more accurate path in interpreter? - bool useAccurateDot = USE_VFPU_DOT || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL; + bool useAccurateDot = AccurateVdot(); for (int a = 0; a < n; a++) { for (int b = 0; b < n; b++) { union { float f; uint32_t u; } sum = { 0.0f }; @@ -1208,7 +1214,7 @@ namespace MIPSInt ReadVector(mips, t, sz, vt); ApplySwizzleT(mips, t, V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d.f = vfpu_dot(s, t); if (my_isnan(d.f)) { d.u = 0x7f800001; @@ -1259,7 +1265,7 @@ namespace MIPSInt ApplyPrefixST(mips, s, VFPURewritePrefix(mips, VFPU_CTRL_SPREFIX, sprefixRemove, sprefixAdd), V_Quad); float sum = 0.0f; - if (USE_VFPU_DOT) { + if (AccurateVdot()) { sum = vfpu_dot(s, t); } else { for (int i = 0; i < 4; i++) { @@ -1479,7 +1485,7 @@ namespace MIPSInt u32 tprefixAdd = VFPU_SWIZZLE(1, 0, 0, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { s[1] = -s[1]; d[0] = vfpu_dot(s, t); } else { @@ -1507,7 +1513,7 @@ namespace MIPSInt u32 tprefixAdd = VFPU_MAKE_CONSTANTS(VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d = vfpu_dot(s, t); } else { d = 0.0f; @@ -1545,7 +1551,7 @@ namespace MIPSInt tprefixAdd = 0; ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d = vfpu_dot(s, t); } else { d = 0.0f; @@ -1709,7 +1715,7 @@ namespace MIPSInt ReadMatrix(mips, s, msz, vs); ReadVector(mips, t, sz, vt); - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t2[4]; for (int i = 0; i < 4; i++) { if (i < tn) { @@ -1763,7 +1769,7 @@ namespace MIPSInt ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); // Really this is the operation all rows probably use (with constant wiring.) - if (USE_VFPU_DOT) { + if (AccurateVdot()) { d.f[ins] = vfpu_dot(&s[ins * 4], t); if (my_isnan(d.f[ins])) { @@ -2164,7 +2170,7 @@ namespace MIPSInt case 8: d.f[i] = s[i] * t[i]; break; //vmul } - if (USE_VFPU_DOT) { + if (USE_VFPU_RESULT_FIXUP) { if (my_isnan(d.f[i])) { d.u[i] = (d.u[i] & 0xff800001) | 1; } else if ((d.u[i] & 0x7F800000) == 0) { @@ -2207,7 +2213,7 @@ namespace MIPSInt switch (sz) { case V_Triple: // vcrsp.t { - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t0[4] = { 0.0f, t[2], -t[1], 0.0f }; float t1[4] = { -t[2], 0.0f, t[0], 0.0f }; d[0] = vfpu_dot(s, t0); @@ -2221,14 +2227,8 @@ namespace MIPSInt tprefixAdd = VFPU_SWIZZLE(1, 0, 3, 2) | VFPU_NEGATE(0, 1, 0, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); ApplySwizzleS(mips, s, V_Quad); - if (USE_VFPU_DOT) { - // TODO: But flush any infs to 0? This seems sketchy. - for (int i = 0; i < 4; ++i) { - if (my_isinf(s[i])) - s[i] = 0.0f; - if (my_isinf(t[i])) - t[i] = 0.0f; - } + if (AccurateVdot()) { + // Infinities stay: inf * 0 gives the VFPU's NaN here (cpu/vfpu/nan_gen). d[2] = vfpu_dot(s, t); } else { d[2] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3]; @@ -2238,7 +2238,7 @@ namespace MIPSInt case V_Quad: // vqmul.q { - if (USE_VFPU_DOT) { + if (AccurateVdot()) { float t0[4] = { t[3], t[2], -t[1], t[0] }; float t1[4] = { -t[2], t[3], t[0], t[1] }; float t2[4] = { t[1], -t[0], t[3], t[2] }; @@ -2255,7 +2255,7 @@ namespace MIPSInt tprefixAdd = VFPU_SWIZZLE(0, 1, 2, 3) | VFPU_NEGATE(1, 1, 1, 0); ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad); ApplySwizzleS(mips, s, sz); - if (USE_VFPU_DOT) + if (AccurateVdot()) d[3] = vfpu_dot(s, t); else d[3] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3]; diff --git a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp index ab940a082d..a12804157f 100644 --- a/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64CompVec.cpp @@ -389,6 +389,11 @@ void LoongArch64JitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/MIPS.h b/Core/MIPS/MIPS.h index 9d5e79ef9b..fa38faa157 100644 --- a/Core/MIPS/MIPS.h +++ b/Core/MIPS/MIPS.h @@ -248,6 +248,10 @@ public: float sincostemp[2]; + // The operands (s, then t) and result of the JITs' VFPU dot product routine (vfpu_dot()). + alignas(16) u32 vfpuDotArgs[8]; + u32 vfpuDotResult; + static const u32 FCR0_VALUE = 0x00003351; #if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/RiscV/RiscVCompVec.cpp b/Core/MIPS/RiscV/RiscVCompVec.cpp index c2c00e5589..c68b0877ef 100644 --- a/Core/MIPS/RiscV/RiscVCompVec.cpp +++ b/Core/MIPS/RiscV/RiscVCompVec.cpp @@ -312,6 +312,11 @@ void RiscVJitBackend::CompIR_VecHoriz(IRInst inst) { } break; + case IROp::Vec4VfpuDot: + // Through the IR interpreter for now. + CompIR_Generic(inst); + break; + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRAsm.cpp b/Core/MIPS/x86/X64IRAsm.cpp index eb26a06c03..2acb6a23ea 100644 --- a/Core/MIPS/x86/X64IRAsm.cpp +++ b/Core/MIPS/x86/X64IRAsm.cpp @@ -23,6 +23,7 @@ #include "Core/MemMap.h" #include "Core/MIPS/x86/X64IRJit.h" #include "Core/MIPS/x86/X64IRRegCache.h" +#include "Core/MIPS/x86/X64VfpuDot.h" #include "Core/MIPS/JitCommon/JitCommon.h" #include "Core/MIPS/JitCommon/JitState.h" #include "Core/Core.h" @@ -295,6 +296,9 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) { } JMP(quitLoop, true); +#if PPSSPP_ARCH(AMD64) + vfpuDot_ = GenerateX64VfpuDot(this, CTXREG); +#endif // Leave this at the end, add more stuff above. if (enableDisasm) { diff --git a/Core/MIPS/x86/X64IRCompVec.cpp b/Core/MIPS/x86/X64IRCompVec.cpp index aea78af280..a25a699f13 100644 --- a/Core/MIPS/x86/X64IRCompVec.cpp +++ b/Core/MIPS/x86/X64IRCompVec.cpp @@ -274,6 +274,49 @@ void X64JitBackend::CompIR_VecHoriz(IRInst inst) { break; } + case IROp::Vec4VfpuDot: + { +#if PPSSPP_ARCH(AMD64) + if (!vfpuDot_ || Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) { + CompIR_Generic(inst); + break; + } + // The operands go to the shared routine in xmm0 and xmm1, and the result comes back in edi. + // Whatever guest state is in the registers it clobbers gets flushed first: these come last in + // the allocation order, so usually that's nothing. Then again after the operands are moved, + // in case they were mapped into one of them. + auto flushClobbered = [&]() { + regs_.FlushHostRegs({ RCX, RDX, RSI, RDI }, { XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM14, XMM15 }); + }; + flushClobbered(); + regs_.SpillLockFPR(inst.src1, inst.src2); + const X64Reg x = regs_.MapVec4(inst.src1); + const X64Reg y = regs_.MapVec4(inst.src2); + if (x == XMM1 && y == XMM0) { + MOVAPS(XMM2, R(XMM0)); + MOVAPS(XMM0, R(XMM1)); + MOVAPS(XMM1, R(XMM2)); + } else if (y == XMM0) { + MOVAPS(XMM1, R(y)); + MOVAPS(XMM0, R(x)); + } else { + if (x != XMM0) + MOVAPS(XMM0, R(x)); + if (y != XMM1) + MOVAPS(XMM1, R(y)); + } + regs_.ReleaseSpillLockFPR(inst.src1, inst.src2); + flushClobbered(); + CALL(vfpuDot_); + regs_.MapFPR(inst.dest, MIPSMap::NOINIT); + MOVD_xmm(regs_.FX(inst.dest), R(EDI)); +#else + // 32-bit x86 has too few registers for the shared routine. + CompIR_Generic(inst); +#endif + break; + } + default: INVALIDOP; break; diff --git a/Core/MIPS/x86/X64IRJit.h b/Core/MIPS/x86/X64IRJit.h index edd86594e0..11733dcdeb 100644 --- a/Core/MIPS/x86/X64IRJit.h +++ b/Core/MIPS/x86/X64IRJit.h @@ -141,6 +141,8 @@ private: const u8 *saveStaticRegisters_ = nullptr; const u8 *loadStaticRegisters_ = nullptr; + // See GenerateX64VfpuDot(). + const u8 *vfpuDot_ = nullptr; typedef struct { float f[4]; } Float4Constant; struct Constants { diff --git a/Core/MIPS/x86/X64IRRegCache.cpp b/Core/MIPS/x86/X64IRRegCache.cpp index f3644e2778..a04ee29773 100644 --- a/Core/MIPS/x86/X64IRRegCache.cpp +++ b/Core/MIPS/x86/X64IRRegCache.cpp @@ -119,6 +119,13 @@ const int *X64IRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c } } +void X64IRRegCache::FlushHostRegs(std::initializer_list gprs, std::initializer_list xmms) { + for (X64Reg r : gprs) + FlushNativeReg(GPRToNativeReg(r)); + for (X64Reg r : xmms) + FlushNativeReg(XMMToNativeReg(r)); +} + void X64IRRegCache::FlushBeforeCall() { // These registers are not preserved by function calls. #if PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/X64IRRegCache.h b/Core/MIPS/x86/X64IRRegCache.h index 8a21f563d8..16ea14cf2b 100644 --- a/Core/MIPS/x86/X64IRRegCache.h +++ b/Core/MIPS/x86/X64IRRegCache.h @@ -18,6 +18,8 @@ #pragma once #include "ppsspp_config.h" + +#include #if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) #include "Common/x64Emitter.h" @@ -95,6 +97,8 @@ public: // Note: may change the high lanes of single-register XMMs. void FlushAll(bool gprs = true, bool fprs = true) override; void FlushBeforeCall(); + // Flushes whatever guest state these registers hold, for code that clobbers them. + void FlushHostRegs(std::initializer_list gprs, std::initializer_list xmms); Gen::X64Reg GetAndLockTempGPR(); Gen::X64Reg GetAndLockTempFPR(); diff --git a/Core/MIPS/x86/X64VfpuDot.h b/Core/MIPS/x86/X64VfpuDot.h new file mode 100644 index 0000000000..9f214a9662 --- /dev/null +++ b/Core/MIPS/x86/X64VfpuDot.h @@ -0,0 +1,237 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include "ppsspp_config.h" + +#if PPSSPP_ARCH(AMD64) + +#include +#include +#include + +#include "Common/x64Emitter.h" +#include "Core/MIPS/MIPS.h" +#include "Core/MIPS/MIPSVFPUUtils.h" + +namespace MIPSComp { + +// Generates the VFPU dot product routine the x86-64 IR JIT CALLs: vfpu_dot() of xmm0 and xmm1, into +// edi. ctx points at mips->f[0], as CTXREG does. It's vfpu_dot_sse2(), SSE2 only, and leaves inf +// and NaN to vfpu_dot_reference(). It clobbers xmm0-xmm5, xmm14, xmm15, rax, rcx, rdx, rsi, rdi and +// the flags, and saves nothing: the call site flushes whatever of those holds guest state. The xmms +// are the last ones allocated, and rsi and rdi are never allocated outside Windows. +inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) { + using namespace Gen; + const int args = (int)(offsetof(MIPSState, vfpuDotArgs) - offsetof(MIPSState, f)); + + // The constants, RIP-relative. + auto constant4 = [&](u32 a, u32 b, u32 c, u32 d) { + const u8 *p = e->AlignCode16(); + u8 *writable = e->GetWritableCodePtr(); + e->ReserveCodeSpace(16); + const u32 values[4] = { a, b, c, d }; + memcpy(writable, values, sizeof(values)); + return p; + }; + auto splat = [&](u32 v) { return constant4(v, v, v, v); }; + const u8 *expBits = splat(0x7F800000); + const u8 *mantMask = splat(0x007FFFFF); + const u8 *implicitBit = splat(0x00800000); + const u8 *lowMask = constant4((1 << 21) - 1, 0, (1 << 21) - 1, 0); + const u8 *ones = splat(1); + const u8 *max28 = splat(28); + const u8 *scaleBase = splat(127 + 28); + + const u8 *start = e->AlignCode16(); + + // xmm0 = x, xmm1 = y. Exponents: a zero or denormal operand makes the product zero (xmm4 = + // flush), and inf or NaN gets 0x7FF, above any real exponent sum, so one maximum finds the + // alignment and the specials (xmm5 = e). They all fit in 15 bits, so the 16-bit max does. + e->PXOR(XMM15, R(XMM15)); + e->MOVDQA(XMM2, R(XMM0)); + e->PAND(XMM2, M(expBits)); + e->MOVDQA(XMM3, R(XMM1)); + e->PAND(XMM3, M(expBits)); + e->MOVDQA(XMM4, R(XMM2)); + e->PCMPEQD(XMM4, R(XMM15)); + e->MOVDQA(XMM5, R(XMM3)); + e->PCMPEQD(XMM5, R(XMM15)); + e->POR(XMM4, R(XMM5)); + e->MOVDQA(XMM14, R(XMM2)); + e->PCMPEQD(XMM14, M(expBits)); + e->MOVDQA(XMM5, R(XMM3)); + e->PCMPEQD(XMM5, M(expBits)); + e->POR(XMM14, R(XMM5)); + e->PSRLD(XMM14, 21); + e->MOVDQA(XMM5, R(XMM2)); + e->PADDD(XMM5, R(XMM3)); + e->PSRLD(XMM5, 23); + e->MOVDQA(XMM2, R(XMM4)); + e->PANDN(XMM2, R(XMM5)); + e->MOVDQA(XMM5, R(XMM2)); + e->POR(XMM5, R(XMM14)); + e->PSHUFD(XMM2, R(XMM5), 0x4E); + e->PMAXSW(XMM2, R(XMM5)); + e->PSHUFD(XMM3, R(XMM2), 0xB1); + e->PMAXSW(XMM2, R(XMM3)); + e->MOVD_xmm(R(EAX), XMM2); + e->CMP(32, R(EAX), Imm32(0x7FF)); + FixupBranch special = e->J_CC(CC_AE, true); + + // 24x24-bit products, even and odd lanes separately, kept to 2 extra bits with round-to-odd + // (xmm4 = p). xmm0 = the signs, while x and y are still around. + e->MOVDQA(XMM3, R(XMM0)); + e->PAND(XMM3, M(mantMask)); + e->POR(XMM3, M(implicitBit)); + e->MOVDQA(XMM14, R(XMM1)); + e->PAND(XMM14, M(mantMask)); + e->POR(XMM14, M(implicitBit)); + e->PXOR(XMM0, R(XMM1)); + e->PSRAD(XMM0, 31); + e->MOVDQA(XMM15, R(XMM3)); + e->PMULUDQ(XMM15, R(XMM14)); // even products + e->PSRLQ(XMM3, 32); + e->PSRLQ(XMM14, 32); + e->PMULUDQ(XMM3, R(XMM14)); // odd products + e->MOVDQA(XMM14, R(XMM15)); + e->PSRLQ(XMM14, 21); + e->PAND(XMM15, M(lowMask)); + e->MOVDQA(XMM1, R(XMM3)); + e->PSRLQ(XMM1, 21); + e->PSLLQ(XMM1, 32); + e->POR(XMM14, R(XMM1)); // kept + e->PAND(XMM3, M(lowMask)); + e->PSLLQ(XMM3, 32); + e->POR(XMM3, R(XMM15)); // dropped + e->PXOR(XMM15, R(XMM15)); + e->PCMPEQD(XMM3, R(XMM15)); + e->PANDN(XMM3, M(ones)); // sticky + e->POR(XMM3, R(XMM14)); + e->PANDN(XMM4, R(XMM3)); + + // Align to the largest exponent by truncation. SSE2 has no per-lane shift, so p >> d is done + // as (p * 2^(28 - d)) >> 28, with the power of two built as float bits and converted by + // truncation (always a normal float, so DAZ doesn't matter). + e->PSHUFD(XMM2, R(XMM2), 0x00); + e->PSUBD(XMM2, R(XMM5)); + e->PMINSW(XMM2, M(max28)); + e->MOVDQA(XMM3, M(scaleBase)); + e->PSUBD(XMM3, R(XMM2)); + e->PSLLD(XMM3, 23); + e->CVTTPS2DQ(XMM3, R(XMM3)); + e->MOVDQA(XMM5, R(XMM4)); + e->PMULUDQ(XMM5, R(XMM3)); + e->PSRLQ(XMM5, 28); + e->PSRLQ(XMM4, 32); + e->PSRLQ(XMM3, 32); + e->PMULUDQ(XMM4, R(XMM3)); + e->PSRLQ(XMM4, 28); + e->PSLLQ(XMM4, 32); + e->POR(XMM4, R(XMM5)); + + // Apply the signs and sum exactly. + e->PXOR(XMM4, R(XMM0)); + e->PSUBD(XMM4, R(XMM0)); + e->PSHUFD(XMM0, R(XMM4), 0x4E); + e->PADDD(XMM4, R(XMM0)); + e->PSHUFD(XMM0, R(XMM4), 0xB1); + e->PADDD(XMM4, R(XMM0)); + + // vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float. + // esi = the sum, then its sign. edi = the largest exponent sum, then the result. edx = the + // significand m. ecx = 62 - its top bit, the shift that puts that at bit 62. + e->MOVD_xmm(R(ESI), XMM4); + e->MOV(32, R(EDI), R(EAX)); + e->MOV(32, R(EAX), R(ESI)); + e->CDQ(); + e->XOR(32, R(EAX), R(EDX)); + e->SUB(32, R(EAX), R(EDX)); + e->AND(32, R(ESI), Imm32(0x80000000)); + e->SHR(32, R(EAX), Imm8(2)); + e->MOV(32, R(EDX), R(EAX)); + e->MOV(32, R(ECX), R(EAX)); + e->OR(32, R(ECX), Imm32(1)); + e->BSR(32, ECX, R(ECX)); + e->NEG(32, R(ECX)); + e->ADD(32, R(ECX), Imm32(62)); + e->SHL(64, R(RAX), R(CL)); + e->SUB(32, R(EDI), R(ECX)); + e->SUB(32, R(EDI), Imm32(89)); + e->MOV(64, R(RCX), R(RAX)); + e->SHR(64, R(RCX), Imm8(39)); + e->AND(32, R(ECX), Imm32(1)); + e->ADD(64, R(RAX), R(RCX)); + e->MOV(64, R(RCX), Imm64((1ULL << 38) - 1)); + e->ADD(64, R(RAX), R(RCX)); + e->SHR(64, R(RAX), Imm8(39)); + e->MOVSX(64, 32, RDI, R(EDI)); + e->SHL(64, R(RDI), Imm8(23)); + e->ADD(64, R(RDI), R(RAX)); + e->MOV(32, R(EAX), Imm32(0x7F800000)); + e->CMP(64, R(RDI), R(RAX)); + e->CMOVcc(64, RDI, R(RAX), CC_GE); + e->XOR(32, R(ECX), R(ECX)); + e->CMP(64, R(RDI), Imm32(0x00800000)); + e->CMOVcc(32, EDI, R(ECX), CC_L); + e->TEST(32, R(EDX), R(EDX)); + e->CMOVcc(32, EDI, R(ECX), CC_Z); + e->OR(32, R(EDI), R(ESI)); + e->RET(); + + // Inf or NaN: the C version, with x and y still in xmm0 and xmm1. The call can clobber registers + // that the call site didn't flush, so those are saved, around an aligned stack. + e->SetJumpTarget(special); + e->MOVUPS(MDisp(ctx, args), XMM0); + e->MOVUPS(MDisp(ctx, args + 16), XMM1); + e->PUSH(R8); + e->PUSH(R9); + e->PUSH(R10); + e->PUSH(R11); + e->PUSH(RBP); + e->MOV(64, R(RBP), R(RSP)); + e->AND(64, R(RSP), Imm32(-16)); + // xmm6-xmm13, and the shadow space Windows wants. + e->SUB(64, R(RSP), Imm32(8 * 16 + 32)); + for (int i = 6; i < 14; i++) + e->MOVUPS(MDisp(RSP, 32 + (i - 6) * 16), (X64Reg)(XMM0 + i)); +#ifdef _WIN32 + e->LEA(64, RCX, MDisp(ctx, args)); + e->LEA(64, RDX, MDisp(ctx, args + 16)); +#else + e->LEA(64, RDI, MDisp(ctx, args)); + e->LEA(64, RSI, MDisp(ctx, args + 16)); +#endif + e->MOV(64, R(RAX), ImmPtr((const void *)&vfpu_dot_reference)); + e->CALLptr(R(RAX)); + e->MOVD_xmm(R(EDI), XMM0); + for (int i = 6; i < 14; i++) + e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, 32 + (i - 6) * 16)); + e->MOV(64, R(RSP), R(RBP)); + e->POP(RBP); + e->POP(R11); + e->POP(R10); + e->POP(R9); + e->POP(R8); + e->RET(); + return start; +} + +} // namespace MIPSComp + +#endif diff --git a/Core/Util/DisArm64.cpp b/Core/Util/DisArm64.cpp index 6e44ae7646..35bc5e07a6 100644 --- a/Core/Util/DisArm64.cpp +++ b/Core/Util/DisArm64.cpp @@ -1357,6 +1357,69 @@ static void SIMDVectorIndexedElement(uint32_t w, Instruction *instr) { Out(instr, "%s %c%d, %c%d, %c%d.%s[%d]", opname, r, Rd, r, Rn, r, Rm, GetArrangement(Q, size & 1), index); } +// Reductions across the lanes of a vector into a scalar. +static void SIMDAcrossLanes(uint32_t w, Instruction *instr) { + int Rd = w & 0x1f; + int Rn = (w >> 5) & 0x1f; + int opcode = (w >> 12) & 0x1F; + int size = (w >> 22) & 3; + bool Q = GetQ(w); + bool U = GetU(w); + char r = Q ? 'q' : 'd'; + if (U && (opcode == 0x0C || opcode == 0x0F)) { + // fmaxnmv, fmaxv, fminnmv, fminv: single precision only. + const char *name = opcode == 0x0C ? ((size & 2) ? "fminnmv" : "fmaxnmv") : ((size & 2) ? "fminv" : "fmaxv"); + Out(instr, "%s s%d, %c%d.4s", name, Rd, r, Rn); + return; + } + const char *name = nullptr; + bool longResult = false; + switch (opcode) { + case 0x03: name = U ? "uaddlv" : "saddlv"; longResult = true; break; + case 0x0A: name = U ? "umaxv" : "smaxv"; break; + case 0x1A: name = U ? "uminv" : "sminv"; break; + case 0x1B: name = U ? nullptr : "addv"; break; + } + if (!name || size == 3) { + Out(instr, "(asimd across lanes %08x)", w); + return; + } + static const char scalarRegs[] = "bhsd"; + Out(instr, "%s.%d %c%d, %c%d", name, 8 << size, scalarRegs[size + (longResult ? 1 : 0)], Rd, r, Rn); +} + +// Long (L), wide (W) and narrowing (HN) ops on lanes of different sizes. +static void SIMDThreeDifferent(uint32_t w, Instruction *instr) { + int Rd = w & 0x1f; + int Rn = (w >> 5) & 0x1f; + int Rm = (w >> 16) & 0x1f; + int opcode = (w >> 12) & 0xF; + int size = (w >> 22) & 3; + bool Q = GetQ(w); + bool U = GetU(w); + static const char * const names[2][16] = { + { "saddl", "saddw", "ssubl", "ssubw", "addhn", "sabal", "subhn", "sabdl", "smlal", "sqdmlal", "smlsl", "sqdmlsl", "smull", "sqdmull", "pmull", nullptr }, + { "uaddl", "uaddw", "usubl", "usubw", "raddhn", "uabal", "rsubhn", "uabdl", "umlal", nullptr, "umlsl", nullptr, "umull", nullptr, nullptr, nullptr }, + }; + const char *name = names[U][opcode]; + if (!name || size == 3) { + Out(instr, "(asimd three-different %08x)", w); + return; + } + const char *two = Q ? "2" : ""; + const char half = Q ? 'q' : 'd'; + const int narrow = 8 << size, wide = 16 << size; + if (opcode == 1 || opcode == 3) { + // Wide: Rd and Rn are the wide lanes. + Out(instr, "%s%s.%d.%d q%d, q%d, %c%d", name, two, wide, narrow, Rd, Rn, half, Rm); + } else if (opcode == 4 || opcode == 6) { + // Narrowing: Rd gets the high halves. + Out(instr, "%s%s.%d.%d %c%d, q%d, q%d", name, two, narrow, wide, half, Rd, Rn, Rm); + } else { + Out(instr, "%s%s.%d.%d q%d, %c%d, %c%d", name, two, wide, narrow, Rd, half, Rn, half, Rm); + } +} + // (w >> 25) & 0xF == 7: AdvSIMD vector instructions. static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) { int Rd = w & 0x1f; @@ -1372,9 +1435,9 @@ static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) { } else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 0) { SIMDTwoRegMisc(w, instr); } else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 8) { - Out(instr, "(asimd across lanes %08x)", w); + SIMDAcrossLanes(w, instr); } else { - Out(instr, "(asimd three-different %08x)", w); + SIMDThreeDifferent(w, instr); } } else if ((w & 0x9FE08400) == 0x0E000400) { SIMDCopy(w, instr); diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index cdb0b5f19b..59badf30a4 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -278,8 +278,10 @@ + + @@ -307,6 +309,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index 6c900373eb..9d0f9ce361 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -597,8 +597,10 @@ + + @@ -626,6 +628,7 @@ + diff --git a/assets/compat.ini b/assets/compat.ini index 30a690c955..2ff55a3984 100644 --- a/assets/compat.ini +++ b/assets/compat.ini @@ -1100,9 +1100,8 @@ NPJH50559 = true NPEH00030 = true ULKS46191 = true -[MoreAccurateVMMUL] -# Fixes leg shaking in Tekken 6. The potential for slowdown in other games is large enough -# that we will not generally apply this accurate mode where not needed. +[ForceAccurateVdot] +# The VFPU's own dot products, even with AccurateVdot turned off. Fixes leg shaking in Tekken 6. ULUS10466 = true ULES01376 = true ULJS00224 = true diff --git a/docs/cpu-equivalence.md b/docs/cpu-equivalence.md index a6e6e5ecab..804f26b375 100644 --- a/docs/cpu-equivalence.md +++ b/docs/cpu-equivalence.md @@ -33,6 +33,7 @@ Environment variables: | `PPSSPP_CPUFUZZ_DUMP` | write every core's end state of every case to a file | | `PPSSPP_CPUFUZZ_STRICT` | fail on any difference, not only on a crash | | `PPSSPP_CPUFUZZ_OPS` | comma-separated instruction name prefixes to generate only; `dot` is the dot products and the prefix ops | +| `PPSSPP_CPUFUZZ_ACCURATE_VDOT` | `0` to run without the accurate dot products, which are on by default | For quick rounds while working on some instructions, fuzz just those, on every core of the machine: `PPSSPP_CPUFUZZ_OPS=dot Tools/cpu_fuzz_parallel.py build/PPSSPPUnitTest --count 2000` splits the seeds diff --git a/unittest/JitHarness.cpp b/unittest/JitHarness.cpp index 292d6f82ea..ccbcca9782 100644 --- a/unittest/JitHarness.cpp +++ b/unittest/JitHarness.cpp @@ -398,6 +398,7 @@ bool TestJitReplacementHooks() { // PPSSPP_CPUFUZZ_SKIP cores not to run, by digit: 1 IR interpreter, 2 JIT, 3 JIT IR. // PPSSPP_CPUFUZZ_REPORTS how many differences to minimize and print (default 20). // PPSSPP_CPUFUZZ_STRICT fail the test on any difference, not only on a crash. +// PPSSPP_CPUFUZZ_ACCURATE_VDOT 0 to run without AccurateVdot (the VFPU's own dot products), which is on by default. // PPSSPP_CPUFUZZ_OPS comma-separated instruction name prefixes to generate only, for fast // runs on the instructions being worked on. "dot" is the dot products // and the prefixes. Without PPSSPP_CPUFUZZ_CLASSES, the class is "v". @@ -899,6 +900,9 @@ bool TestCPUEquivalence() { // terminating syscall doesn't. const bool oldSkipDeadbeef = g_Config.bSkipDeadbeefFilling; g_Config.bSkipDeadbeefFilling = true; + const bool oldAccurateVdot = g_Config.bAccurateVdot; + const char *accurateVdot = getenv("PPSSPP_CPUFUZZ_ACCURATE_VDOT"); + g_Config.bAccurateVdot = !accurateVdot || strcmp(accurateVdot, "0") != 0; const char *countStr = getenv("PPSSPP_CPUFUZZ_COUNT"); const char *seedStr = getenv("PPSSPP_CPUFUZZ_SEED"); @@ -1049,6 +1053,7 @@ bool TestCPUEquivalence() { mipsr4k.UpdateCore(CPUCore::INTERPRETER); DestroyJitHarness(); g_Config.bSkipDeadbeefFilling = oldSkipDeadbeef; + g_Config.bAccurateVdot = oldAccurateVdot; return success; } diff --git a/unittest/TestArm64Disasm.cpp b/unittest/TestArm64Disasm.cpp index 245d3ab16e..b1c4d68ef8 100644 --- a/unittest/TestArm64Disasm.cpp +++ b/unittest/TestArm64Disasm.cpp @@ -698,6 +698,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float // Vector logical DIS("and q1, q2, q3", fp.AND(Q1, Q2, Q3)); DIS("and d1, d2, d3", fp.AND(D1, D2, D3)); + DIS("bic q1, q2, q3", fp.BIC(Q1, Q2, Q3)); + DIS("bic d1, d2, d3", fp.BIC(D1, D2, D3)); DIS("eor q0, q1, q2", fp.EOR(Q0, Q1, Q2)); DIS("eor d0, d1, d2", fp.EOR(D0, D1, D2)); DIS("bsl q1, q2, q3", fp.BSL(Q1, Q2, Q3)); @@ -772,6 +774,20 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float DIS("umin.8 q0, q3, q4", fp.UMIN(8, Q0, Q3, Q4)); DIS("umax.16 q0, q3, q4", fp.UMAX(16, Q0, Q3, Q4)); DIS("umax.32 d0, d3, d4", fp.UMAX(32, D0, D3, D4)); + DIS("add.32 q0, q3, q4", fp.ADD(32, Q0, Q3, Q4)); + DIS("add.16 d0, d3, d4", fp.ADD(16, D0, D3, D4)); + DIS("sub.32 q0, q3, q4", fp.SUB(32, Q0, Q3, Q4)); + DIS("sub.8 d0, d3, d4", fp.SUB(8, D0, D3, D4)); + DIS("umaxp.32 q0, q3, q4", fp.UMAXP(32, Q0, Q3, Q4)); + DIS("umaxp.8 d0, d3, d4", fp.UMAXP(8, D0, D3, D4)); + DIS("ushl.32 q1, q2, q3", fp.USHL(32, Q1, Q2, Q3)); + DIS("ushl.16 d1, d2, d3", fp.USHL(16, D1, D2, D3)); + DIS("umull.64.32 q1, d2, d3", fp.UMULL(32, Q1, D2, D3)); + DIS("umull.16.8 q4, d5, d6", fp.UMULL(8, Q4, D5, D6)); + DIS("umull2.64.32 q1, q2, q3", fp.UMULL2(32, Q1, Q2, Q3)); + DIS("umull2.32.16 q7, q8, q9", fp.UMULL2(16, Q7, Q8, Q9)); + DIS("addv.32 s1, q2", fp.ADDV(32, S1, Q2)); + DIS("addv.16 h3, d4", fp.ADDV(16, S3, D4)); DIS("smin.8 d0, d3, d4", fp.SMIN(8, D0, D3, D4)); DIS("smin.16 q0, q3, q4", fp.SMIN(16, Q0, Q3, Q4)); DIS("smax.16 d0, d3, d4", fp.SMAX(16, D0, D3, D4)); @@ -967,6 +983,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float DIS("shll2.32.16 q1, q2, #16", fp.SHLL2(16, Q1, Q2)); DIS("shll2.16.8 q1, q2, #8", fp.SHLL2(8, Q1, Q2)); DIS("shrn.32.64 d0, q3, #8", fp.SHRN(32, D0, Q3, 8)); + DIS("usra.32 q1, q2, #21", fp.USRA(32, Q1, Q2, 21)); + DIS("usra.16 d1, d2, #3", fp.USRA(16, D1, D2, 3)); DIS("shrn.8.16 d0, q3, #4", fp.SHRN(8, D0, Q3, 4)); DIS("shrn2.16.32 q0, q3, #16", fp.SHRN2(16, Q0, Q3, 16)); DIS("shrn2.32.64 q0, q3, #1", fp.SHRN2(32, Q0, Q3, 1));