Merge pull request #22568 from hrydgard/accurate-vdot-ir

Accurate VFPU dot products in every IR backend, on by default
This commit is contained in:
Henrik Rydgård authored and GitHub committed 2026-10-15 00:38:49 +02:00
commit f672916c77
49 files changed
+1291 -46

No files matched your search

+41
View File
@@ -3080,6 +3080,10 @@ void ARM64FloatEmitter::AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(0, 0, 3, Rd, Rn, Rm);
}
void ARM64FloatEmitter::BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(0, 1, 3, Rd, Rn, Rm);
}
void ARM64FloatEmitter::EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(1, 0, 3, Rd, Rn, Rm);
@@ -3196,6 +3200,37 @@ void ARM64FloatEmitter::UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(1, EncodeSize(size), 0xC, Rd, Rn, Rm);
}
void ARM64FloatEmitter::ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(0, EncodeSize(size), 0x10, Rd, Rn, Rm);
}
void ARM64FloatEmitter::SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(1, EncodeSize(size), 0x10, Rd, Rn, Rm);
}
void ARM64FloatEmitter::UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(1, EncodeSize(size), 0x14, Rd, Rn, Rm);
}
void ARM64FloatEmitter::USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(1, EncodeSize(size), 0x8, Rd, Rn, Rm);
}
void ARM64FloatEmitter::UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
_assert_msg_(IsQuad(Rd) && !IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__);
Write32((1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd));
}
void ARM64FloatEmitter::UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
_assert_msg_(IsQuad(Rd) && IsQuad(Rn) && src_size < 64, "%s: bad registers or size", __FUNCTION__);
Write32((1 << 30) | (1 << 29) | (0xE << 24) | (EncodeSize(src_size) << 22) | (1 << 21) | (DecodeReg(Rm) << 16) | (0xC << 12) | (DecodeReg(Rn) << 5) | DecodeReg(Rd));
}
void ARM64FloatEmitter::ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn)
{
_assert_msg_(size < 64 && (size < 32 || IsQuad(Rn)), "%s: bad size", __FUNCTION__);
Write32((IsQuad(Rn) << 30) | (0xE << 24) | (EncodeSize(size) << 22) | (0x18 << 17) | (0x1B << 12) | (2 << 10) | (DecodeReg(Rn) << 5) | DecodeReg(Rd));
}
void ARM64FloatEmitter::SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm)
{
EmitThreeSame(0, EncodeSize(size), 0xD, Rd, Rn, Rm);
@@ -3846,6 +3881,12 @@ void ARM64FloatEmitter::SHL(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) {
EmitShiftImm(IsQuad(Rd), false, imm >> 3, imm & 7, 0xA, Rd, Rn);
}
void ARM64FloatEmitter::USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) {
_assert_msg_(shift > 0 && shift <= dest_size, "%s shift amount out of range!", __FUNCTION__);
u32 imm = EncodeImmShiftRight(dest_size, shift);
EmitShiftImm(IsQuad(Rd), true, imm >> 3, imm & 7, 0x2, Rd, Rn);
}
void ARM64FloatEmitter::USHR(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift) {
_assert_msg_(shift < dest_size, "%s shift amount must less than the element size!", __FUNCTION__);
u32 imm = EncodeImmShiftRight(dest_size, shift);
+15
View File
@@ -854,6 +854,8 @@ public:
// Vector
void AND(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
// Rn & ~Rm.
void BIC(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void EOR(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void BSL(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void BIT(ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
@@ -888,6 +890,17 @@ public:
void UMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void UMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void UMAXP(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
// Integer vector add and subtract.
void ADD(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void SUB(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void USHL(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
// Widening multiplies: Rd (a quad) gets the products of the lower (UMULL) or upper (UMULL2)
// halves of Rn and Rm, src_size bits each.
void UMULL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void UMULL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
// Sum of all lanes, into a scalar of the lane size.
void ADDV(u8 size, ARM64Reg Rd, ARM64Reg Rn);
void SMIN(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
void SMAX(u8 size, ARM64Reg Rd, ARM64Reg Rn, ARM64Reg Rm);
@@ -995,6 +1008,8 @@ public:
void SHLL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn);
void SHLL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn);
void SHRN(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
// Rd += Rn >> shift, per lane.
void USRA(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
void SHRN2(u8 dest_size, ARM64Reg Rd, ARM64Reg Rn, u32 shift);
void SXTL(u8 src_size, ARM64Reg Rd, ARM64Reg Rn);
void SXTL2(u8 src_size, ARM64Reg Rd, ARM64Reg Rn);
+3
View File
@@ -36,6 +36,7 @@ list(APPEND CoreExtra
MIPS/ARM/ArmIRRegCache.cpp
MIPS/ARM/ArmIRJit.h
MIPS/ARM/ArmIRRegCache.h
MIPS/ARM/ArmVfpuDot.h
)
list(APPEND CoreExtra
@@ -50,6 +51,7 @@ list(APPEND CoreExtra
MIPS/ARM64/Arm64IRJit.h
MIPS/ARM64/Arm64IRRegCache.cpp
MIPS/ARM64/Arm64IRRegCache.h
MIPS/ARM64/Arm64VfpuDot.h
Util/DisArm64.cpp
)
@@ -65,6 +67,7 @@ list(APPEND CoreExtra
MIPS/x86/X64IRJit.h
MIPS/x86/X64IRRegCache.cpp
MIPS/x86/X64IRRegCache.h
MIPS/x86/X64VfpuDot.h
)
list(APPEND CoreExtra
+5
View File
@@ -199,6 +199,7 @@ static const CommandLineParam g_autoParams[] = {
{POFF(compare), CmdParamType::Bool, "compare", 'c', "Enable comparison mode", CmdLineMode::Headless},
{POFF(bench), CmdParamType::Bool, "bench", 'b', "Enable benchmark mode", CmdLineMode::Headless},
{POFF(oldAtrac), CmdParamType::Bool, "old-atrac", '\0', "Use old ATRAC decoder"},
{POFF(accurateVdot), CmdParamType::Bool, "accurate-vdot", '\0', "Compute VFPU dot products bit-exactly like the PSP (default; =false for the faster approximation)"},
{POFF(memStickInserted), CmdParamType::BoolInverse, "no-memstick", '\0', "Run with the memory stick removed"},
{POFF(log), CmdParamType::String, "log", '\0', "Output log to FILE", CmdLineMode::Application},
{POFF(enableLogging), CmdParamType::Bool, "log", '\0', "Full log output, not just emulated printfs", CmdLineMode::Headless},
@@ -610,6 +611,10 @@ void CommandLineOptions::ApplyToConfig() const {
g_Config.bUseOldAtrac = oldAtrac.value();
g_Config.DoNotSaveSetting(&g_Config.bUseOldAtrac);
}
if (accurateVdot.has_value()) {
g_Config.bAccurateVdot = accurateVdot.value();
g_Config.DoNotSaveSetting(&g_Config.bAccurateVdot);
}
if (root.has_value()) {
// No DoNotSaveSetting() here - mountRoot isn't an ordinary setting and never gets
+1
View File
@@ -166,6 +166,7 @@ struct CommandLineOptions {
bool optionS = false; // a legacy option
std::optional<bool> oldAtrac;
std::optional<bool> accurateVdot;
// Headless options that may also be mildly useful in application mode
std::optional<int> resolutionScale;
+1 -1
View File
@@ -114,7 +114,7 @@ void Compatibility::CheckSettings(IniFile &iniFile, const std::string &gameID) {
CheckSetting(iniFile, gameID, "GoWFramerateHack60", &flags_.GoWFramerateHack60);
CheckSetting(iniFile, gameID, "FramerateHack30", &flags_.FramerateHack30);
CheckSetting(iniFile, gameID, "HideISOFiles", &flags_.HideISOFiles);
CheckSetting(iniFile, gameID, "MoreAccurateVMMUL", &flags_.MoreAccurateVMMUL);
CheckSetting(iniFile, gameID, "ForceAccurateVdot", &flags_.ForceAccurateVdot);
CheckSetting(iniFile, gameID, "ForceSoftwareRenderer", &flags_.ForceSoftwareRenderer);
CheckSetting(iniFile, gameID, "DarkStalkersPresentHack", &flags_.DarkStalkersPresentHack);
CheckSetting(iniFile, gameID, "ReportSmallMemstick", &flags_.ReportSmallMemstick);
+1 -1
View File
@@ -69,7 +69,7 @@ struct CompatFlags {
bool GoWFramerateHack60;
bool FramerateHack30;
bool HideISOFiles;
bool MoreAccurateVMMUL;
bool ForceAccurateVdot;
bool ForceSoftwareRenderer;
bool DarkStalkersPresentHack;
bool ReportSmallMemstick;
+1
View File
@@ -436,6 +436,7 @@ static const ConfigSetting cpuSettings[] = {
ConfigSetting("IOTimingMethod", SETTING(g_Config, iIOTimingMethod), IOTIMING_FAST, CfgFlag::PER_GAME | CfgFlag::REPORT),
ConfigSetting("FastMemoryAccess", SETTING(g_Config, bFastMemory), true, CfgFlag::PER_GAME),
ConfigSetting("FunctionReplacements", SETTING(g_Config, bFuncReplacements), true, CfgFlag::PER_GAME | CfgFlag::REPORT),
ConfigSetting("AccurateVdot", SETTING(g_Config, bAccurateVdot), true, CfgFlag::PER_GAME | CfgFlag::REPORT),
ConfigSetting("HideSlowWarnings", SETTING(g_Config, bHideSlowWarnings), false, CfgFlag::DEFAULT),
ConfigSetting("HideStateWarnings", SETTING(g_Config, bHideStateWarnings), false, CfgFlag::DEFAULT),
ConfigSetting("JitDisableFlags", SETTING(g_Config, uJitDisableFlags), (uint32_t)0, CfgFlag::PER_GAME),
+2
View File
@@ -235,6 +235,8 @@ public:
bool bWow64WarningDismissed;
bool bForceLagSync;
bool bFuncReplacements;
// Compute the VFPU's dot products (vdot, vmmul, vtfm and the rest) bit-exactly like the PSP.
bool bAccurateVdot;
bool bHideSlowWarnings;
bool bHideStateWarnings;
uint32_t uJitDisableFlags;
+3
View File
@@ -870,6 +870,7 @@
<ClInclude Include="MemFault.h" />
<ClInclude Include="MIPS\ARM64\Arm64IRJit.h" />
<ClInclude Include="MIPS\ARM64\Arm64IRRegCache.h" />
<ClInclude Include="MIPS\ARM64\Arm64VfpuDot.h" />
<ClInclude Include="MIPS\fake\FakeJit.h" />
<ClInclude Include="MIPS\IR\IRAnalysis.h" />
<ClInclude Include="MIPS\IR\IRFrontend.h" />
@@ -1018,6 +1019,7 @@
<ClInclude Include="MemMapHelpers.h" />
<ClInclude Include="MIPS\ARM\ArmIRJit.h" />
<ClInclude Include="MIPS\ARM\ArmIRRegCache.h" />
<ClInclude Include="MIPS\ARM\ArmVfpuDot.h" />
<ClInclude Include="MIPS\JitCommon\JitBlockCache.h" />
<ClInclude Include="MIPS\JitCommon\JitCommon.h" />
<ClInclude Include="MIPS\JitCommon\JitState.h" />
@@ -1063,6 +1065,7 @@
<ClInclude Include="WaveFile.h" />
<ClInclude Include="WebServer.h" />
<ClInclude Include="MIPS\x86\X64IRRegCache.h" />
<ClInclude Include="MIPS\x86\X64VfpuDot.h" />
</ItemGroup>
<ItemGroup>
<None Include="..\assets\infra-dns.json" />
+9
View File
@@ -1725,6 +1725,9 @@
<ClInclude Include="MIPS\ARM\ArmIRRegCache.h">
<Filter>MIPS\ARM</Filter>
</ClInclude>
<ClInclude Include="MIPS\ARM\ArmVfpuDot.h">
<Filter>MIPS\ARM</Filter>
</ClInclude>
<ClInclude Include="Font\PGF.h">
<Filter>Font</Filter>
</ClInclude>
@@ -2157,6 +2160,9 @@
<ClInclude Include="MIPS\x86\X64IRRegCache.h">
<Filter>MIPS\x86</Filter>
</ClInclude>
<ClInclude Include="MIPS\x86\X64VfpuDot.h">
<Filter>MIPS\x86</Filter>
</ClInclude>
<ClInclude Include="MIPS\x86\X64IRJit.h">
<Filter>MIPS\x86</Filter>
</ClInclude>
@@ -2166,6 +2172,9 @@
<ClInclude Include="MIPS\ARM64\Arm64IRRegCache.h">
<Filter>MIPS\ARM64</Filter>
</ClInclude>
<ClInclude Include="MIPS\ARM64\Arm64VfpuDot.h">
<Filter>MIPS\ARM64</Filter>
</ClInclude>
<ClInclude Include="Util\GameDB.h">
<Filter>Util</Filter>
</ClInclude>
+3
View File
@@ -21,6 +21,7 @@
#include "Common/Log.h"
#include "Common/MemoryUtil.h"
#include "Core/CoreTiming.h"
#include "Core/MIPS/ARM/ArmVfpuDot.h"
#include "Core/MemMap.h"
#include "Core/MIPS/ARM/ArmIRJit.h"
#include "Core/MIPS/ARM/ArmIRRegCache.h"
@@ -100,6 +101,8 @@ void ArmJitBackend::GenerateFixedCode(MIPSState *mipsState) {
B(R_LR);
}
vfpuDot_ = GenerateArmVfpuDotCall(this, CTXREG, GenerateArmVfpuDotSpecial(this, CTXREG));
hooks_.enterDispatcher = (IRNativeFuncNoArg)AlignCode16();
// Nine registers, so we're misaligned by 4 for calls, which the SUB fixes.
+27
View File
@@ -18,8 +18,10 @@
#include "ppsspp_config.h"
#if PPSSPP_ARCH(ARM)
#include "Common/CPUDetect.h"
#include "Core/MIPS/ARM/ArmIRJit.h"
#include "Core/MIPS/ARM/ArmIRRegCache.h"
#include "Core/MIPS/ARM/ArmVfpuDot.h"
// This file contains compilation for Vec4 operations.
//
@@ -170,12 +172,37 @@ void ArmJitBackend::CompIR_VecHoriz(IRInst inst) {
VMOV(regs_.F(inst.dest), SCRATCHF1);
break;
case IROp::Vec4VfpuDot:
if (cpu_info.bNEON) {
CompVfpuDot(inst);
} else {
CompIR_Generic(inst);
}
break;
default:
INVALIDOP;
break;
}
}
void ArmJitBackend::CompVfpuDot(IRInst inst) {
regs_.Map(inst);
// The lanes into Q8 and Q9, through S0-S3 (Q0), since Q8-Q15 have no S registers.
for (int i = 0; i < 4; i++)
VMOV((ARMReg)(S0 + i), regs_.F(inst.src1 + i));
VMOV_neon(Q8, Q0);
for (int i = 0; i < 4; i++)
VMOV((ARMReg)(S0 + i), regs_.F(inst.src2 + i));
VMOV_neon(Q9, Q0);
// The shared routine clobbers these too, so whatever guest state is there gets flushed. They come
// late in the allocation order, so usually that's nothing.
regs_.FlushHostRegs({ R2, R3, R12, R14 });
BL(vfpuDot_);
VMOV(regs_.F(inst.dest), R3);
}
void ArmJitBackend::CompIR_VecPack(IRInst inst) {
CONDITIONAL_DISABLE;
+6 -1
View File
@@ -63,7 +63,12 @@ bool ArmJitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) {
return false;
IRBlock *block = irBlockCache->GetBlock(block_num);
BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32));
// An accurate dot product gathers its lanes and can flush a few registers around its call.
size_t sizeEstimate = 0;
const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block);
for (int i = 0; i < block->GetNumIRInstructions(); ++i)
sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32;
BeginWrite(std::min(GetSpaceLeft(), sizeEstimate));
u32 startPC = block->GetOriginalStart();
bool wroteCheckedOffset = false;
+4
View File
@@ -116,6 +116,8 @@ private:
void CompFloatToIntWithRounding(ArmGen::ARMReg dest, ArmGen::ARMReg src, int rmode);
// Puts the masked address in SCRATCH1, as an offset from MEMBASEREG. Destroys SCRATCH2.
void PrepareSrc1Address(IRInst inst);
// Vec4VfpuDot with NEON (see ArmVfpuDot.h).
void CompVfpuDot(IRInst inst);
JitOptions &jo;
ArmIRRegCache regs_;
@@ -127,6 +129,8 @@ private:
const u8 *dispatcherNoCheck_ = nullptr;
const u8 *restoreRoundingMode_ = nullptr;
const u8 *applyRoundingMode_ = nullptr;
// See GenerateArmVfpuDotCall().
const u8 *vfpuDot_ = nullptr;
int jitStartOffset_ = 0;
int compilingBlockNum_ = -1;
+5
View File
@@ -71,6 +71,11 @@ const int *ArmIRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c
}
}
void ArmIRRegCache::FlushHostRegs(std::initializer_list<ARMReg> regs) {
for (ARMReg r : regs)
FlushNativeReg((IRNativeReg)r);
}
void ArmIRRegCache::FlushBeforeCall() {
// These are not preserved by calls.
for (int r : { R2, R3, R12, R14 })
+4
View File
@@ -18,6 +18,8 @@
#pragma once
#include "ppsspp_config.h"
#include <initializer_list>
#if PPSSPP_ARCH(ARM)
#include "Common/ArmEmitter.h"
@@ -58,6 +60,8 @@ public:
ArmGen::ARMReg MapWithFPRTemp(const IRInst &inst);
void FlushBeforeCall();
// Flushes whatever guest state these registers hold, for code that clobbers them.
void FlushHostRegs(std::initializer_list<ArmGen::ARMReg> regs);
void FlushAll(bool gprs = true, bool fprs = true) override;
ArmGen::ARMReg GetAndLockTempGPR();
+199
View File
@@ -0,0 +1,199 @@
// Copyright (c) 2012- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#pragma once
#include <cstddef>
#include <cstring>
#include "Common/ArmEmitter.h"
#include "Core/MIPS/MIPS.h"
#include "Core/MIPS/MIPSVFPUUtils.h"
namespace MIPSComp {
// The core registers EmitArmVfpuDot() works in, all different. The result ends up in result.
struct ArmVfpuDotRegs {
ArmGen::ARMReg sum;
ArmGen::ARMReg ehi;
ArmGen::ARMReg sign;
ArmGen::ARMReg result;
ArmGen::ARMReg lz;
ArmGen::ARMReg temp;
};
// Emits vfpu_dot() of the Vec4s in Q8 (x) and Q9 (y): vfpu_dot_neon(), in ARMv7 NEON after what clang
// makes of it, with Q0 and Q8-Q15 as temps. The end is in 32-bit integers rather than
// vfpu_dot_finish()'s 64-bit ones (checked bit-exact against it): the significand goes to bit 30
// instead of 62, the exponent is clamped to [-2, 254], and the zero test is on the result plus 2^24, so
// that it and the inf test are unsigned compares. Inf and NaN aren't handled: the returned branch is
// taken for them, with Q8 and Q9 intact. Clobbers the flags.
inline ArmGen::FixupBranch EmitArmVfpuDot(ArmGen::ARMXEmitter *e, const ArmVfpuDotRegs &r) {
using namespace ArmGen;
const ARMReg x = Q8, y = Q9;
// Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF added,
// above any real exponent sum, so one maximum finds the alignment and the specials (q11 = e,
// q13 = normal).
e->MOVI2R(r.ehi, 0x7F800000);
e->VDUP(I_32, Q10, r.ehi);
e->VAND(Q11, x, Q10);
e->VAND(Q12, y, Q10);
e->VTST(I_32, Q13, x, Q10);
e->VTST(I_32, Q14, y, Q10);
e->VAND(Q13, Q13, Q14);
e->VMAX(I_32 | I_UNSIGNED, Q14, Q11, Q12);
e->VCEQ(I_32, Q14, Q14, Q10);
e->VADD(I_32, Q11, Q11, Q12);
e->VSHR(I_32 | I_UNSIGNED, Q11, Q11, 23);
e->VAND(Q11, Q11, Q13);
e->VMOV_imm(I_32, Q12, VIMM__x1__x1, 0x07); // 0x7FF
e->VAND(Q14, Q14, Q12);
e->VADD(I_32, Q11, Q11, Q14);
e->VPMAX(I_32 | I_UNSIGNED, D24, D22, D23);
e->VPMAX(I_32 | I_UNSIGNED, D24, D24, D24);
e->VMOV_neon(I_32, r.ehi, D24, 0);
// Real exponent sums are at most 508.
e->CMP(r.ehi, AssumeMakeOperand2(0x7F0));
FixupBranch special = e->B_CC(CC_HS);
// q0 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q10 = p).
e->VEOR(Q0, x, y);
e->VSHR(I_32 | I_SIGNED, Q0, Q0, 31);
e->VMOV_imm(I_32, Q10, VIMM_x11_x11, 0x7F); // 0x7FFFFF
e->VAND(Q15, x, Q10);
e->VORR_imm(I_32, Q15, VIMM_x___x__, 0x80);
e->VAND(Q14, y, Q10);
e->VORR_imm(I_32, Q14, VIMM_x___x__, 0x80);
e->VMULL(I_32 | I_UNSIGNED, Q8, D30, D28);
e->VMULL(I_32 | I_UNSIGNED, Q9, D31, D29);
e->VSHRN(I_64, D20, Q8, 21);
e->VSHRN(I_64, D21, Q9, 21);
e->VMOVN(I_64, D28, Q8);
e->VMOVN(I_64, D29, Q9);
e->VMOV_imm(I_32, Q15, VIMM_x11_x11, 0x1F); // (1 << 21) - 1
e->VAND(Q14, Q14, Q15);
e->VMOV_imm(I_32, Q15, VIMM___x___x, 1);
e->VMIN(I_32 | I_UNSIGNED, Q14, Q14, Q15);
e->VORR(Q10, Q10, Q14);
e->VAND(Q10, Q10, Q13);
// Align to the largest exponent by truncation, apply the signs, and sum exactly.
e->VDUP(I_32, Q12, D24, 0);
e->VSUB(I_32, Q11, Q11, Q12);
e->VMVN_imm(I_32, Q15, VIMM___x___x, 27); // -28
e->VMAX(I_32 | I_SIGNED, Q11, Q11, Q15);
e->VSHL(I_32 | I_UNSIGNED, Q10, Q10, Q11);
e->VEOR(Q10, Q10, Q0);
e->VSUB(I_32, Q10, Q10, Q0);
e->VADD(I_32, D20, D20, D21);
e->VPADD(I_32, D20, D20, D20);
e->VMOV_neon(I_32, r.sum, D20, 0);
// The end of vfpu_dot(): ehi = the largest exponent sum, sum = the sum, still scaled by 4.
e->AND(r.sign, r.sum, AssumeMakeOperand2(0x80000000));
e->CMP(r.sum, Operand2(0));
e->SetCC(CC_LT);
e->RSB(r.sum, r.sum, Operand2(0));
e->SetCC(CC_AL);
e->MOV(r.result, Operand2(r.sum, ST_LSR, 2));
e->ORR(r.lz, r.result, Operand2(1));
e->CLZ(r.lz, r.lz);
// Round to 24 bits, nearest even, with the top bit at bit 30.
e->SUB(r.temp, r.lz, Operand2(1));
e->MOV(r.result, Operand2(r.result, ST_LSL, r.temp));
e->UBFX(r.temp, r.result, 7, 1);
e->ADD(r.result, r.result, r.temp);
e->ADD(r.result, r.result, Operand2(0x3F));
e->MOV(r.result, Operand2(r.result, ST_LSR, 7));
// The exponent, clamped to [-2, 254], which doesn't change the result.
e->SUB(r.ehi, r.ehi, r.lz);
e->SUB(r.ehi, r.ehi, Operand2(120));
e->CMN(r.ehi, Operand2(2));
e->SetCC(CC_LT);
e->MVN(r.ehi, Operand2(1));
e->SetCC(CC_AL);
e->CMP(r.ehi, Operand2(254));
e->SetCC(CC_GT);
e->MOV(r.ehi, Operand2(254));
e->SetCC(CC_AL);
// Adding the significand, implicit bit and all, lets a carry from the rounding bump the exponent.
e->ADD(r.result, r.result, Operand2(r.ehi, ST_LSL, 23));
e->ADD(r.temp, r.result, AssumeMakeOperand2(0x01000000));
// 0x7F800000 isn't an immediate, so compare the exponent field.
e->MOV(r.lz, Operand2(r.result, ST_LSR, 23));
e->CMP(r.lz, Operand2(0xFF));
e->SetCC(CC_HS);
e->MOVI2R(r.result, 0x7F800000);
e->SetCC(CC_AL);
// Zero if the result is below the smallest normal, or the sum is.
e->CMP(r.temp, AssumeMakeOperand2(0x01800000));
e->SetCC(CC_LO);
e->MOV(r.result, Operand2(0));
e->SetCC(CC_AL);
e->CMP(r.sum, Operand2(4));
e->SetCC(CC_LO);
e->MOV(r.result, Operand2(0));
e->SetCC(CC_AL);
e->ORR(r.result, r.result, r.sign);
return special;
}
static void ArmVfpuDotSpecialC(MIPSState *mips) {
const float *args = (const float *)mips->vfpuDotArgs;
const float result = vfpu_dot_reference(args, args + 4);
memcpy(&mips->vfpuDotResult, &result, sizeof(result));
}
// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in
// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. Called with the
// stack aligned to 8, and preserves everything that a call may clobber, but Q8-Q15.
inline const u8 *GenerateArmVfpuDotSpecial(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx) {
using namespace ArmGen;
const u8 *start = e->AlignCode16();
e->PUSH(8, R0, R1, R2, R3, R4, R5, R12, R_LR);
e->VPUSH(D0, 8);
e->QuickCallFunctionR(R12, &ArmVfpuDotSpecialC, ctx);
e->VPOP(D0, 8);
e->POP(8, R0, R1, R2, R3, R4, R5, R12, R_PC);
return start;
}
// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of Q8 and Q9, into R3,
// with special (from GenerateArmVfpuDotSpecial()) for inf and NaN. It clobbers Q0, Q8-Q15, R0-R3, R12,
// LR and the flags, and saves only R4: the call site flushes whatever of the others holds guest
// state, which in most blocks is nothing. Q8-Q15 are never allocated.
inline const u8 *GenerateArmVfpuDotCall(ArmGen::ARMXEmitter *e, ArmGen::ARMReg ctx, const u8 *special) {
using namespace ArmGen;
const u8 *start = e->AlignCode16();
e->PUSH(2, R4, R_LR);
const ArmVfpuDotRegs regs{ R0, R1, R2, R3, R12, R4 };
FixupBranch specialBranch = EmitArmVfpuDot(e, regs);
e->POP(2, R4, R_PC);
// The operands are still in Q8 and Q9.
e->SetJumpTarget(specialBranch);
e->ADDI2R(R0, ctx, offsetof(MIPSState, vfpuDotArgs), R1);
for (int i = 0; i < 4; i++)
e->VSTR((ARMReg)(D16 + i), R0, i * 8);
e->BL(special);
e->LDR(R3, ctx, offsetof(MIPSState, vfpuDotResult));
e->POP(2, R4, R_PC);
return start;
}
} // namespace MIPSComp
+4
View File
@@ -22,6 +22,7 @@
#include "Common/Log.h"
#include "Core/CoreTiming.h"
#include "Core/MemMap.h"
#include "Core/MIPS/ARM64/Arm64VfpuDot.h"
#include "Core/MIPS/ARM64/Arm64IRJit.h"
#include "Core/MIPS/ARM64/Arm64IRRegCache.h"
#include "Core/MIPS/JitCommon/JitCommon.h"
@@ -264,6 +265,9 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) {
RET();
}
vfpuDotSpecial_ = GenerateArm64VfpuDotSpecial(this, &fp_, CTXREG);
vfpuDot_ = GenerateArm64VfpuDotCall(this, &fp_, CTXREG, vfpuDotSpecial_);
// Leave this at the end, add more stuff above.
if (enableDisasm) {
std::vector<std::string> lines = DisassembleArm64(disasmStart, (int)(GetCodePtr() - disasmStart));
+27
View File
@@ -24,6 +24,7 @@
#include "Core/MemMap.h"
#include "Core/MIPS/ARM64/Arm64IRJit.h"
#include "Core/MIPS/ARM64/Arm64IRRegCache.h"
#include "Core/MIPS/ARM64/Arm64VfpuDot.h"
// This file contains compilation for vector instructions.
//
@@ -599,6 +600,32 @@ void Arm64JitBackend::CompIR_VecHoriz(IRInst inst) {
}
break;
case IROp::Vec4VfpuDot:
{
// The operands go to the shared routine in q0 and q1, and the result comes back in w1. The
// routine clobbers q4-q6 and x0-x3 too, so whatever guest state is there gets flushed: these
// come late in the allocation order, so usually that's nothing.
regs_.SpillLockFPR(inst.src1, inst.src2);
regs_.MapVec4(inst.src1);
regs_.MapVec4(inst.src2);
fp_.MOV(Q0, regs_.FQ(inst.src1));
fp_.MOV(Q1, regs_.FQ(inst.src2));
regs_.ReleaseSpillLockFPR(inst.src1, inst.src2);
regs_.FlushHostRegs({ Q4, Q5, Q6, W0, W1, W2, W3 });
BL(vfpuDot_);
fp_.FMOV(SCRATCHF1, W1);
if (Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) {
// The dest lane is in a mapped Vec4.
regs_.MapVec4(inst.dest & ~3, MIPSMap::DIRTY);
fp_.INS(32, regs_.FQ(inst.dest & ~3), inst.dest & 3, EncodeRegToQuad(SCRATCHF1), 0);
} else {
regs_.MapFPR(inst.dest, MIPSMap::NOINIT);
fp_.FMOV(regs_.F(inst.dest), SCRATCHF1);
}
break;
}
default:
INVALIDOP;
break;
+6 -1
View File
@@ -71,7 +71,12 @@ bool Arm64JitBackend::CompileBlock(IRBlockCache *irBlockCache, int block_num) {
return false;
IRBlock *block = irBlockCache->GetBlock(block_num);
BeginWrite(std::min(GetSpaceLeft(), (size_t)block->GetNumIRInstructions() * 32));
// An accurate dot product can flush several registers around its call.
size_t sizeEstimate = 0;
const IRInst *estimateInsts = irBlockCache->GetBlockInstructionPtr(*block);
for (int i = 0; i < block->GetNumIRInstructions(); ++i)
sizeEstimate += estimateInsts[i].op == IROp::Vec4VfpuDot ? 96 : 32;
BeginWrite(std::min(GetSpaceLeft(), sizeEstimate));
u32 startPC = block->GetOriginalStart();
bool wroteCheckedOffset = false;
+4
View File
@@ -145,6 +145,10 @@ private:
// Indexed by FPCR FZ:RN bits for convenience. Uses SCRATCH2.
const u8 *convertS0ToSCRATCH1_[8];
// See GenerateArm64VfpuDotSpecial().
const u8 *vfpuDotSpecial_ = nullptr;
// See GenerateArm64VfpuDotCall().
const u8 *vfpuDot_ = nullptr;
// Note: mutable state used at runtime.
const u8 *currentRoundingFunc_ = nullptr;
+9
View File
@@ -147,6 +147,15 @@ void Arm64IRRegCache::EmitSaveStaticRegisters() {
}
}
void Arm64IRRegCache::FlushHostRegs(std::initializer_list<ARM64Reg> regs) {
for (ARM64Reg r : regs) {
if (IsVector(r))
FlushNativeReg(VFPToNativeReg(EncodeRegToSingle(r)));
else
FlushNativeReg(GPRToNativeReg(DecodeReg(r)));
}
}
void Arm64IRRegCache::FlushBeforeCall() {
// These registers are not preserved by function calls.
auto isGPRSaved = [&](IRNativeReg nreg) {
+5
View File
@@ -18,6 +18,9 @@
#pragma once
#include "ppsspp_config.h"
#include <initializer_list>
// In other words, PPSSPP_ARCH(ARM64) || DISASM_ALL.
#if PPSSPP_ARCH(ARM64) || (PPSSPP_PLATFORM(WINDOWS) && !defined(__LIBRETRO__))
@@ -65,6 +68,8 @@ public:
Arm64Gen::ARM64Reg MapWithFPRTemp(const IRInst &inst);
void FlushBeforeCall();
// Flushes whatever guest state these registers hold, for code that clobbers them.
void FlushHostRegs(std::initializer_list<Arm64Gen::ARM64Reg> regs);
void FlushAll(bool gprs = true, bool fprs = true) override;
Arm64Gen::ARM64Reg GetAndLockTempGPR();
+193
View File
@@ -0,0 +1,193 @@
// Copyright (c) 2012- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#pragma once
#include <cstddef>
#include <vector>
#include "Common/Arm64Emitter.h"
#include "Core/MIPS/MIPS.h"
#include "Core/MIPS/MIPSVFPUUtils.h"
namespace MIPSComp {
// The registers EmitArm64VfpuDot() works in. x and y are only read. q are seven vector temps and w
// six GPR temps, none of them x or y, except that q[0] and q[1] may be x and y themselves.
struct Arm64VfpuDotRegs {
Arm64Gen::ARM64Reg x;
Arm64Gen::ARM64Reg y;
Arm64Gen::ARM64Reg q[7];
Arm64Gen::ARM64Reg w[6];
};
// Emits vfpu_dot() of the Vec4s x and y: vfpu_dot_neon(), scheduled after what clang makes of it.
// The result ends up in w[3]. Inf and NaN aren't handled: the returned branch is taken for them,
// and the caller sends those to GenerateArm64VfpuDotSpecial()'s routine. Clobbers the flags.
inline Arm64Gen::FixupBranch EmitArm64VfpuDot(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, const Arm64VfpuDotRegs &r) {
using namespace Arm64Gen;
const ARM64Reg x = EncodeRegToQuad(r.x);
const ARM64Reg y = EncodeRegToQuad(r.y);
ARM64Reg q[7];
for (int i = 0; i < 7; i++)
q[i] = EncodeRegToQuad(r.q[i]);
const ARM64Reg emax = DecodeReg(r.w[0]);
const ARM64Reg sum = DecodeReg(r.w[1]);
const ARM64Reg sign = DecodeReg(r.w[2]);
const ARM64Reg m = DecodeReg(r.w[3]);
const ARM64Reg t2 = DecodeReg(r.w[4]);
const ARM64Reg t3 = DecodeReg(r.w[5]);
// Exponents: a zero or denormal operand makes the product zero, and inf or NaN gets 0x7FF
// added, above any real exponent sum, so one maximum finds the alignment and the specials
// (q3 = e, q5 = normal).
e->MOVI2R(emax, 0x7F800000);
fp->DUP(32, q[2], emax);
fp->AND(q[3], x, q[2]);
fp->AND(q[4], y, q[2]);
fp->CMTST(32, q[5], x, q[2]);
fp->CMEQ(32, q[6], q[4]);
fp->BIC(q[5], q[5], q[6]);
fp->UMAX(32, q[6], q[3], q[4]);
fp->CMEQ(32, q[6], q[6], q[2]);
fp->ADD(32, q[3], q[3], q[4]);
fp->USHR(32, q[3], q[3], 23);
fp->AND(q[3], q[3], q[5]);
fp->MOVI(32, q[2], 0x07, 8, true); // 0x7FF
fp->AND(q[6], q[6], q[2]);
fp->ADD(32, q[3], q[3], q[6]);
fp->UMAXP(32, q[2], q[3], q[3]);
fp->UMAXP(32, q[2], q[2], q[2]);
fp->UMOV(32, emax, q[2], 0);
e->CMPI2R(emax, 0x7FF);
FixupBranch special = e->B(CC_HS);
// q4 = the signs. 24x24-bit products, kept to 2 extra bits with round-to-odd (q1 = p).
fp->EOR(q[4], x, y);
fp->SSHR(32, q[4], q[4], 31);
fp->MOVI(32, q[6], 0x7F, 16, true); // 0x7FFFFF
fp->AND(q[0], x, q[6]);
fp->ORR(32, q[0], 0x80, 16);
fp->AND(q[1], y, q[6]);
fp->ORR(32, q[1], 0x80, 16);
fp->UMULL(32, q[6], EncodeRegToDouble(q[0]), EncodeRegToDouble(q[1]));
fp->UMULL2(32, q[0], q[0], q[1]);
fp->SHRN(32, EncodeRegToDouble(q[1]), q[6], 21);
fp->SHRN2(32, q[1], q[0], 21);
fp->UZP1(32, q[6], q[6], q[0]);
fp->MOVI(32, q[0], 0x1F, 16, true); // (1 << 21) - 1
fp->AND(q[6], q[6], q[0]);
fp->MOVI(32, q[0], 1);
fp->UMIN(32, q[6], q[6], q[0]);
fp->ORR(q[1], q[1], q[6]);
fp->AND(q[1], q[1], q[5]);
// Align to the largest exponent by truncation, apply the signs, and sum exactly.
fp->DUP(32, q[2], q[2], 0);
fp->SUB(32, q[3], q[3], q[2]);
fp->MVNI(32, q[2], 27); // -28
fp->SMAX(32, q[3], q[3], q[2]);
fp->USHL(32, q[1], q[1], q[3]);
fp->EOR(q[1], q[1], q[4]);
fp->SUB(32, q[1], q[1], q[4]);
fp->ADDV(32, EncodeRegToSingle(q[1]), q[1]);
fp->UMOV(32, sum, q[1], 0);
// vfpu_dot_finish(): emax = the largest exponent sum, sum = the sum.
e->ANDI2R(sign, sum, 0x80000000);
e->CMPI2R(sum, 0);
e->CSNEG(sum, sum, sum, CC_GE);
e->LSR(m, sum, 2);
e->ORRI2R(t2, m, 1);
e->CLZ(t2, t2);
e->ADDI2R(t3, t2, 31);
e->LSLV(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3));
e->UBFX(EncodeRegTo64(t3), EncodeRegTo64(m), 39, 1);
e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3));
e->MOVI2R(EncodeRegTo64(t3), (1ULL << 38) - 1);
e->ADD(EncodeRegTo64(m), EncodeRegTo64(m), EncodeRegTo64(t3));
e->LSR(EncodeRegTo64(m), EncodeRegTo64(m), 39);
e->SUB(emax, emax, t2);
e->SUBI2R(emax, emax, 120);
e->SXTW(EncodeRegTo64(emax), emax);
e->ADD(EncodeRegTo64(emax), EncodeRegTo64(m), EncodeRegTo64(emax), ArithOption(EncodeRegTo64(emax), ST_LSL, 23));
e->MOVI2R(m, 0x7F7FFFFF);
e->CMP(EncodeRegTo64(emax), EncodeRegTo64(m));
e->MOVI2R(m, 0x7F800000);
e->CSEL(m, m, emax, CC_GT);
// Zero if the result is below the smallest normal, or the sum (still scaled by 4) is.
e->MOVI2R(t2, 0x00800000);
e->CMP(EncodeRegTo64(emax), EncodeRegTo64(t2));
e->CCMP(sum, 4, 0, CC_GE);
e->CSEL(m, WZR, m, CC_LO);
e->ORR(m, m, sign);
return special;
}
// Generates the routine for the inf and NaN dot products: vfpu_dot_reference() of the two Vec4s in
// mips->vfpuDotArgs, written to mips->vfpuDotResult. ctx holds the MIPSState pointer. It clobbers
// q0-q3, x16, x17, x30 and the flags, which the IR JIT keeps free of guest state, and preserves
// everything else.
inline const u8 *GenerateArm64VfpuDotSpecial(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx) {
using namespace Arm64Gen;
const s32 args = (s32)offsetof(MIPSState, vfpuDotArgs);
const s32 result = (s32)offsetof(MIPSState, vfpuDotResult);
const u8 *start = e->AlignCode16();
for (int i = 0; i < 16; i += 2)
e->STP(INDEX_PRE, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, -16);
e->STP(INDEX_PRE, X18, X30, SP, -16);
for (int i = 4; i < 32; i += 2)
fp->STP(128, INDEX_PRE, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, -32);
e->ADDI2R(X0, ctx, args);
e->ADDI2R(X1, ctx, args + 16);
e->MOVP2R(X16, &vfpu_dot_reference);
e->BLR(X16);
fp->STR(32, INDEX_UNSIGNED, S0, ctx, result);
for (int i = 30; i >= 4; i -= 2)
fp->LDP(128, INDEX_POST, (ARM64Reg)(Q0 + i), (ARM64Reg)(Q0 + i + 1), SP, 32);
e->LDP(INDEX_POST, X18, X30, SP, 16);
for (int i = 14; i >= 0; i -= 2)
e->LDP(INDEX_POST, (ARM64Reg)(X0 + i), (ARM64Reg)(X0 + i + 1), SP, 16);
e->RET();
return start;
}
// Generates the VFPU dot product routine the IR JIT calls with BL: vfpu_dot() of q0 and q1, into w1,
// with special (from GenerateArm64VfpuDotSpecial()) for inf and NaN. It clobbers q0-q6, x0-x3, x16,
// x17, x30 and the flags, and saves nothing: the call site flushes whatever of those holds guest
// state, which in most blocks is nothing.
inline const u8 *GenerateArm64VfpuDotCall(Arm64Gen::ARM64XEmitter *e, Arm64Gen::ARM64FloatEmitter *fp, Arm64Gen::ARM64Reg ctx, const u8 *special) {
using namespace Arm64Gen;
const u8 *start = e->AlignCode16();
const Arm64VfpuDotRegs regs{ Q0, Q1, { Q0, Q1, Q2, Q3, Q4, Q5, Q6 }, { W16, W17, W0, W1, W2, W3 } };
FixupBranch specialBranch = EmitArm64VfpuDot(e, fp, regs);
e->RET();
// The operands are still in q0 and q1.
e->SetJumpTarget(specialBranch);
e->ADDI2R(X16, ctx, offsetof(MIPSState, vfpuDotArgs));
fp->STP(128, INDEX_SIGNED, Q0, Q1, X16, 0);
e->STR(INDEX_PRE, X30, SP, -16);
e->BL(special);
e->LDR(INDEX_POST, X30, SP, 16);
e->LDR(INDEX_UNSIGNED, W1, ctx, offsetof(MIPSState, vfpuDotResult));
e->RET();
return start;
}
} // namespace MIPSComp
+257 -15
View File
@@ -150,6 +150,89 @@ namespace MIPSComp {
return true;
}
bool IRFrontend::UseAccurateVdot() const {
return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot;
}
// The lanes are already a Vec4 reg, used as is.
static bool DotLanesAreVec4(const IRFrontend::DotLane lanes[4]) {
for (int i = 0; i < 4; i++) {
if (lanes[i].reg < 0 || lanes[i].negate || lanes[i].reg != lanes[0].reg + i)
return false;
}
return (lanes[0].reg & 3) == 0;
}
static bool DotLanesRead(const IRFrontend::DotLane lanes[4], int vec) {
for (int i = 0; i < 4; i++) {
if (lanes[i].reg >= vec && lanes[i].reg < vec + 4)
return true;
}
return false;
}
// Building lane i into vec + i only clobbers what a later lane still has to read.
static bool DotLanesBuildableIn(const IRFrontend::DotLane lanes[4], int vec) {
for (int i = 0; i < 4; i++) {
for (int j = i + 1; j < 4; j++) {
if (lanes[j].reg == vec + i)
return false;
}
}
return true;
}
bool IRFrontend::BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec) {
const bool sReady = DotLanesAreVec4(s), tReady = DotLanesAreVec4(t);
// The D temp holds the destination when a D prefix masks lanes.
const int temps[] = { IRVTEMP_0, js.prefixD == 0 ? IRVTEMP_PFX_D : -1, IRVTEMP_PFX_T, IRVTEMP_PFX_S };
int sTemp = -1, tTemp = -1;
if (!sReady) {
for (int temp : temps) {
if (temp >= 0 && temp != avoid && !DotLanesRead(t, temp) && DotLanesBuildableIn(s, temp)) {
sTemp = temp;
break;
}
}
if (sTemp < 0)
return false;
}
if (!tReady) {
for (int temp : temps) {
// Built after s, so it mustn't overwrite what s is (built or not).
const int sVecReg = sReady ? s[0].reg : sTemp;
const bool hitsS = sVecReg >= temp && sVecReg < temp + 4;
if (temp >= 0 && temp != avoid && temp != sTemp && !hitsS && DotLanesBuildableIn(t, temp)) {
tTemp = temp;
break;
}
}
if (tTemp < 0)
return false;
}
auto build = [&](const DotLane lanes[4], int vec) {
for (int i = 0; i < 4; i++) {
if (lanes[i].reg < 0) {
ir.WriteSetConstantFloat(vec + i, lanes[i].constant);
} else if (lanes[i].negate) {
ir.Write(IROp::FNeg, vec + i, lanes[i].reg);
} else if (lanes[i].reg != vec + i) {
ir.Write(IROp::FMov, vec + i, lanes[i].reg);
}
}
};
// s's temp holds no lane of t, and t's temp isn't where s is, so neither clobbers the other.
if (!sReady)
build(s, sTemp);
if (!tReady)
build(t, tTemp);
*sVec = sReady ? s[0].reg : sTemp;
*tVec = tReady ? t[0].reg : tTemp;
return true;
}
// True if the prefix only touches lanes the op has. A position past the size may only be the
// identity, and a position within it may not name a lane past it (which zeroes the result lane
// on hardware, see cpu/vfpu/prefix_ctrl - the interpreter handles that).
@@ -698,6 +781,21 @@ namespace MIPSComp {
GetVectorRegsPrefixT(tregs, sz, vt);
GetVectorRegsPrefixD(dregs, V_Single, vd);
if (UseAccurateVdot()) {
// s's last lane is 1, so t's is added in.
DotLane sl[4], tl[4];
for (int i = 0; i < 4; i++) {
sl[i] = i < n - 1 ? DotReg(sregs[i]) : DotConst(i == n - 1 ? 1.0f : 0.0f);
tl[i] = i < n ? DotReg(tregs[i]) : DotConst(0.0f);
}
u8 sv, tv;
if (!BuildDotOperands(sl, tl, -1, &sv, &tv))
DISABLE;
ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv);
ApplyPrefixD(dregs, V_Single, vd);
return;
}
ir.Write(IROp::FMul, IRVTEMP_0, sregs[0], tregs[0]);
for (int i = 1; i < n; i++) {
@@ -731,16 +829,39 @@ namespace MIPSComp {
int n = GetNumVectorElements(sz);
// On the PSP, an S constant past n is added in too (cpu/vfpu/prefix_lanes), which
// IsPrefixWithinSize() lets through. Rare enough for the interpreter.
for (int i = n; i < 4; i++) {
if (VFPUPrefixIsConstant(js.prefixS, i))
DISABLE;
// IsPrefixWithinSize() lets through. The accurate dot product includes it, the rest leave it
// to the interpreter, as it's rare.
if (!UseAccurateVdot()) {
for (int i = n; i < 4; i++) {
if (VFPUPrefixIsConstant(js.prefixS, i))
DISABLE;
}
}
u8 sregs[4], dregs[1];
GetVectorRegsPrefixS(sregs, sz, _VS);
GetVectorRegsPrefixD(dregs, V_Single, _VD);
if (UseAccurateVdot()) {
// Dot products with constants from the T prefix in every lane: ones for vfad, 1/n for
// vavg (0 for vavg.s), as in the interpreter.
const bool vavg = ((op >> 16) & 31) == 7;
static const float vavgConst[4] = { 0.0f, 0.5f, 1.0f / 3.0f, 0.25f };
const float c = vavg ? vavgConst[n - 1] : 1.0f;
DotLane sl[4], tl[4];
for (int i = 0; i < 4; i++) {
const bool constantLane = i >= n && VFPUPrefixIsConstant(js.prefixS, i);
sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane ? VFPUPrefixConstant(js.prefixS, i) : 0.0f);
tl[i] = DotConst(c);
}
u8 sv, tv;
if (!BuildDotOperands(sl, tl, -1, &sv, &tv))
DISABLE;
ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv);
ApplyPrefixD(dregs, V_Single, _VD);
return;
}
// We have to start at +0.000 in case any values are -0.000.
ir.WriteSetConstantFloat(IRVTEMP_0, 0.0f);
for (int i = 0; i < n; ++i) {
@@ -777,11 +898,16 @@ namespace MIPSComp {
int n = GetNumVectorElements(sz);
// On the PSP, lanes past n where both prefixes are constants add their product, and the rest
// read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through, and they're
// rare enough for the interpreter.
for (int i = n; i < 4; i++) {
if (VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i))
DISABLE;
// read zero (cpu/vfpu/prefix_lanes). IsPrefixWithinSize() lets those through. The accurate
// dot product includes them, the rest leave them to the interpreter, as they're rare.
auto constantLane = [&](int i) {
return i >= n && VFPUPrefixIsConstant(js.prefixS, i) && VFPUPrefixIsConstant(js.prefixT, i);
};
if (!UseAccurateVdot()) {
for (int i = n; i < 4; i++) {
if (constantLane(i))
DISABLE;
}
}
// TODO: Force read one of them into regs? probably not.
@@ -790,6 +916,20 @@ namespace MIPSComp {
GetVectorRegsPrefixT(tregs, sz, vt);
GetVectorRegsPrefixD(dregs, V_Single, vd);
if (UseAccurateVdot()) {
DotLane sl[4], tl[4];
for (int i = 0; i < 4; i++) {
sl[i] = i < n ? DotReg(sregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixS, i) : 0.0f);
tl[i] = i < n ? DotReg(tregs[i]) : DotConst(constantLane(i) ? VFPUPrefixConstant(js.prefixT, i) : 0.0f);
}
u8 sv, tv;
if (!BuildDotOperands(sl, tl, -1, &sv, &tv))
DISABLE;
ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv);
ApplyPrefixD(dregs, V_Single, vd);
return;
}
if (IsVec4(sz, sregs) && IsVec4(sz, tregs)) {
if (IsOverlapSafe(dregs[0], n, sregs, n, tregs)) {
ir.Write(IROp::Vec4Dot, dregs[0], sregs[0], tregs[0]);
@@ -1615,12 +1755,6 @@ namespace MIPSComp {
DISABLE;
}
if (PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) {
// Fall back to interpreter, which has the accurate implementation.
// Later we might do something more optimized here.
DISABLE;
}
// Matrix multiply (weird prefixes)
// D[0 .. N, 0 .. M] = S[0 .. N, 0 .. M]' * T[0 .. N, 0 .. M]
// Note: Behaves as if it's implemented through a series of vdots.
@@ -1635,6 +1769,35 @@ namespace MIPSComp {
MatrixOverlapType soverlap = GetMatrixOverlap(vs, vd, sz);
MatrixOverlapType toverlap = GetMatrixOverlap(vt, vd, sz);
if (UseAccurateVdot()) {
// d[a][b] = s[b] dot t[a], each a row of n lanes and zeros, as in the interpreter. With
// no overlap, the results can go straight to d.
if (soverlap || toverlap)
DISABLE;
u8 sregs[16], tregs[16], dregs[16];
GetMatrixRegs(sregs, sz, vs);
GetMatrixRegs(tregs, sz, vt);
GetMatrixRegs(dregs, sz, vd);
for (int a = 0; a < n; a++) {
DotLane tl[4];
for (int c = 0; c < 4; c++)
tl[c] = c < n ? DotReg(tregs[a * 4 + c]) : DotConst(0.0f);
for (int b = 0; b < n; b++) {
DotLane sl[4];
for (int c = 0; c < 4; c++)
sl[c] = c < n ? DotReg(sregs[b * 4 + c]) : DotConst(0.0f);
u8 sv, tv;
if (!BuildDotOperands(sl, tl, -1, &sv, &tv))
DISABLE;
// Built once per row of t.
for (int c = 0; c < 4; c++)
tl[c] = DotReg(tv + c);
ir.Write(IROp::Vec4VfpuDot, dregs[a * 4 + b], sv, tv);
}
}
return;
}
// A very common arrangment. Rearrange to something we can handle.
// A 1x1 matrix has no transposed form, the bit then picks a different register.
if (n > 1 && IsMatrixTransposed(vd)) {
@@ -1737,6 +1900,30 @@ namespace MIPSComp {
GetVectorRegs(tregs, sz, _VT);
GetVectorRegs(dregs, sz, _VD);
if (UseAccurateVdot()) {
// A dot per row, against t with the lanes past it zero, and lane ins 1 for the
// homogenous form. The results go to the D temp first, since d may overlap s or t.
const int tn = homogenous ? n - 1 : n;
DotLane tl[4];
for (int i = 0; i < 4; i++)
tl[i] = i < tn ? DotReg(tregs[i]) : DotConst(homogenous && i == ins ? 1.0f : 0.0f);
for (int i = 0; i < n; i++) {
DotLane sl[4];
for (int k = 0; k < 4; k++)
sl[k] = k < n ? DotReg(sregs[i * 4 + k]) : DotConst(0.0f);
u8 sv, tv;
if (!BuildDotOperands(sl, tl, IRVTEMP_PFX_D, &sv, &tv))
DISABLE;
// Built once, t is a Vec4 for the rest.
for (int k = 0; k < 4; k++)
tl[k] = DotReg(tv + k);
ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv);
}
for (int i = 0; i < n; i++)
ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i);
return;
}
// SIMD-optimized implementations - if sregs[0..3] is non-consecutive, it's transposed.
if (msz == M_4x4 && IsMatrixTransposedVec4(msz, sregs)) {
int s0 = IRVTEMP_0;
@@ -1871,6 +2058,18 @@ namespace MIPSComp {
GetVectorRegsPrefixT(tregs, sz, _VT);
GetVectorRegsPrefixD(dregs, V_Single, _VD);
if (UseAccurateVdot()) {
// The T prefix's swizzle is forced to yx: s0 * t1 + (-s1) * t0.
const DotLane sl[4] = { DotReg(sregs[0]), DotReg(sregs[1], true), DotConst(0.0f), DotConst(0.0f) };
const DotLane tl[4] = { DotReg(tregs[1]), DotReg(tregs[0]), DotConst(0.0f), DotConst(0.0f) };
u8 sv, tv;
if (!BuildDotOperands(sl, tl, -1, &sv, &tv))
DISABLE;
ir.Write(IROp::Vec4VfpuDot, dregs[0], sv, tv);
ApplyPrefixD(dregs, V_Single, _VD);
return;
}
ir.Write(IROp::FMul, IRVTEMP_0, sregs[1], tregs[0]);
ir.Write(IROp::FMul, dregs[0], sregs[0], tregs[1]);
ir.Write(IROp::FSub, dregs[0], dregs[0], IRVTEMP_0);
@@ -2086,6 +2285,49 @@ namespace MIPSComp {
GetVectorRegs(tregs, sz, _VT);
GetVectorRegs(dregs, sz, _VD);
if (UseAccurateVdot() && (sz == V_Triple || sz == V_Quad)) {
// The dots the interpreter does, against permuted and negated copies of t. The results
// go to the D temp first, since d may overlap s or t.
const DotLane zero = DotConst(0.0f);
auto tl = [&](int i, bool neg = false) { return i < n ? DotReg(tregs[i], neg) : zero; };
DotLane tDots[4][4];
int dots;
if (sz == V_Triple) {
const DotLane t0[4] = { zero, tl(2), tl(1, true), zero };
const DotLane t1[4] = { tl(2, true), zero, tl(0), zero };
const DotLane t2[4] = { tl(1), tl(0, true), zero, tl(2) };
memcpy(tDots[0], t0, sizeof(t0));
memcpy(tDots[1], t1, sizeof(t1));
memcpy(tDots[2], t2, sizeof(t2));
dots = 3;
} else {
const DotLane t0[4] = { tl(3), tl(2), tl(1, true), tl(0) };
const DotLane t1[4] = { tl(2, true), tl(3), tl(0), tl(1) };
const DotLane t2[4] = { tl(1), tl(0, true), tl(3), tl(2) };
const DotLane t3[4] = { tl(0, true), tl(1, true), tl(2, true), tl(3) };
memcpy(tDots[0], t0, sizeof(t0));
memcpy(tDots[1], t1, sizeof(t1));
memcpy(tDots[2], t2, sizeof(t2));
memcpy(tDots[3], t3, sizeof(t3));
dots = 4;
}
DotLane sl[4];
for (int i = 0; i < 4; i++)
sl[i] = i < n ? DotReg(sregs[i]) : zero;
for (int i = 0; i < dots; i++) {
u8 sv, tv;
if (!BuildDotOperands(sl, tDots[i], IRVTEMP_PFX_D, &sv, &tv))
DISABLE;
// Built once, s is a Vec4 for the rest.
for (int j = 0; j < 4; j++)
sl[j] = DotReg(sv + j);
ir.Write(IROp::Vec4VfpuDot, IRVTEMP_PFX_D + i, sv, tv);
}
for (int i = 0; i < dots; i++)
ir.Write(IROp::FMov, dregs[i], IRVTEMP_PFX_D + i);
return;
}
if (sz == V_Triple) {
u8 tempregs[4]{};
for (int i = 0; i < n; ++i) {
+15
View File
@@ -99,6 +99,13 @@ public:
opts = o;
}
// One lane of a Vec4VfpuDot operand: a reg, maybe negated, or a constant (reg < 0).
struct DotLane {
int reg;
float constant;
bool negate;
};
private:
void RestoreRoundingMode(bool force = false);
void ApplyRoundingMode(bool force = false);
@@ -134,6 +141,14 @@ private:
void GetVectorRegs(u8 regs[4], VectorSize N, int vectorReg);
void GetMatrixRegs(u8 regs[16], MatrixSize N, int matrixReg);
static DotLane DotReg(int reg, bool negate = false) { return DotLane{ reg, 0.0f, negate }; }
static DotLane DotConst(float f) { return DotLane{ -1, f, false }; }
// Puts both operands of a Vec4VfpuDot in Vec4 regs, building them in free temps where they
// aren't one already. avoid is a Vec4 temp that must not be used (or -1). Returns false if
// there aren't enough free temps, and then nothing was written.
bool BuildDotOperands(const DotLane s[4], const DotLane t[4], int avoid, u8 *sVec, u8 *tVec);
bool UseAccurateVdot() const;
// State
JitState js;
IRWriter ir;
+1
View File
@@ -158,6 +158,7 @@ static const IRMeta irMeta[] = {
{ IROp::Vec4Mul, "Vec4Mul", "VVV" },
{ IROp::Vec4Scale, "Vec4Scale", "VVF" },
{ IROp::Vec4Dot, "Vec4Dot", "FVV" },
{ IROp::Vec4VfpuDot, "Vec4VfpuDot", "FVV" },
{ IROp::Vec4Neg, "Vec4Neg", "VV" },
{ IROp::Vec4Abs, "Vec4Abs", "VV" },
+1
View File
@@ -187,6 +187,7 @@ enum class IROp : uint8_t {
Vec4Div,
Vec4Scale,
Vec4Dot,
Vec4VfpuDot, // The VFPU's own dot product, bit-exact (vfpu_dot).
Vec4Neg,
Vec4Abs,
+5
View File
@@ -171,6 +171,7 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) {
dispatch[(int)IROp::FCmpVfpuAggregate] = &&L_FCmpVfpuAggregate;
dispatch[(int)IROp::FCmovVfpuCC] = &&L_FCmovVfpuCC;
dispatch[(int)IROp::Vec4Dot] = &&L_Vec4Dot;
dispatch[(int)IROp::Vec4VfpuDot] = &&L_Vec4VfpuDot;
dispatch[(int)IROp::FSin] = &&L_FSin;
dispatch[(int)IROp::FCos] = &&L_FCos;
dispatch[(int)IROp::FRSqrt] = &&L_FRSqrt;
@@ -798,6 +799,10 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst) {
IR_NEXT;
}
IR_CASE(Vec4VfpuDot)
mips->f[inst->dest] = vfpu_dot(&mips->f[inst->src1], &mips->f[inst->src2]);
IR_NEXT;
IR_CASE(FSin)
mips->f[inst->dest] = vfpu_sin(mips->f[inst->src1]);
IR_NEXT;
+2
View File
@@ -421,9 +421,11 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
break;
case IROp::Vec4Dot:
case IROp::Vec4VfpuDot:
CompIR_VecHoriz(inst);
break;
case IROp::Vec2Unpack16To31:
case IROp::Vec2Unpack16To32:
case IROp::Vec4Unpack8To32:
+2
View File
@@ -847,6 +847,7 @@ bool PropagateConstants(const IRWriter &in, IRWriter &out, const IROptions &opts
case IROp::Vec4Mul:
case IROp::Vec4Div:
case IROp::Vec4Dot:
case IROp::Vec4VfpuDot:
case IROp::Vec4Scale:
case IROp::Vec4Shuffle:
case IROp::Vec4Blend:
@@ -2185,6 +2186,7 @@ bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts) {
break;
case IROp::Vec4Dot:
case IROp::Vec4VfpuDot:
if (overlapped(inst.dest, 1, inst.src1, 4, inst.src2, 4) && findAvailTempVec4()) {
out.Write(inst.op, temp, inst.src1, inst.src2, inst.constant);
if (usedLaterAsVec4(inst.dest & ~3)) {
+22 -22
View File
@@ -28,6 +28,7 @@
#include "Core/Core.h"
#include "Core/MemMap.h"
#include "Core/Reporting.h"
#include "Core/Config.h"
#include "Core/System.h"
#include "Core/MIPS/MIPS.h"
@@ -74,7 +75,13 @@
#define M_SQRT1_2 0.707106781186547524401f
#endif
static const bool USE_VFPU_DOT = false;
// The VFPU's own handling of NaN and denormal results in vadd, vsub, vmul and vdiv (see Int_VecDo3).
static const bool USE_VFPU_RESULT_FIXUP = false;
// The dot products (vdot, vmmul, vtfm and the rest) through vfpu_dot(), bit-exact with the PSP.
static inline bool AccurateVdot() {
return g_Config.bAccurateVdot || PSP_CoreParameter().compat.flags().ForceAccurateVdot;
}
union FloatBits {
float f[4];
@@ -507,8 +514,7 @@ namespace MIPSInt
ReadMatrix(mips, s, sz, vs);
ReadMatrix(mips, t, sz, vt);
// TODO: Always use the more accurate path in interpreter?
bool useAccurateDot = USE_VFPU_DOT || PSP_CoreParameter().compat.flags().MoreAccurateVMMUL;
bool useAccurateDot = AccurateVdot();
for (int a = 0; a < n; a++) {
for (int b = 0; b < n; b++) {
union { float f; uint32_t u; } sum = { 0.0f };
@@ -1208,7 +1214,7 @@ namespace MIPSInt
ReadVector(mips, t, sz, vt);
ApplySwizzleT(mips, t, V_Quad);
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
d.f = vfpu_dot(s, t);
if (my_isnan(d.f)) {
d.u = 0x7f800001;
@@ -1259,7 +1265,7 @@ namespace MIPSInt
ApplyPrefixST(mips, s, VFPURewritePrefix(mips, VFPU_CTRL_SPREFIX, sprefixRemove, sprefixAdd), V_Quad);
float sum = 0.0f;
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
sum = vfpu_dot(s, t);
} else {
for (int i = 0; i < 4; i++) {
@@ -1479,7 +1485,7 @@ namespace MIPSInt
u32 tprefixAdd = VFPU_SWIZZLE(1, 0, 0, 0);
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
s[1] = -s[1];
d[0] = vfpu_dot(s, t);
} else {
@@ -1507,7 +1513,7 @@ namespace MIPSInt
u32 tprefixAdd = VFPU_MAKE_CONSTANTS(VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE, VFPUConst::ONE);
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
d = vfpu_dot(s, t);
} else {
d = 0.0f;
@@ -1545,7 +1551,7 @@ namespace MIPSInt
tprefixAdd = 0;
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
d = vfpu_dot(s, t);
} else {
d = 0.0f;
@@ -1709,7 +1715,7 @@ namespace MIPSInt
ReadMatrix(mips, s, msz, vs);
ReadVector(mips, t, sz, vt);
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
float t2[4];
for (int i = 0; i < 4; i++) {
if (i < tn) {
@@ -1763,7 +1769,7 @@ namespace MIPSInt
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
// Really this is the operation all rows probably use (with constant wiring.)
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
d.f[ins] = vfpu_dot(&s[ins * 4], t);
if (my_isnan(d.f[ins])) {
@@ -2164,7 +2170,7 @@ namespace MIPSInt
case 8: d.f[i] = s[i] * t[i]; break; //vmul
}
if (USE_VFPU_DOT) {
if (USE_VFPU_RESULT_FIXUP) {
if (my_isnan(d.f[i])) {
d.u[i] = (d.u[i] & 0xff800001) | 1;
} else if ((d.u[i] & 0x7F800000) == 0) {
@@ -2207,7 +2213,7 @@ namespace MIPSInt
switch (sz) {
case V_Triple: // vcrsp.t
{
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
float t0[4] = { 0.0f, t[2], -t[1], 0.0f };
float t1[4] = { -t[2], 0.0f, t[0], 0.0f };
d[0] = vfpu_dot(s, t0);
@@ -2221,14 +2227,8 @@ namespace MIPSInt
tprefixAdd = VFPU_SWIZZLE(1, 0, 3, 2) | VFPU_NEGATE(0, 1, 0, 0);
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
ApplySwizzleS(mips, s, V_Quad);
if (USE_VFPU_DOT) {
// TODO: But flush any infs to 0? This seems sketchy.
for (int i = 0; i < 4; ++i) {
if (my_isinf(s[i]))
s[i] = 0.0f;
if (my_isinf(t[i]))
t[i] = 0.0f;
}
if (AccurateVdot()) {
// Infinities stay: inf * 0 gives the VFPU's NaN here (cpu/vfpu/nan_gen).
d[2] = vfpu_dot(s, t);
} else {
d[2] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3];
@@ -2238,7 +2238,7 @@ namespace MIPSInt
case V_Quad: // vqmul.q
{
if (USE_VFPU_DOT) {
if (AccurateVdot()) {
float t0[4] = { t[3], t[2], -t[1], t[0] };
float t1[4] = { -t[2], t[3], t[0], t[1] };
float t2[4] = { t[1], -t[0], t[3], t[2] };
@@ -2255,7 +2255,7 @@ namespace MIPSInt
tprefixAdd = VFPU_SWIZZLE(0, 1, 2, 3) | VFPU_NEGATE(1, 1, 1, 0);
ApplyPrefixST(mips, t, VFPURewritePrefix(mips, VFPU_CTRL_TPREFIX, tprefixRemove, tprefixAdd), V_Quad);
ApplySwizzleS(mips, s, sz);
if (USE_VFPU_DOT)
if (AccurateVdot())
d[3] = vfpu_dot(s, t);
else
d[3] = s[0] * t[0] + s[1] * t[1] + s[2] * t[2] + s[3] * t[3];
@@ -389,6 +389,11 @@ void LoongArch64JitBackend::CompIR_VecHoriz(IRInst inst) {
}
break;
case IROp::Vec4VfpuDot:
// Through the IR interpreter for now.
CompIR_Generic(inst);
break;
default:
INVALIDOP;
break;
+4
View File
@@ -248,6 +248,10 @@ public:
float sincostemp[2];
// The operands (s, then t) and result of the JITs' VFPU dot product routine (vfpu_dot()).
alignas(16) u32 vfpuDotArgs[8];
u32 vfpuDotResult;
static const u32 FCR0_VALUE = 0x00003351;
#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64)
+5
View File
@@ -312,6 +312,11 @@ void RiscVJitBackend::CompIR_VecHoriz(IRInst inst) {
}
break;
case IROp::Vec4VfpuDot:
// Through the IR interpreter for now.
CompIR_Generic(inst);
break;
default:
INVALIDOP;
break;
+4
View File
@@ -23,6 +23,7 @@
#include "Core/MemMap.h"
#include "Core/MIPS/x86/X64IRJit.h"
#include "Core/MIPS/x86/X64IRRegCache.h"
#include "Core/MIPS/x86/X64VfpuDot.h"
#include "Core/MIPS/JitCommon/JitCommon.h"
#include "Core/MIPS/JitCommon/JitState.h"
#include "Core/Core.h"
@@ -295,6 +296,9 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) {
}
JMP(quitLoop, true);
#if PPSSPP_ARCH(AMD64)
vfpuDot_ = GenerateX64VfpuDot(this, CTXREG);
#endif
// Leave this at the end, add more stuff above.
if (enableDisasm) {
+43
View File
@@ -274,6 +274,49 @@ void X64JitBackend::CompIR_VecHoriz(IRInst inst) {
break;
}
case IROp::Vec4VfpuDot:
{
#if PPSSPP_ARCH(AMD64)
if (!vfpuDot_ || Overlap(inst.dest, 1, inst.src1, 4) || Overlap(inst.dest, 1, inst.src2, 4)) {
CompIR_Generic(inst);
break;
}
// The operands go to the shared routine in xmm0 and xmm1, and the result comes back in edi.
// Whatever guest state is in the registers it clobbers gets flushed first: these come last in
// the allocation order, so usually that's nothing. Then again after the operands are moved,
// in case they were mapped into one of them.
auto flushClobbered = [&]() {
regs_.FlushHostRegs({ RCX, RDX, RSI, RDI }, { XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM14, XMM15 });
};
flushClobbered();
regs_.SpillLockFPR(inst.src1, inst.src2);
const X64Reg x = regs_.MapVec4(inst.src1);
const X64Reg y = regs_.MapVec4(inst.src2);
if (x == XMM1 && y == XMM0) {
MOVAPS(XMM2, R(XMM0));
MOVAPS(XMM0, R(XMM1));
MOVAPS(XMM1, R(XMM2));
} else if (y == XMM0) {
MOVAPS(XMM1, R(y));
MOVAPS(XMM0, R(x));
} else {
if (x != XMM0)
MOVAPS(XMM0, R(x));
if (y != XMM1)
MOVAPS(XMM1, R(y));
}
regs_.ReleaseSpillLockFPR(inst.src1, inst.src2);
flushClobbered();
CALL(vfpuDot_);
regs_.MapFPR(inst.dest, MIPSMap::NOINIT);
MOVD_xmm(regs_.FX(inst.dest), R(EDI));
#else
// 32-bit x86 has too few registers for the shared routine.
CompIR_Generic(inst);
#endif
break;
}
default:
INVALIDOP;
break;
+2
View File
@@ -141,6 +141,8 @@ private:
const u8 *saveStaticRegisters_ = nullptr;
const u8 *loadStaticRegisters_ = nullptr;
// See GenerateX64VfpuDot().
const u8 *vfpuDot_ = nullptr;
typedef struct { float f[4]; } Float4Constant;
struct Constants {
+7
View File
@@ -119,6 +119,13 @@ const int *X64IRRegCache::GetAllocationOrder(MIPSLoc type, MIPSMap flags, int &c
}
}
void X64IRRegCache::FlushHostRegs(std::initializer_list<X64Reg> gprs, std::initializer_list<X64Reg> xmms) {
for (X64Reg r : gprs)
FlushNativeReg(GPRToNativeReg(r));
for (X64Reg r : xmms)
FlushNativeReg(XMMToNativeReg(r));
}
void X64IRRegCache::FlushBeforeCall() {
// These registers are not preserved by function calls.
#if PPSSPP_ARCH(AMD64)
+4
View File
@@ -18,6 +18,8 @@
#pragma once
#include "ppsspp_config.h"
#include <initializer_list>
#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64)
#include "Common/x64Emitter.h"
@@ -95,6 +97,8 @@ public:
// Note: may change the high lanes of single-register XMMs.
void FlushAll(bool gprs = true, bool fprs = true) override;
void FlushBeforeCall();
// Flushes whatever guest state these registers hold, for code that clobbers them.
void FlushHostRegs(std::initializer_list<Gen::X64Reg> gprs, std::initializer_list<Gen::X64Reg> xmms);
Gen::X64Reg GetAndLockTempGPR();
Gen::X64Reg GetAndLockTempFPR();
+237
View File
@@ -0,0 +1,237 @@
// Copyright (c) 2012- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#pragma once
#include "ppsspp_config.h"
#if PPSSPP_ARCH(AMD64)
#include <cstddef>
#include <cstring>
#include <vector>
#include "Common/x64Emitter.h"
#include "Core/MIPS/MIPS.h"
#include "Core/MIPS/MIPSVFPUUtils.h"
namespace MIPSComp {
// Generates the VFPU dot product routine the x86-64 IR JIT CALLs: vfpu_dot() of xmm0 and xmm1, into
// edi. ctx points at mips->f[0], as CTXREG does. It's vfpu_dot_sse2(), SSE2 only, and leaves inf
// and NaN to vfpu_dot_reference(). It clobbers xmm0-xmm5, xmm14, xmm15, rax, rcx, rdx, rsi, rdi and
// the flags, and saves nothing: the call site flushes whatever of those holds guest state. The xmms
// are the last ones allocated, and rsi and rdi are never allocated outside Windows.
inline const u8 *GenerateX64VfpuDot(Gen::XEmitter *e, Gen::X64Reg ctx) {
using namespace Gen;
const int args = (int)(offsetof(MIPSState, vfpuDotArgs) - offsetof(MIPSState, f));
// The constants, RIP-relative.
auto constant4 = [&](u32 a, u32 b, u32 c, u32 d) {
const u8 *p = e->AlignCode16();
u8 *writable = e->GetWritableCodePtr();
e->ReserveCodeSpace(16);
const u32 values[4] = { a, b, c, d };
memcpy(writable, values, sizeof(values));
return p;
};
auto splat = [&](u32 v) { return constant4(v, v, v, v); };
const u8 *expBits = splat(0x7F800000);
const u8 *mantMask = splat(0x007FFFFF);
const u8 *implicitBit = splat(0x00800000);
const u8 *lowMask = constant4((1 << 21) - 1, 0, (1 << 21) - 1, 0);
const u8 *ones = splat(1);
const u8 *max28 = splat(28);
const u8 *scaleBase = splat(127 + 28);
const u8 *start = e->AlignCode16();
// xmm0 = x, xmm1 = y. Exponents: a zero or denormal operand makes the product zero (xmm4 =
// flush), and inf or NaN gets 0x7FF, above any real exponent sum, so one maximum finds the
// alignment and the specials (xmm5 = e). They all fit in 15 bits, so the 16-bit max does.
e->PXOR(XMM15, R(XMM15));
e->MOVDQA(XMM2, R(XMM0));
e->PAND(XMM2, M(expBits));
e->MOVDQA(XMM3, R(XMM1));
e->PAND(XMM3, M(expBits));
e->MOVDQA(XMM4, R(XMM2));
e->PCMPEQD(XMM4, R(XMM15));
e->MOVDQA(XMM5, R(XMM3));
e->PCMPEQD(XMM5, R(XMM15));
e->POR(XMM4, R(XMM5));
e->MOVDQA(XMM14, R(XMM2));
e->PCMPEQD(XMM14, M(expBits));
e->MOVDQA(XMM5, R(XMM3));
e->PCMPEQD(XMM5, M(expBits));
e->POR(XMM14, R(XMM5));
e->PSRLD(XMM14, 21);
e->MOVDQA(XMM5, R(XMM2));
e->PADDD(XMM5, R(XMM3));
e->PSRLD(XMM5, 23);
e->MOVDQA(XMM2, R(XMM4));
e->PANDN(XMM2, R(XMM5));
e->MOVDQA(XMM5, R(XMM2));
e->POR(XMM5, R(XMM14));
e->PSHUFD(XMM2, R(XMM5), 0x4E);
e->PMAXSW(XMM2, R(XMM5));
e->PSHUFD(XMM3, R(XMM2), 0xB1);
e->PMAXSW(XMM2, R(XMM3));
e->MOVD_xmm(R(EAX), XMM2);
e->CMP(32, R(EAX), Imm32(0x7FF));
FixupBranch special = e->J_CC(CC_AE, true);
// 24x24-bit products, even and odd lanes separately, kept to 2 extra bits with round-to-odd
// (xmm4 = p). xmm0 = the signs, while x and y are still around.
e->MOVDQA(XMM3, R(XMM0));
e->PAND(XMM3, M(mantMask));
e->POR(XMM3, M(implicitBit));
e->MOVDQA(XMM14, R(XMM1));
e->PAND(XMM14, M(mantMask));
e->POR(XMM14, M(implicitBit));
e->PXOR(XMM0, R(XMM1));
e->PSRAD(XMM0, 31);
e->MOVDQA(XMM15, R(XMM3));
e->PMULUDQ(XMM15, R(XMM14)); // even products
e->PSRLQ(XMM3, 32);
e->PSRLQ(XMM14, 32);
e->PMULUDQ(XMM3, R(XMM14)); // odd products
e->MOVDQA(XMM14, R(XMM15));
e->PSRLQ(XMM14, 21);
e->PAND(XMM15, M(lowMask));
e->MOVDQA(XMM1, R(XMM3));
e->PSRLQ(XMM1, 21);
e->PSLLQ(XMM1, 32);
e->POR(XMM14, R(XMM1)); // kept
e->PAND(XMM3, M(lowMask));
e->PSLLQ(XMM3, 32);
e->POR(XMM3, R(XMM15)); // dropped
e->PXOR(XMM15, R(XMM15));
e->PCMPEQD(XMM3, R(XMM15));
e->PANDN(XMM3, M(ones)); // sticky
e->POR(XMM3, R(XMM14));
e->PANDN(XMM4, R(XMM3));
// Align to the largest exponent by truncation. SSE2 has no per-lane shift, so p >> d is done
// as (p * 2^(28 - d)) >> 28, with the power of two built as float bits and converted by
// truncation (always a normal float, so DAZ doesn't matter).
e->PSHUFD(XMM2, R(XMM2), 0x00);
e->PSUBD(XMM2, R(XMM5));
e->PMINSW(XMM2, M(max28));
e->MOVDQA(XMM3, M(scaleBase));
e->PSUBD(XMM3, R(XMM2));
e->PSLLD(XMM3, 23);
e->CVTTPS2DQ(XMM3, R(XMM3));
e->MOVDQA(XMM5, R(XMM4));
e->PMULUDQ(XMM5, R(XMM3));
e->PSRLQ(XMM5, 28);
e->PSRLQ(XMM4, 32);
e->PSRLQ(XMM3, 32);
e->PMULUDQ(XMM4, R(XMM3));
e->PSRLQ(XMM4, 28);
e->PSLLQ(XMM4, 32);
e->POR(XMM4, R(XMM5));
// Apply the signs and sum exactly.
e->PXOR(XMM4, R(XMM0));
e->PSUBD(XMM4, R(XMM0));
e->PSHUFD(XMM0, R(XMM4), 0x4E);
e->PADDD(XMM4, R(XMM0));
e->PSHUFD(XMM0, R(XMM4), 0xB1);
e->PADDD(XMM4, R(XMM0));
// vfpu_dot_finish(): drop the extra bits, round to 24 bits (nearest even), build the float.
// esi = the sum, then its sign. edi = the largest exponent sum, then the result. edx = the
// significand m. ecx = 62 - its top bit, the shift that puts that at bit 62.
e->MOVD_xmm(R(ESI), XMM4);
e->MOV(32, R(EDI), R(EAX));
e->MOV(32, R(EAX), R(ESI));
e->CDQ();
e->XOR(32, R(EAX), R(EDX));
e->SUB(32, R(EAX), R(EDX));
e->AND(32, R(ESI), Imm32(0x80000000));
e->SHR(32, R(EAX), Imm8(2));
e->MOV(32, R(EDX), R(EAX));
e->MOV(32, R(ECX), R(EAX));
e->OR(32, R(ECX), Imm32(1));
e->BSR(32, ECX, R(ECX));
e->NEG(32, R(ECX));
e->ADD(32, R(ECX), Imm32(62));
e->SHL(64, R(RAX), R(CL));
e->SUB(32, R(EDI), R(ECX));
e->SUB(32, R(EDI), Imm32(89));
e->MOV(64, R(RCX), R(RAX));
e->SHR(64, R(RCX), Imm8(39));
e->AND(32, R(ECX), Imm32(1));
e->ADD(64, R(RAX), R(RCX));
e->MOV(64, R(RCX), Imm64((1ULL << 38) - 1));
e->ADD(64, R(RAX), R(RCX));
e->SHR(64, R(RAX), Imm8(39));
e->MOVSX(64, 32, RDI, R(EDI));
e->SHL(64, R(RDI), Imm8(23));
e->ADD(64, R(RDI), R(RAX));
e->MOV(32, R(EAX), Imm32(0x7F800000));
e->CMP(64, R(RDI), R(RAX));
e->CMOVcc(64, RDI, R(RAX), CC_GE);
e->XOR(32, R(ECX), R(ECX));
e->CMP(64, R(RDI), Imm32(0x00800000));
e->CMOVcc(32, EDI, R(ECX), CC_L);
e->TEST(32, R(EDX), R(EDX));
e->CMOVcc(32, EDI, R(ECX), CC_Z);
e->OR(32, R(EDI), R(ESI));
e->RET();
// Inf or NaN: the C version, with x and y still in xmm0 and xmm1. The call can clobber registers
// that the call site didn't flush, so those are saved, around an aligned stack.
e->SetJumpTarget(special);
e->MOVUPS(MDisp(ctx, args), XMM0);
e->MOVUPS(MDisp(ctx, args + 16), XMM1);
e->PUSH(R8);
e->PUSH(R9);
e->PUSH(R10);
e->PUSH(R11);
e->PUSH(RBP);
e->MOV(64, R(RBP), R(RSP));
e->AND(64, R(RSP), Imm32(-16));
// xmm6-xmm13, and the shadow space Windows wants.
e->SUB(64, R(RSP), Imm32(8 * 16 + 32));
for (int i = 6; i < 14; i++)
e->MOVUPS(MDisp(RSP, 32 + (i - 6) * 16), (X64Reg)(XMM0 + i));
#ifdef _WIN32
e->LEA(64, RCX, MDisp(ctx, args));
e->LEA(64, RDX, MDisp(ctx, args + 16));
#else
e->LEA(64, RDI, MDisp(ctx, args));
e->LEA(64, RSI, MDisp(ctx, args + 16));
#endif
e->MOV(64, R(RAX), ImmPtr((const void *)&vfpu_dot_reference));
e->CALLptr(R(RAX));
e->MOVD_xmm(R(EDI), XMM0);
for (int i = 6; i < 14; i++)
e->MOVUPS((X64Reg)(XMM0 + i), MDisp(RSP, 32 + (i - 6) * 16));
e->MOV(64, R(RSP), R(RBP));
e->POP(RBP);
e->POP(R11);
e->POP(R10);
e->POP(R9);
e->POP(R8);
e->RET();
return start;
}
} // namespace MIPSComp
#endif
+65 -2
View File
@@ -1357,6 +1357,69 @@ static void SIMDVectorIndexedElement(uint32_t w, Instruction *instr) {
Out(instr, "%s %c%d, %c%d, %c%d.%s[%d]", opname, r, Rd, r, Rn, r, Rm, GetArrangement(Q, size & 1), index);
}
// Reductions across the lanes of a vector into a scalar.
static void SIMDAcrossLanes(uint32_t w, Instruction *instr) {
int Rd = w & 0x1f;
int Rn = (w >> 5) & 0x1f;
int opcode = (w >> 12) & 0x1F;
int size = (w >> 22) & 3;
bool Q = GetQ(w);
bool U = GetU(w);
char r = Q ? 'q' : 'd';
if (U && (opcode == 0x0C || opcode == 0x0F)) {
// fmaxnmv, fmaxv, fminnmv, fminv: single precision only.
const char *name = opcode == 0x0C ? ((size & 2) ? "fminnmv" : "fmaxnmv") : ((size & 2) ? "fminv" : "fmaxv");
Out(instr, "%s s%d, %c%d.4s", name, Rd, r, Rn);
return;
}
const char *name = nullptr;
bool longResult = false;
switch (opcode) {
case 0x03: name = U ? "uaddlv" : "saddlv"; longResult = true; break;
case 0x0A: name = U ? "umaxv" : "smaxv"; break;
case 0x1A: name = U ? "uminv" : "sminv"; break;
case 0x1B: name = U ? nullptr : "addv"; break;
}
if (!name || size == 3) {
Out(instr, "(asimd across lanes %08x)", w);
return;
}
static const char scalarRegs[] = "bhsd";
Out(instr, "%s.%d %c%d, %c%d", name, 8 << size, scalarRegs[size + (longResult ? 1 : 0)], Rd, r, Rn);
}
// Long (L), wide (W) and narrowing (HN) ops on lanes of different sizes.
static void SIMDThreeDifferent(uint32_t w, Instruction *instr) {
int Rd = w & 0x1f;
int Rn = (w >> 5) & 0x1f;
int Rm = (w >> 16) & 0x1f;
int opcode = (w >> 12) & 0xF;
int size = (w >> 22) & 3;
bool Q = GetQ(w);
bool U = GetU(w);
static const char * const names[2][16] = {
{ "saddl", "saddw", "ssubl", "ssubw", "addhn", "sabal", "subhn", "sabdl", "smlal", "sqdmlal", "smlsl", "sqdmlsl", "smull", "sqdmull", "pmull", nullptr },
{ "uaddl", "uaddw", "usubl", "usubw", "raddhn", "uabal", "rsubhn", "uabdl", "umlal", nullptr, "umlsl", nullptr, "umull", nullptr, nullptr, nullptr },
};
const char *name = names[U][opcode];
if (!name || size == 3) {
Out(instr, "(asimd three-different %08x)", w);
return;
}
const char *two = Q ? "2" : "";
const char half = Q ? 'q' : 'd';
const int narrow = 8 << size, wide = 16 << size;
if (opcode == 1 || opcode == 3) {
// Wide: Rd and Rn are the wide lanes.
Out(instr, "%s%s.%d.%d q%d, q%d, %c%d", name, two, wide, narrow, Rd, Rn, half, Rm);
} else if (opcode == 4 || opcode == 6) {
// Narrowing: Rd gets the high halves.
Out(instr, "%s%s.%d.%d %c%d, q%d, q%d", name, two, narrow, wide, half, Rd, Rn, Rm);
} else {
Out(instr, "%s%s.%d.%d q%d, %c%d, %c%d", name, two, wide, narrow, Rd, half, Rn, half, Rm);
}
}
// (w >> 25) & 0xF == 7: AdvSIMD vector instructions.
static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) {
int Rd = w & 0x1f;
@@ -1372,9 +1435,9 @@ static void FPandASIMD1(uint32_t w, uint64_t addr, Instruction *instr) {
} else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 0) {
SIMDTwoRegMisc(w, instr);
} else if (((w >> 10) & 3) == 2 && ((w >> 17) & 0xF) == 8) {
Out(instr, "(asimd across lanes %08x)", w);
SIMDAcrossLanes(w, instr);
} else {
Out(instr, "(asimd three-different %08x)", w);
SIMDThreeDifferent(w, instr);
}
} else if ((w & 0x9FE08400) == 0x0E000400) {
SIMDCopy(w, instr);
+3
View File
@@ -278,8 +278,10 @@
<ClInclude Include="..\..\Core\MemMapHelpers.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64IRJit.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64IRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64VfpuDot.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmIRJit.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmIRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmVfpuDot.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRFrontend.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRInst.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRInterpreter.h" />
@@ -307,6 +309,7 @@
<ClInclude Include="..\..\Core\MIPS\MIPSTracer.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64IRJit.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64IRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64VfpuDot.h" />
<ClInclude Include="..\..\Core\Opcode.h" />
<ClInclude Include="..\..\Core\PSPLoaders.h" />
<ClInclude Include="..\..\Core\Reporting.h" />
+3
View File
@@ -597,8 +597,10 @@
<ClInclude Include="..\..\Core\MemMapHelpers.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64IRJit.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64IRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\ARM64\Arm64VfpuDot.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmIRJit.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmIRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\ARM\ArmVfpuDot.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRFrontend.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRInst.h" />
<ClInclude Include="..\..\Core\MIPS\IR\IRInterpreter.h" />
@@ -626,6 +628,7 @@
<ClInclude Include="..\..\Core\MIPS\MIPSTracer.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64IRJit.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64IRRegCache.h" />
<ClInclude Include="..\..\Core\MIPS\x86\X64VfpuDot.h" />
<ClInclude Include="..\..\Core\Opcode.h" />
<ClInclude Include="..\..\Core\PSPLoaders.h" />
<ClInclude Include="..\..\Core\Reporting.h" />
+2 -3
View File
@@ -1100,9 +1100,8 @@ NPJH50559 = true
NPEH00030 = true
ULKS46191 = true
[MoreAccurateVMMUL]
# Fixes leg shaking in Tekken 6. The potential for slowdown in other games is large enough
# that we will not generally apply this accurate mode where not needed.
[ForceAccurateVdot]
# The VFPU's own dot products, even with AccurateVdot turned off. Fixes leg shaking in Tekken 6.
ULUS10466 = true
ULES01376 = true
ULJS00224 = true
+1
View File
@@ -33,6 +33,7 @@ Environment variables:
| `PPSSPP_CPUFUZZ_DUMP` | write every core's end state of every case to a file |
| `PPSSPP_CPUFUZZ_STRICT` | fail on any difference, not only on a crash |
| `PPSSPP_CPUFUZZ_OPS` | comma-separated instruction name prefixes to generate only; `dot` is the dot products and the prefix ops |
| `PPSSPP_CPUFUZZ_ACCURATE_VDOT` | `0` to run without the accurate dot products, which are on by default |
For quick rounds while working on some instructions, fuzz just those, on every core of the machine:
`PPSSPP_CPUFUZZ_OPS=dot Tools/cpu_fuzz_parallel.py build/PPSSPPUnitTest --count 2000` splits the seeds
+5
View File
@@ -398,6 +398,7 @@ bool TestJitReplacementHooks() {
// PPSSPP_CPUFUZZ_SKIP cores not to run, by digit: 1 IR interpreter, 2 JIT, 3 JIT IR.
// PPSSPP_CPUFUZZ_REPORTS how many differences to minimize and print (default 20).
// PPSSPP_CPUFUZZ_STRICT fail the test on any difference, not only on a crash.
// PPSSPP_CPUFUZZ_ACCURATE_VDOT 0 to run without AccurateVdot (the VFPU's own dot products), which is on by default.
// PPSSPP_CPUFUZZ_OPS comma-separated instruction name prefixes to generate only, for fast
// runs on the instructions being worked on. "dot" is the dot products
// and the prefixes. Without PPSSPP_CPUFUZZ_CLASSES, the class is "v".
@@ -899,6 +900,9 @@ bool TestCPUEquivalence() {
// terminating syscall doesn't.
const bool oldSkipDeadbeef = g_Config.bSkipDeadbeefFilling;
g_Config.bSkipDeadbeefFilling = true;
const bool oldAccurateVdot = g_Config.bAccurateVdot;
const char *accurateVdot = getenv("PPSSPP_CPUFUZZ_ACCURATE_VDOT");
g_Config.bAccurateVdot = !accurateVdot || strcmp(accurateVdot, "0") != 0;
const char *countStr = getenv("PPSSPP_CPUFUZZ_COUNT");
const char *seedStr = getenv("PPSSPP_CPUFUZZ_SEED");
@@ -1049,6 +1053,7 @@ bool TestCPUEquivalence() {
mipsr4k.UpdateCore(CPUCore::INTERPRETER);
DestroyJitHarness();
g_Config.bSkipDeadbeefFilling = oldSkipDeadbeef;
g_Config.bAccurateVdot = oldAccurateVdot;
return success;
}
+18
View File
@@ -698,6 +698,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float
// Vector logical
DIS("and q1, q2, q3", fp.AND(Q1, Q2, Q3));
DIS("and d1, d2, d3", fp.AND(D1, D2, D3));
DIS("bic q1, q2, q3", fp.BIC(Q1, Q2, Q3));
DIS("bic d1, d2, d3", fp.BIC(D1, D2, D3));
DIS("eor q0, q1, q2", fp.EOR(Q0, Q1, Q2));
DIS("eor d0, d1, d2", fp.EOR(D0, D1, D2));
DIS("bsl q1, q2, q3", fp.BSL(Q1, Q2, Q3));
@@ -772,6 +774,20 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float
DIS("umin.8 q0, q3, q4", fp.UMIN(8, Q0, Q3, Q4));
DIS("umax.16 q0, q3, q4", fp.UMAX(16, Q0, Q3, Q4));
DIS("umax.32 d0, d3, d4", fp.UMAX(32, D0, D3, D4));
DIS("add.32 q0, q3, q4", fp.ADD(32, Q0, Q3, Q4));
DIS("add.16 d0, d3, d4", fp.ADD(16, D0, D3, D4));
DIS("sub.32 q0, q3, q4", fp.SUB(32, Q0, Q3, Q4));
DIS("sub.8 d0, d3, d4", fp.SUB(8, D0, D3, D4));
DIS("umaxp.32 q0, q3, q4", fp.UMAXP(32, Q0, Q3, Q4));
DIS("umaxp.8 d0, d3, d4", fp.UMAXP(8, D0, D3, D4));
DIS("ushl.32 q1, q2, q3", fp.USHL(32, Q1, Q2, Q3));
DIS("ushl.16 d1, d2, d3", fp.USHL(16, D1, D2, D3));
DIS("umull.64.32 q1, d2, d3", fp.UMULL(32, Q1, D2, D3));
DIS("umull.16.8 q4, d5, d6", fp.UMULL(8, Q4, D5, D6));
DIS("umull2.64.32 q1, q2, q3", fp.UMULL2(32, Q1, Q2, Q3));
DIS("umull2.32.16 q7, q8, q9", fp.UMULL2(16, Q7, Q8, Q9));
DIS("addv.32 s1, q2", fp.ADDV(32, S1, Q2));
DIS("addv.16 h3, d4", fp.ADDV(16, S3, D4));
DIS("smin.8 d0, d3, d4", fp.SMIN(8, D0, D3, D4));
DIS("smin.16 q0, q3, q4", fp.SMIN(16, Q0, Q3, Q4));
DIS("smax.16 d0, d3, d4", fp.SMAX(16, D0, D3, D4));
@@ -967,6 +983,8 @@ static void TestFloatInstructions(DisasmChecker &c, ARM64XEmitter &e, ARM64Float
DIS("shll2.32.16 q1, q2, #16", fp.SHLL2(16, Q1, Q2));
DIS("shll2.16.8 q1, q2, #8", fp.SHLL2(8, Q1, Q2));
DIS("shrn.32.64 d0, q3, #8", fp.SHRN(32, D0, Q3, 8));
DIS("usra.32 q1, q2, #21", fp.USRA(32, Q1, Q2, 21));
DIS("usra.16 d1, d2, #3", fp.USRA(16, D1, D2, 3));
DIS("shrn.8.16 d0, q3, #4", fp.SHRN(8, D0, Q3, 4));
DIS("shrn2.16.32 q0, q3, #16", fp.SHRN2(16, Q0, Q3, 16));
DIS("shrn2.32.64 q0, q3, #1", fp.SHRN2(32, Q0, Q3, 1));