diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 31726d525c..61bf4de6b0 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -41,19 +41,6 @@ list(APPEND CoreExtra ) list(APPEND CoreExtra - MIPS/ARM64/Arm64Asm.cpp - MIPS/ARM64/Arm64CompALU.cpp - MIPS/ARM64/Arm64CompBranch.cpp - MIPS/ARM64/Arm64CompFPU.cpp - MIPS/ARM64/Arm64CompLoadStore.cpp - MIPS/ARM64/Arm64CompVFPU.cpp - MIPS/ARM64/Arm64CompReplace.cpp - MIPS/ARM64/Arm64Jit.cpp - MIPS/ARM64/Arm64Jit.h - MIPS/ARM64/Arm64RegCache.cpp - MIPS/ARM64/Arm64RegCache.h - MIPS/ARM64/Arm64RegCacheFPU.cpp - MIPS/ARM64/Arm64RegCacheFPU.h MIPS/ARM64/Arm64IRAsm.cpp MIPS/ARM64/Arm64IRCompALU.cpp MIPS/ARM64/Arm64IRCompBranch.cpp @@ -69,21 +56,6 @@ list(APPEND CoreExtra ) list(APPEND CoreExtra - MIPS/x86/Asm.cpp - MIPS/x86/CompALU.cpp - MIPS/x86/CompBranch.cpp - MIPS/x86/CompFPU.cpp - MIPS/x86/CompLoadStore.cpp - MIPS/x86/CompVFPU.cpp - MIPS/x86/CompReplace.cpp - MIPS/x86/Jit.cpp - MIPS/x86/Jit.h - MIPS/x86/JitSafeMem.cpp - MIPS/x86/JitSafeMem.h - MIPS/x86/RegCache.cpp - MIPS/x86/RegCache.h - MIPS/x86/RegCacheFPU.cpp - MIPS/x86/RegCacheFPU.h MIPS/x86/X64IRAsm.cpp MIPS/x86/X64IRCompALU.cpp MIPS/x86/X64IRCompBranch.cpp diff --git a/Core/Config.cpp b/Core/Config.cpp index 875d23eda3..8d461a4212 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -1587,6 +1587,12 @@ bool Config::Save(const char *saveReason) { // A lot more cleanup tasks should be moved into here, and some of these are severely outdated. void Config::PostLoadCleanup() { + // JIT_IR is the same core as JIT now that the legacy x86 and ARM64 JITs are gone, and the + // settings no longer offer it. + if (iCpuCore == (int)CPUCore::JIT_IR) { + iCpuCore = (int)CPUCore::JIT; + } + // Override ppsspp.ini JIT value to prevent crashing jitForcedOff = DefaultCpuCore() != (int)CPUCore::JIT && (g_Config.iCpuCore == (int)CPUCore::JIT || g_Config.iCpuCore == (int)CPUCore::JIT_IR); if (jitForcedOff) { diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 58f6c8b408..192b4c0ef7 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -685,86 +685,6 @@ - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - true true @@ -861,50 +781,6 @@ - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - - - true - true - @@ -1204,30 +1080,6 @@ - - true - true - true - false - true - false - - - true - true - true - false - true - false - - - true - true - true - false - true - false - true true @@ -1261,22 +1113,6 @@ - - true - true - - - true - true - - - true - true - - - true - true - diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 4e8a93738b..57782a769b 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -135,30 +135,6 @@ MIPS - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - - - MIPS\x86 - MIPS\JitCommon @@ -471,9 +447,6 @@ Ext - - MIPS\x86 - MIPS\ARM @@ -571,18 +544,12 @@ HLE - - MIPS\x86 - MIPS\ARM HW - - MIPS\x86 - HLE @@ -625,36 +592,6 @@ HLE\Libraries - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - Util @@ -1485,12 +1422,6 @@ MIPS - - MIPS\x86 - - - MIPS\x86 - MIPS\JitCommon @@ -1797,9 +1728,6 @@ MIPS\ARM - - MIPS\x86 - MIPS\ARM @@ -1908,9 +1836,6 @@ HW - - MIPS\x86 - HLE @@ -1968,15 +1893,6 @@ Core - - MIPS\ARM64 - - - MIPS\ARM64 - - - MIPS\ARM64 - Util diff --git a/Core/HLE/ReplaceTables.cpp b/Core/HLE/ReplaceTables.cpp index f3a84fb1ae..122754c20d 100644 --- a/Core/HLE/ReplaceTables.cpp +++ b/Core/HLE/ReplaceTables.cpp @@ -1722,7 +1722,7 @@ static const ReplacementTableEntry entries[] = { { "strcmp", &Replace_strcmp, 0, REPFLAG_DISABLED }, { "strncmp", &Replace_strncmp, 0, REPFLAG_DISABLED }, { "fabsf", &Replace_fabsf, JITFUNC(Replace_fabsf), REPFLAG_DISABLED }, - { "dl_write_matrix", &Replace_dl_write_matrix, 0, REPFLAG_DISABLED }, // &MIPSComp::Jit::Replace_dl_write_matrix, REPFLAG_DISABLED }, + { "dl_write_matrix", &Replace_dl_write_matrix, 0, REPFLAG_DISABLED }, { "dl_write_matrix_2", &Replace_dl_write_matrix, 0, REPFLAG_DISABLED }, { "gta_dl_write_matrix", &Replace_gta_dl_write_matrix, 0, REPFLAG_DISABLED }, // dl_write_matrix_3 doesn't take the dl as a parameter, it accesses a global instead. Need to extract the address of the global from the code when replacing... diff --git a/Core/MIPS/ARM64/Arm64Asm.cpp b/Core/MIPS/ARM64/Arm64Asm.cpp deleted file mode 100644 index 12bc8948d1..0000000000 --- a/Core/MIPS/ARM64/Arm64Asm.cpp +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright (c) 2015- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" - -#if PPSSPP_ARCH(ARM64) - -#include "Common/Log.h" -#include "Common/MemoryUtil.h" -#include "Common/CPUDetect.h" -#include "Common/Arm64Emitter.h" -#include "Core/MemMap.h" -#include "Core/MIPS/MIPS.h" -#include "Core/Core.h" -#include "Core/CoreTiming.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/JitCommon/JitCommon.h" - -using namespace Arm64Gen; - -//static int temp32; // unused? - -static const bool enableDebug = false; -static const bool enableDisasm = false; - -//static bool enableStatistics = false; //unused? - - -// ARM64 calling conventions -// Standard: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf -// Apple: https://developer.apple.com/library/ios/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARM64FunctionCallingConventions.html - -// Summary: -// =========== -// SP ("x31") is not a GPR so irrelevant. -// x0-x7: 8 parameter/result registers -// x8: "Indirect result location register" (points to struct return values? I think we can map this) -// x9-x15: 7 temporary registers (no need to save) -// x16: temporary register/procedure call scratch register 1 -// x17: temporary register/procedure call scratch register 2 -// x18: unavailable (reserved for use by the OS or linker or whatever - iOS, for example, uses it) -// x19-x28: 10 callee-saved registers -// x29: the frame pointer register -// x30: link register for procedure calls - -// So: Scratch registers: x16, x17 -// Mappable registers in priority order: -// x19, x20, x21, x22, x23, (x24, x25, x26, x27, x28), x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x0, x1, -// That's a whole lot of registers so we might be able to statically allocate a bunch of common MIPS registers. -// We should put statically allocated registers in the 7 callee-save regs that are left over after the system regs (x19-x25), so we don't have to bother with -// saving them when we call out of the JIT. We will perform regular dynamic register allocation in the rest (x0-x15) - -// STATIC ALLOCATION ARM64 (these are all callee-save registers): -// x23 : Down counter -// x24 : PC save on JR with non-nice delay slot (to be eliminated later?) -// x25 : MSR/MRS temporary (to be eliminated later) -// x26 : JIT base reg -// x27 : MIPS state (Could eliminate by placing the MIPS state right at the memory base) -// x28 : Memory base pointer. - -extern volatile CoreState coreState; - -void ShowPC(u32 downcount, void *membase, void *jitbase) { - static int count = 0; - if (currentMIPS) { - ERROR_LOG(Log::JIT, "ShowPC : %08x Downcount : %08x %d %p %p", currentMIPS->pc, downcount, count, membase, jitbase); - } else { - ERROR_LOG(Log::JIT, "Universe corrupt?"); - } - //if (count > 2000) - // exit(0); - count++; -} - -void DisassembleArm(const u8 *data, int size); - -// PLAN: no more block numbers - crazy opcodes just contain offset within -// dynarec buffer -// At this offset - 4, there is an int specifying the block number. - -namespace MIPSComp { - -using namespace Arm64JitConstants; - -void Arm64Jit::GenerateFixedCode(const JitOptions &jo) { - BeginWrite(GetMemoryProtectPageSize()); - const u8 *start = AlignCodePage(); - - if (jo.useStaticAlloc) { - saveStaticRegisters = AlignCode16(); - STR(INDEX_UNSIGNED, DOWNCOUNTREG, CTXREG, offsetof(MIPSState, downcount)); - gpr.EmitSaveStaticRegisters(); - RET(); - - loadStaticRegisters = AlignCode16(); - gpr.EmitLoadStaticRegisters(); - LDR(INDEX_UNSIGNED, DOWNCOUNTREG, CTXREG, offsetof(MIPSState, downcount)); - RET(); - - start = saveStaticRegisters; - } else { - saveStaticRegisters = nullptr; - loadStaticRegisters = nullptr; - } - - restoreRoundingMode = AlignCode16(); { - MRS(SCRATCH2_64, FIELD_FPCR); - // We are not in flush-to-zero mode outside the JIT, so let's turn it off. - uint32_t mask = ~(4 << 22); - // Assume we're always in round-to-nearest mode beforehand. - mask &= ~(3 << 22); - ANDI2R(SCRATCH2, SCRATCH2, mask); - _MSR(FIELD_FPCR, SCRATCH2_64); - RET(); - } - - applyRoundingMode = AlignCode16(); { - LDR(INDEX_UNSIGNED, SCRATCH2, CTXREG, offsetof(MIPSState, fcr31)); - TSTI2R(SCRATCH2, 1 << 24); - ANDI2R(SCRATCH2, SCRATCH2, 3); - FixupBranch skip1 = B(CC_EQ); - ADDI2R(SCRATCH2, SCRATCH2, 4); - SetJumpTarget(skip1); - - // We can skip if the rounding mode is nearest (0) and flush is not set. - // (as restoreRoundingMode cleared it out anyway) - CMPI2R(SCRATCH2, 0); - FixupBranch skip = B(CC_EQ); - - // MIPS Rounding Mode: ARM Rounding Mode - // 0: Round nearest 0 - // 1: Round to zero 3 - // 2: Round up (ceil) 1 - // 3: Round down (floor) 2 - ANDI2R(SCRATCH1, SCRATCH2, 3); - CMPI2R(SCRATCH1, 1); - - FixupBranch skipadd = B(CC_NEQ); - ADDI2R(SCRATCH2, SCRATCH2, 2); - SetJumpTarget(skipadd); - FixupBranch skipsub = B(CC_LE); - SUBI2R(SCRATCH2, SCRATCH2, 1); - SetJumpTarget(skipsub); - - // Actually change the system FPCR register - MRS(SCRATCH1_64, FIELD_FPCR); - // Clear both flush-to-zero and rounding before re-setting them. - ANDI2R(SCRATCH1, SCRATCH1, ~((4 | 3) << 22)); - ORR(SCRATCH1, SCRATCH1, SCRATCH2, ArithOption(SCRATCH2, ST_LSL, 22)); - _MSR(FIELD_FPCR, SCRATCH1_64); - - SetJumpTarget(skip); - RET(); - } - - updateRoundingMode = AlignCode16(); { - LDR(INDEX_UNSIGNED, SCRATCH2, CTXREG, offsetof(MIPSState, fcr31)); - - // Set SCRATCH2 to FZ:RM (FZ is bit 24, and RM are lowest 2 bits.) - TSTI2R(SCRATCH2, 1 << 24); - ANDI2R(SCRATCH2, SCRATCH2, 3); - FixupBranch skip = B(CC_EQ); - ADDI2R(SCRATCH2, SCRATCH2, 4); - SetJumpTarget(skip); - - // Let's update js.currentRoundingFunc with the right convertS0ToSCRATCH1 func. - MOVP2R(SCRATCH1_64, convertS0ToSCRATCH1); - LSL(SCRATCH2, SCRATCH2, 3); - LDR(SCRATCH2_64, SCRATCH1_64, SCRATCH2); - MOVP2R(SCRATCH1_64, &js.currentRoundingFunc); - STR(INDEX_UNSIGNED, SCRATCH2_64, SCRATCH1_64, 0); - RET(); - } - - enterDispatcher = AlignCode16(); - - uint32_t regs_to_save = Arm64Gen::ALL_CALLEE_SAVED; - uint32_t regs_to_save_fp = Arm64Gen::ALL_CALLEE_SAVED_FP; - fp.ABI_PushRegisters(regs_to_save, regs_to_save_fp); - - // Fixed registers, these are always kept when in Jit context. - MOVP2R(MEMBASEREG, Memory::base); - MOVP2R(CTXREG, mips_); - MOVP2R(JITBASEREG, GetBasePtr()); - - LoadStaticRegisters(); - MovFromPC(SCRATCH1); - outerLoopPCInSCRATCH1 = GetCodePtr(); - MovToPC(SCRATCH1); - outerLoop = GetCodePtr(); - SaveStaticRegisters(); // Advance can change the downcount, so must save/restore - RestoreRoundingMode(true); - QuickCallFunctionR(SCRATCH1_64, &CoreTiming::Advance, CTXREG); - ApplyRoundingMode(true); - LoadStaticRegisters(); - FixupBranch skipToCoreStateCheck = B(); //skip the downcount check - - dispatcherCheckCoreState = GetCodePtr(); - - // The result of slice decrementation should be in flags if somebody jumped here - // IMPORTANT - We jump on negative, not carry!!! - FixupBranch bailCoreState = B(CC_MI); - - SetJumpTarget(skipToCoreStateCheck); - - MOVP2R(SCRATCH1_64, &coreState); - LDR(INDEX_UNSIGNED, SCRATCH1, SCRATCH1_64, 0); - CMP(SCRATCH1, 0); - FixupBranch badCoreState = B(CC_NEQ); - FixupBranch skipToRealDispatch2 = B(); //skip the sync and compare first time - - dispatcherPCInSCRATCH1 = GetCodePtr(); - // TODO: Do we always need to write PC to RAM here? - MovToPC(SCRATCH1); - - // At this point : flags = EQ. Fine for the next check, no need to jump over it. - dispatcher = GetCodePtr(); - - // The result of slice decrementation should be in flags if somebody jumped here - // IMPORTANT - We jump on negative, not carry!!! - FixupBranch bail = B(CC_MI); - - SetJumpTarget(skipToRealDispatch2); - - dispatcherNoCheck = GetCodePtr(); - - // Debug - if (enableDebug) { - MOV(W0, DOWNCOUNTREG); - MOV(X1, MEMBASEREG); - MOV(X2, JITBASEREG); - QuickCallFunction(SCRATCH1_64, (void *)&ShowPC); - } - - LDR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, pc)); - ANDI2R(SCRATCH1, SCRATCH1, Memory::BLOCK_SHADOW_ENTRY_MASK); - MOVP2R(SCRATCH2_64, Memory::blockShadow); - dispatcherFetch = GetCodePtr(); - LDR(SCRATCH1, SCRATCH2_64, SCRATCH1_64); - // The entry is the block's offset into the code space, or 0 for none. - FixupBranch skipJump = CBZ(SCRATCH1); - ADD(SCRATCH1_64, JITBASEREG, SCRATCH1_64); - BR(SCRATCH1_64); - SetJumpTarget(skipJump); - - // No block found, let's jit. I don't think we actually need to save static regs that are in callee-save regs here but whatever. - // Also, rounding mode gotta be irrelevant here.. - // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt - // replaces it with the new block's entry, and destroying that block puts the hook value back. - blockShadowHook = GetCodePtr(); - SaveStaticRegisters(); - RestoreRoundingMode(true); - QuickCallFunctionR(SCRATCH1_64, (void *)&MIPSComp::JitAt, CTXREG); - ApplyRoundingMode(true); - LoadStaticRegisters(); - - B(dispatcherNoCheck); // no point in special casing this - - SetJumpTarget(bail); - SetJumpTarget(bailCoreState); - - MOVP2R(SCRATCH1_64, &coreState); - LDR(INDEX_UNSIGNED, SCRATCH1, SCRATCH1_64, 0); - CMP(SCRATCH1, 0); - B(CC_EQ, outerLoop); - - const uint8_t *quitLoop = GetCodePtr(); - SetJumpTarget(badCoreState); - - SaveStaticRegisters(); - RestoreRoundingMode(true); - - fp.ABI_PopRegisters(regs_to_save, regs_to_save_fp); - - RET(); - - crashHandler = GetCodePtr(); - MOVP2R(SCRATCH1_64, &coreState); - MOVI2R(SCRATCH2, CORE_RUNTIME_ERROR); - STR(INDEX_UNSIGNED, SCRATCH2, SCRATCH1_64, 0); - B(quitLoop); - - // Generate some integer conversion funcs. - // MIPS order! - static const RoundingMode roundModes[8] = { ROUND_N, ROUND_Z, ROUND_P, ROUND_M, ROUND_N, ROUND_Z, ROUND_P, ROUND_M }; - for (size_t i = 0; i < ARRAY_SIZE(roundModes); ++i) { - convertS0ToSCRATCH1[i] = AlignCode16(); - - fp.FCMP(S0, S0); // Detect NaN - fp.FCVTS(S0, S0, roundModes[i]); - FixupBranch skip = B(CC_VC); - MOVI2R(SCRATCH2, 0x7FFFFFFF); - fp.FMOV(S0, SCRATCH2); - SetJumpTarget(skip); - - RET(); - } - - // Leave this at the end, add more stuff above. - if (enableDisasm) { - std::vector lines = DisassembleArm64(start, (int)(GetCodePtr() - start)); - for (auto s : lines) { - INFO_LOG(Log::JIT, "%s", s.c_str()); - } - } - - // Let's spare the pre-generated code from unprotect-reprotect. - AlignCodePage(); - jitStartOffset = (int)(GetCodePtr() - start); - // Don't forget to zap the instruction cache! This must stay at the end of this function. - FlushIcache(); - EndWrite(); -} - -} // namespace MIPSComp - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompALU.cpp b/Core/MIPS/ARM64/Arm64CompALU.cpp deleted file mode 100644 index 15d8d1fc4b..0000000000 --- a/Core/MIPS/ARM64/Arm64CompALU.cpp +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include - -#include "Common/BitSet.h" -#include "Common/CPUDetect.h" -#include "Common/Data/Convert/SmallDataConvert.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" - -using namespace MIPSAnalyst; - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -//#define CONDITIONAL_DISABLE(flag) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } - -namespace MIPSComp { -using namespace Arm64Gen; -using namespace Arm64JitConstants; - -static u32 EvalOr(u32 a, u32 b) { return a | b; } -static u32 EvalEor(u32 a, u32 b) { return a ^ b; } -static u32 EvalAnd(u32 a, u32 b) { return a & b; } -static u32 EvalAdd(u32 a, u32 b) { return a + b; } -static u32 EvalSub(u32 a, u32 b) { return a - b; } - -void Arm64Jit::CompImmLogic(MIPSGPReg rs, MIPSGPReg rt, u32 uimm, void (ARM64XEmitter::*arith)(ARM64Reg dst, ARM64Reg src, ARM64Reg src2), bool (ARM64XEmitter::*tryArithI2R)(ARM64Reg dst, ARM64Reg src, u64 val), u32 (*eval)(u32 a, u32 b)) { - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, (*eval)(gpr.GetImm(rs), uimm)); - } else { - gpr.MapDirtyIn(rt, rs); - if (!(this->*tryArithI2R)(gpr.R(rt), gpr.R(rs), uimm)) { - MOVI2R(SCRATCH1, uimm); - (this->*arith)(gpr.R(rt), gpr.R(rs), SCRATCH1); - } - } -} - -void Arm64Jit::Comp_IType(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU_IMM); - u32 uimm = op & 0xFFFF; - s32 simm = SignExtend16ToS32(op); - u32 suimm = SignExtend16ToU32(op); - - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - // noop, won't write to ZERO. - if (rt == 0) - return; - - switch (op >> 26) { - case 8: // same as addiu? - case 9: // R(rt) = R(rs) + simm; break; //addiu - // Special-case for small adjustments of pointerified registers. Commonly for SP but happens for others. - if (rs == rt && gpr.IsMappedAsPointer(rs) && IsImmArithmetic(simm < 0 ? -simm : simm, nullptr, nullptr)) { - ARM64Reg r32 = gpr.RPtr(rs); - gpr.MarkDirty(r32); - ARM64Reg r = EncodeRegTo64(r32); - ADDI2R(r, r, simm); - } else { - if (simm >= 0) { - CompImmLogic(rs, rt, simm, &ARM64XEmitter::ADD, &ARM64XEmitter::TryADDI2R, &EvalAdd); - } else if (simm < 0) { - CompImmLogic(rs, rt, -simm, &ARM64XEmitter::SUB, &ARM64XEmitter::TrySUBI2R, &EvalSub); - } - } - break; - - case 12: CompImmLogic(rs, rt, uimm, &ARM64XEmitter::AND, &ARM64XEmitter::TryANDI2R, &EvalAnd); break; - case 13: CompImmLogic(rs, rt, uimm, &ARM64XEmitter::ORR, &ARM64XEmitter::TryORRI2R, &EvalOr); break; - case 14: CompImmLogic(rs, rt, uimm, &ARM64XEmitter::EOR, &ARM64XEmitter::TryEORI2R, &EvalEor); break; - - case 10: // R(rt) = (s32)R(rs) < simm; break; //slti - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, (s32)gpr.GetImm(rs) < simm ? 1 : 0); - break; - } else if (simm == 0) { - gpr.MapDirtyIn(rt, rs); - // Grab the sign bit (< 0) as 1/0. Slightly faster than a shift. - UBFX(gpr.R(rt), gpr.R(rs), 31, 1); - break; - } - gpr.MapDirtyIn(rt, rs); - if (!TryCMPI2R(gpr.R(rs), (u32)simm)) { - MOVI2R(SCRATCH1, simm); - CMP(gpr.R(rs), SCRATCH1); - } - CSET(gpr.R(rt), CC_LT); - break; - - case 11: // R(rt) = R(rs) < suimm; break; //sltiu - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, gpr.GetImm(rs) < suimm ? 1 : 0); - break; - } - gpr.MapDirtyIn(rt, rs); - if (!TryCMPI2R(gpr.R(rs), suimm)) { - MOVI2R(SCRATCH1, suimm); - CMP(gpr.R(rs), SCRATCH1); - } - CSET(gpr.R(rt), CC_LO); - break; - - case 15: // R(rt) = uimm << 16; //lui - gpr.SetImm(rt, uimm << 16); - break; - - default: - Comp_Generic(op); - break; - } -} - -void Arm64Jit::Comp_RType2(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU_BIT); - - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // Don't change $zr. - if (rd == 0) - return; - - switch (op & 63) { - case 22: //clz - if (gpr.IsImm(rs)) { - u32 value = gpr.GetImm(rs); - int x = 31; - int count = 0; - while (x >= 0 && !(value & (1 << x))) { - count++; - x--; - } - gpr.SetImm(rd, count); - break; - } - gpr.MapDirtyIn(rd, rs); - CLZ(gpr.R(rd), gpr.R(rs)); - break; - case 23: //clo - if (gpr.IsImm(rs)) { - u32 value = gpr.GetImm(rs); - int x = 31; - int count = 0; - while (x >= 0 && (value & (1 << x))) { - count++; - x--; - } - gpr.SetImm(rd, count); - break; - } - gpr.MapDirtyIn(rd, rs); - MVN(gpr.R(rd), gpr.R(rs)); - CLZ(gpr.R(rd), gpr.R(rd)); - break; - default: - DISABLE; - } -} - -void Arm64Jit::CompType3(MIPSGPReg rd, MIPSGPReg rs, MIPSGPReg rt, void (ARM64XEmitter::*arith)(ARM64Reg dst, ARM64Reg rm, ARM64Reg rn), bool (ARM64XEmitter::*tryArithI2R)(ARM64Reg dst, ARM64Reg rm, u64 val), u32(*eval)(u32 a, u32 b), bool symmetric) { - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, (*eval)(gpr.GetImm(rs), gpr.GetImm(rt))); - return; - } - - // Optimize anything against zero. - if (gpr.IsImm(rs) && gpr.GetImm(rs) == 0) { - gpr.MapDirtyIn(rd, rt); - (this->*arith)(gpr.R(rd), WZR, gpr.R(rt)); - return; - } - if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0) { - gpr.MapDirtyIn(rd, rs); - (this->*arith)(gpr.R(rd), gpr.R(rs), WZR); - return; - } - - if (gpr.IsImm(rt) || (gpr.IsImm(rs) && symmetric)) { - MIPSGPReg lhs = gpr.IsImm(rs) ? rt : rs; - MIPSGPReg rhs = gpr.IsImm(rs) ? rs : rt; - u32 rhsImm = gpr.GetImm(rhs); - gpr.MapDirtyIn(rd, lhs); - if ((this->*tryArithI2R)(gpr.R(rd), gpr.R(lhs), rhsImm)) { - return; - } - // If rd is rhs, we may have lost it in the MapDirtyIn(). lhs was kept. - // This means the rhsImm value was never flushed to rhs, and would be garbage. - if (rd == rhs) { - // Luckily, it was just an imm. - gpr.SetImm(rhs, rhsImm); - } - } - - // Can't do the RSB optimization on ARM64 - no RSB! - - // Generic solution. If it's an imm, better to flush at this point. - gpr.MapDirtyInIn(rd, rs, rt); - (this->*arith)(gpr.R(rd), gpr.R(rs), gpr.R(rt)); -} - -void Arm64Jit::Comp_RType3(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU); - - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // noop, won't write to ZERO. - if (rd == 0) - return; - - switch (op & 63) { - case 10: //if (!R(rt)) R(rd) = R(rs); break; //movz - gpr.MapDirtyInIn(rd, rt, rs, false); - CMP(gpr.R(rt), 0); - CSEL(gpr.R(rd), gpr.R(rs), gpr.R(rd), CC_EQ); - break; - case 11:// if (R(rt)) R(rd) = R(rs); break; //movn - gpr.MapDirtyInIn(rd, rt, rs, false); - CMP(gpr.R(rt), 0); - CSEL(gpr.R(rd), gpr.R(rs), gpr.R(rd), CC_NEQ); - break; - - case 32: //R(rd) = R(rs) + R(rt); break; //add - case 33: //R(rd) = R(rs) + R(rt); break; //addu - if (gpr.IsImm(rs) && gpr.GetImm(rs) == 0 && !gpr.IsImm(rt)) { - // Special case: actually a mov, avoid arithmetic. - gpr.MapDirtyIn(rd, rt); - MOV(gpr.R(rd), gpr.R(rt)); - } else if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0 && !gpr.IsImm(rs)) { - gpr.MapDirtyIn(rd, rs); - MOV(gpr.R(rd), gpr.R(rs)); - } else { - CompType3(rd, rs, rt, &ARM64XEmitter::ADD, &ARM64XEmitter::TryADDI2R, &EvalAdd, true); - } - break; - - case 34: //R(rd) = R(rs) - R(rt); break; //sub - case 35: //R(rd) = R(rs) - R(rt); break; //subu - CompType3(rd, rs, rt, &ARM64XEmitter::SUB, &ARM64XEmitter::TrySUBI2R, &EvalSub, false); - break; - - case 36: //R(rd) = R(rs) & R(rt); break; //and - CompType3(rd, rs, rt, &ARM64XEmitter::AND, &ARM64XEmitter::TryANDI2R, &EvalAnd, true); - break; - case 37: //R(rd) = R(rs) | R(rt); break; //or - CompType3(rd, rs, rt, &ARM64XEmitter::ORR, &ARM64XEmitter::TryORRI2R, &EvalOr, true); - break; - case 38: //R(rd) = R(rs) ^ R(rt); break; //xor/eor - CompType3(rd, rs, rt, &ARM64XEmitter::EOR, &ARM64XEmitter::TryEORI2R, &EvalEor, true); - break; - - case 39: // R(rd) = ~(R(rs) | R(rt)); break; //nor - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, ~(gpr.GetImm(rs) | gpr.GetImm(rt))); - } else if (gpr.IsImm(rs) || gpr.IsImm(rt)) { - MIPSGPReg lhs = gpr.IsImm(rs) ? rt : rs; - MIPSGPReg rhs = gpr.IsImm(rs) ? rs : rt; - u32 rhsImm = gpr.GetImm(rhs); - if (rhsImm == 0) { - gpr.MapDirtyIn(rd, lhs); - MVN(gpr.R(rd), gpr.R(lhs)); - } else { - // Ignored, just for IsImmLogical. - unsigned int n, imm_s, imm_r; - if (IsImmLogical(rhsImm, 32, &n, &imm_s, &imm_r)) { - // Great, we can avoid flushing a reg. - gpr.MapDirtyIn(rd, lhs); - ORRI2R(gpr.R(rd), gpr.R(lhs), rhsImm); - } else { - gpr.MapDirtyInIn(rd, rs, rt); - ORR(gpr.R(rd), gpr.R(rs), gpr.R(rt)); - } - MVN(gpr.R(rd), gpr.R(rd)); - } - } else { - gpr.MapDirtyInIn(rd, rs, rt); - ORR(gpr.R(rd), gpr.R(rs), gpr.R(rt)); - MVN(gpr.R(rd), gpr.R(rd)); - } - break; - - case 42: //R(rd) = (int)R(rs) < (int)R(rt); break; //slt - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, (s32)gpr.GetImm(rs) < (s32)gpr.GetImm(rt)); - } else { - gpr.MapDirtyInIn(rd, rs, rt); - CMP(gpr.R(rs), gpr.R(rt)); - CSET(gpr.R(rd), CC_LT); - } - break; - - case 43: //R(rd) = R(rs) < R(rt); break; //sltu - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, gpr.GetImm(rs) < gpr.GetImm(rt)); - } else { - gpr.MapDirtyInIn(rd, rs, rt); - CMP(gpr.R(rs), gpr.R(rt)); - CSET(gpr.R(rd), CC_LO); - } - break; - - case 44: //R(rd) = max(R(rs), R(rt); break; //max - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, (u32)std::max((int)gpr.GetImm(rs), (int)gpr.GetImm(rt))); - break; - } - gpr.MapDirtyInIn(rd, rs, rt); - CMP(gpr.R(rs), gpr.R(rt)); - CSEL(gpr.R(rd), gpr.R(rs), gpr.R(rt), CC_GE); - break; - - case 45: //R(rd) = min(R(rs), R(rt)); break; //min - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, (u32)std::min((int)gpr.GetImm(rs), (int)gpr.GetImm(rt))); - break; - } - gpr.MapDirtyInIn(rd, rs, rt); - CMP(gpr.R(rs), gpr.R(rt)); - CSEL(gpr.R(rd), gpr.R(rs), gpr.R(rt), CC_LE); - break; - - default: - Comp_Generic(op); - break; - } -} - -void Arm64Jit::CompShiftImm(MIPSOpcode op, Arm64Gen::ShiftType shiftType, int sa) { - MIPSGPReg rd = _RD; - MIPSGPReg rt = _RT; - if (gpr.IsImm(rt)) { - switch (shiftType) { - case ST_LSL: - gpr.SetImm(rd, gpr.GetImm(rt) << sa); - break; - case ST_LSR: - gpr.SetImm(rd, gpr.GetImm(rt) >> sa); - break; - case ST_ASR: - gpr.SetImm(rd, (int)gpr.GetImm(rt) >> sa); - break; - case ST_ROR: - gpr.SetImm(rd, (gpr.GetImm(rt) >> sa) | (gpr.GetImm(rt) << (32 - sa))); - break; - default: - DISABLE; - } - } else { - gpr.MapDirtyIn(rd, rt); - MOV(gpr.R(rd), gpr.R(rt), ArithOption(gpr.R(rd), shiftType, sa)); - } -} - -void Arm64Jit::CompShiftVar(MIPSOpcode op, Arm64Gen::ShiftType shiftType) { - MIPSGPReg rd = _RD; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - if (gpr.IsImm(rs)) { - int sa = gpr.GetImm(rs) & 0x1F; - CompShiftImm(op, shiftType, sa); - return; - } - gpr.MapDirtyInIn(rd, rs, rt); - switch (shiftType) { - case ST_LSL: LSLV(gpr.R(rd), gpr.R(rt), gpr.R(rs)); break; - case ST_LSR: LSRV(gpr.R(rd), gpr.R(rt), gpr.R(rs)); break; - case ST_ASR: ASRV(gpr.R(rd), gpr.R(rt), gpr.R(rs)); break; - case ST_ROR: RORV(gpr.R(rd), gpr.R(rt), gpr.R(rs)); break; - } -} - -void Arm64Jit::Comp_ShiftType(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU); - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - int fd = _FD; - int sa = _SA; - - // noop, won't write to ZERO. - if (rd == 0) - return; - - // WARNING : ROTR - switch (op & 0x3f) { - case 0: CompShiftImm(op, ST_LSL, sa); break; //sll - case 2: CompShiftImm(op, (rs & 1) ? ST_ROR : ST_LSR, sa); break; //srl - case 3: CompShiftImm(op, ST_ASR, sa); break; //sra - case 4: CompShiftVar(op, ST_LSL); break; //sllv - case 6: CompShiftVar(op, (fd & 1) ? ST_ROR : ST_LSR); break; //srlv - case 7: CompShiftVar(op, ST_ASR); break; //srav - default: - DISABLE; - break; - } -} - -void Arm64Jit::Comp_Special3(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rs = _RS; - MIPSGPReg rt = _RT; - - int pos = _POS; - int size = _SIZE + 1; - u32 mask = 0xFFFFFFFFUL >> (32 - size); - - // Don't change $zr. - if (rt == 0) - return; - - switch (op & 0x3f) { - case 0x0: //ext - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, (gpr.GetImm(rs) >> pos) & mask); - return; - } - - gpr.MapDirtyIn(rt, rs); - // pos + size can go past 32 (the field then just ends at bit 31), which UBFX can't encode. - UBFX(gpr.R(rt), gpr.R(rs), pos, std::min(size, 32 - pos)); - break; - - case 0x4: //ins - { - const u32 sourcemask = mask >> pos; - // With msb below pos there's nothing to insert, and no BFI for it. - if (sourcemask == 0) - return; - if (gpr.IsImm(rs)) { - const u32 inserted = (gpr.GetImm(rs) & sourcemask) << pos; - if (gpr.IsImm(rt)) { - const u32 destmask = ~(sourcemask << pos); - gpr.SetImm(rt, (gpr.GetImm(rt) & destmask) | inserted); - return; - } - - // It might be nice to avoid flushing rs, but it's a little slower and - // usually more instructions. Not worth it. - gpr.MapDirtyIn(rt, rs, false); - BFI(gpr.R(rt), gpr.R(rs), pos, size - pos); - } else { - gpr.MapDirtyIn(rt, rs, false); - BFI(gpr.R(rt), gpr.R(rs), pos, size - pos); - } - } - break; - } -} - -void Arm64Jit::Comp_Allegrex(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rt = _RT; - MIPSGPReg rd = _RD; - // Don't change $zr. - if (rd == 0) - return; - - switch (AllegrexBitOp(op)) { - case 16: // seb // R(rd) = SignExtend8ToU32(R(rt)); - if (gpr.IsImm(rt)) { - gpr.SetImm(rd, SignExtend8ToU32(gpr.GetImm(rt))); - return; - } - gpr.MapDirtyIn(rd, rt); - SXTB(gpr.R(rd), gpr.R(rt)); - break; - - case 24: // seh - if (gpr.IsImm(rt)) { - gpr.SetImm(rd, SignExtend16ToU32(gpr.GetImm(rt))); - return; - } - gpr.MapDirtyIn(rd, rt); - SXTH(gpr.R(rd), gpr.R(rt)); - break; - - case 20: //bitrev - if (gpr.IsImm(rt)) { - // http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel - u32 v = gpr.GetImm(rt); - v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1); // odd<->even - v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2); // pair<->pair - v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4); // nibb<->nibb - v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8); // byte<->byte - v = (v >> 16) | (v << 16); // hword<->hword - gpr.SetImm(rd, v); - return; - } - - gpr.MapDirtyIn(rd, rt); - RBIT(gpr.R(rd), gpr.R(rt)); - break; - - default: - Comp_Generic(op); - return; - } -} - -void Arm64Jit::Comp_Allegrex2(MIPSOpcode op) { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rt = _RT; - MIPSGPReg rd = _RD; - // Don't change $zr. - if (rd == 0) - return; - - switch (AllegrexBitOp(op)) { - case 2: //wsbh - if (gpr.IsImm(rt)) { - gpr.SetImm(rd, ((gpr.GetImm(rt) & 0xFF00FF00) >> 8) | ((gpr.GetImm(rt) & 0x00FF00FF) << 8)); - } else { - gpr.MapDirtyIn(rd, rt); - REV16(gpr.R(rd), gpr.R(rt)); - } - break; - case 3: //wsbw - if (gpr.IsImm(rt)) { - gpr.SetImm(rd, swap32(gpr.GetImm(rt))); - } else { - gpr.MapDirtyIn(rd, rt); - REV32(gpr.R(rd), gpr.R(rt)); - } - break; - default: - Comp_Generic(op); - break; - } -} - -void Arm64Jit::Comp_MulDivType(MIPSOpcode op) { - CONDITIONAL_DISABLE(MULDIV); - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // Note that in all cases below, LO is actually mapped to HI:LO. - // That is, the host reg is 64 bits and has HI at the top. - // HI is not mappable. - - switch (op & 63) { - case 16: // R(rd) = HI; //mfhi - if (rd == MIPS_REG_ZERO) { - break; - } - // LO and HI are in the same reg. - if (gpr.IsImm(MIPS_REG_LO)) { - gpr.SetImm(rd, gpr.GetImm(MIPS_REG_LO) >> 32); - break; - } - gpr.MapDirtyIn(rd, MIPS_REG_LO); - UBFX(EncodeRegTo64(gpr.R(rd)), EncodeRegTo64(gpr.R(MIPS_REG_LO)), 32, 32); - break; - - case 17: // HI = R(rs); //mthi - if (gpr.IsImm(rs) && gpr.IsImm(MIPS_REG_LO)) { - gpr.SetImm(MIPS_REG_LO, (gpr.GetImm(rs) << 32) | (gpr.GetImm(MIPS_REG_LO) & 0xFFFFFFFFULL)); - break; - } - gpr.MapDirtyIn(MIPS_REG_LO, rs, false); - BFI(EncodeRegTo64(gpr.R(MIPS_REG_LO)), EncodeRegTo64(gpr.R(rs)), 32, 32); - break; - - case 18: // R(rd) = LO; break; //mflo - if (rd == MIPS_REG_ZERO) { - break; - } - if (gpr.IsImm(MIPS_REG_LO)) { - gpr.SetImm(rd, gpr.GetImm(MIPS_REG_LO) & 0xFFFFFFFFULL); - break; - } - gpr.MapDirtyIn(rd, MIPS_REG_LO); - MOV(gpr.R(rd), gpr.R(MIPS_REG_LO)); - break; - - case 19: // LO = R(rs); break; //mtlo - if (gpr.IsImm(rs) && gpr.IsImm(MIPS_REG_LO)) { - gpr.SetImm(MIPS_REG_LO, gpr.GetImm(rs) | (gpr.GetImm(MIPS_REG_LO) & ~0xFFFFFFFFULL)); - break; - } - gpr.MapDirtyIn(MIPS_REG_LO, rs, false); - BFI(EncodeRegTo64(gpr.R(MIPS_REG_LO)), EncodeRegTo64(gpr.R(rs)), 0, 32); - break; - - case 24: //mult (the most popular one). lo,hi = signed mul (rs * rt) - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - s64 result = (s64)(s32)gpr.GetImm(rs) * (s64)(s32)gpr.GetImm(rt); - gpr.SetImm(MIPS_REG_LO, (u64)result); - break; - } - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt); - SMULL(EncodeRegTo64(gpr.R(MIPS_REG_LO)), gpr.R(rs), gpr.R(rt)); - break; - - case 25: //multu (2nd) lo,hi = unsigned mul (rs * rt) - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - u64 resultBits = (u64)gpr.GetImm(rs) * (u64)gpr.GetImm(rt); - gpr.SetImm(MIPS_REG_LO, resultBits); - break; - } - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt); - // In case of pointerification, let's use UMULL. - UMULL(EncodeRegTo64(gpr.R(MIPS_REG_LO)), gpr.R(rs), gpr.R(rt)); - break; - - case 26: //div - { - // TODO: Does this handle INT_MAX, 0, etc. correctly? - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt); - SDIV(gpr.R(MIPS_REG_LO), gpr.R(rs), gpr.R(rt)); - MSUB(SCRATCH1, gpr.R(rt), gpr.R(MIPS_REG_LO), gpr.R(rs)); - - CMPI2R(gpr.R(rt), 0); - FixupBranch skipZero = B(CC_NEQ); - // HI set properly already, we just need to set LO. - MOVI2R(gpr.R(MIPS_REG_LO), -1); - CMPI2R(gpr.R(rs), 0); - FixupBranch moreThan16Bit = B(CC_GE); - MOVI2R(gpr.R(MIPS_REG_LO), 1); - SetJumpTarget(moreThan16Bit); - SetJumpTarget(skipZero); - - BFI(EncodeRegTo64(gpr.R(MIPS_REG_LO)), SCRATCH1_64, 32, 32); - break; - } - - case 27: //divu - // Do we have a known power-of-two denominator? Yes, this happens. - if (gpr.IsImm(rt) && (gpr.GetImm(rt) & (gpr.GetImm(rt) - 1)) == 0 && gpr.GetImm(rt) != 0) { - u32 denominator = gpr.GetImm(rt); - gpr.MapDirtyIn(MIPS_REG_LO, rs); - // Remainder is just an AND, neat. - ANDI2R(SCRATCH1, gpr.R(rs), denominator - 1, SCRATCH1); - int shift = 0; - while (denominator != 0) { - ++shift; - denominator >>= 1; - } - // The shift value is one too much for the divide by the same value. - if (shift > 1) { - LSR(gpr.R(MIPS_REG_LO), gpr.R(rs), shift - 1); - } else { - MOV(gpr.R(MIPS_REG_LO), gpr.R(rs)); - } - BFI(EncodeRegTo64(gpr.R(MIPS_REG_LO)), SCRATCH1_64, 32, 32); - } else { - // TODO: Does this handle INT_MAX, 0, etc. correctly? - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt); - UDIV(gpr.R(MIPS_REG_LO), gpr.R(rs), gpr.R(rt)); - MSUB(SCRATCH1, gpr.R(rt), gpr.R(MIPS_REG_LO), gpr.R(rs)); - - CMPI2R(gpr.R(rt), 0); - FixupBranch skipZero = B(CC_NEQ); - // HI set properly, we just need to set LO. - MOVI2R(SCRATCH2, 0xFFFF); - MOVI2R(gpr.R(MIPS_REG_LO), -1); - CMP(gpr.R(rs), SCRATCH2); - FixupBranch moreThan16Bit = B(CC_HI); - MOV(gpr.R(MIPS_REG_LO), SCRATCH2); - SetJumpTarget(moreThan16Bit); - SetJumpTarget(skipZero); - - BFI(EncodeRegTo64(gpr.R(MIPS_REG_LO)), SCRATCH1_64, 32, 32); - } - break; - - case 28: //madd - { - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt, false); - ARM64Reg lo64 = EncodeRegTo64(gpr.R(MIPS_REG_LO)); - SMADDL(lo64, gpr.R(rs), gpr.R(rt), lo64); // Operands are backwards in the emitter! - } - break; - - case 29: //maddu - { - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt, false); - ARM64Reg lo64 = EncodeRegTo64(gpr.R(MIPS_REG_LO)); - UMADDL(lo64, gpr.R(rs), gpr.R(rt), lo64); // Operands are backwards in the emitter! - } - break; - - case 46: // msub - { - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt, false); - ARM64Reg lo64 = EncodeRegTo64(gpr.R(MIPS_REG_LO)); - SMSUBL(lo64, gpr.R(rs), gpr.R(rt), lo64); // Operands are backwards in the emitter! - } - break; - - case 47: // msubu - { - gpr.MapDirtyInIn(MIPS_REG_LO, rs, rt, false); - ARM64Reg lo64 = EncodeRegTo64(gpr.R(MIPS_REG_LO)); - UMSUBL(lo64, gpr.R(rs), gpr.R(rt), lo64); // Operands are backwards in the emitter! - break; - } - - default: - DISABLE; - } -} - -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompBranch.cpp b/Core/MIPS/ARM64/Arm64CompBranch.cpp deleted file mode 100644 index ccdc4913d8..0000000000 --- a/Core/MIPS/ARM64/Arm64CompBranch.cpp +++ /dev/null @@ -1,670 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include "Common/Data/Convert/SmallDataConvert.h" -#include "Common/Profiler/Profiler.h" - -#include "Core/Config.h" -#include "Core/Core.h" -#include "Core/Reporting.h" -#include "Core/MemMap.h" -#include "Core/HLE/HLE.h" -#include "Core/HLE/HLETables.h" - -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSTables.h" - -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Core/MIPS/JitCommon/JitBlockCache.h" - -#include "Common/Arm64Emitter.h" - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM26 (op & 0x03FFFFFF) -#define TARGET16 ((int)(SignExtend16ToU32(op) << 2)) -#define TARGET26 (_IMM26 << 2) - -#define LOOPOPTIMIZATION 0 - -// We can disable nice delay slots. -// #define CONDITIONAL_NICE_DELAYSLOT branchInfo.delaySlotIsNice = false; -#define CONDITIONAL_NICE_DELAYSLOT ; - -using namespace MIPSAnalyst; - -namespace MIPSComp -{ - using namespace Arm64Gen; - using namespace Arm64JitConstants; - -void Arm64Jit::BranchRSRTComp(MIPSOpcode op, CCFlags cc, bool likely) -{ - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in RSRTComp delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceReg(op, branchInfo.delaySlotOp, rt, rs); - CONDITIONAL_NICE_DELAYSLOT; - - bool immBranch = false; - bool immBranchTaken = false; - if (gpr.IsImm(rs) && gpr.IsImm(rt) && !branchInfo.delaySlotIsBranch) { - // The cc flags are opposites: when NOT to take the branch. - bool immBranchNotTaken; - s32 rsImm = (s32)gpr.GetImm(rs); - s32 rtImm = (s32)gpr.GetImm(rt); - - switch (cc) - { - case CC_EQ: immBranchNotTaken = rsImm == rtImm; break; - case CC_NEQ: immBranchNotTaken = rsImm != rtImm; break; - default: immBranchNotTaken = false; _dbg_assert_msg_(false, "Bad cc flag in BranchRSRTComp()."); - } - immBranch = true; - immBranchTaken = !immBranchNotTaken; - } - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - if (immBranch) { - // Continuing is handled above, this is just static jumping. - if (immBranchTaken || !likely) - CompileDelaySlot(DELAYSLOT_FLUSH); - else - FlushAll(); - - const u32 destAddr = immBranchTaken ? targetAddr : notTakenTarget; - WriteExit(destAddr, js.nextExit++); - } else { - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - // We might be able to flip the condition (EQ/NEQ are easy.) - const bool canFlip = cc == CC_EQ || cc == CC_NEQ; - const bool rsIsZero = gpr.IsImm(rs) && gpr.GetImm(rs) == 0; - const bool rtIsZero = gpr.IsImm(rt) && gpr.GetImm(rt) == 0; - - Arm64Gen::FixupBranch ptr; - if ((likely || branchInfo.delaySlotIsNice) && (rsIsZero || rtIsZero) && canFlip) { - // Special case, we can just use CBZ/CBNZ directly. - MIPSGPReg r = rsIsZero ? rt : rs; - gpr.MapReg(r); - // Flush should keep r in the same armreg. - ARM64Reg ar = gpr.R(r); - FlushAll(); - if (cc == CC_EQ) { - ptr = CBZ(ar); - } else { - ptr = CBNZ(ar); - } - } else { - u32 val; - bool shift; - if (gpr.IsImm(rt) && IsImmArithmetic(gpr.GetImm(rt), &val, &shift)) { - gpr.MapReg(rs); - CMP(gpr.R(rs), val, shift); - } else if (gpr.IsImm(rt) && IsImmArithmetic((u64)(s64)-(s32)gpr.GetImm(rt), &val, &shift)) { - gpr.MapReg(rs); - CMN(gpr.R(rs), val, shift); - } else if (gpr.IsImm(rs) && IsImmArithmetic(gpr.GetImm(rs), &val, &shift) && canFlip) { - gpr.MapReg(rt); - CMP(gpr.R(rt), val, shift); - } else if (gpr.IsImm(rs) && IsImmArithmetic((u64)(s64)-(s32)gpr.GetImm(rs), &val, &shift) && canFlip) { - gpr.MapReg(rt); - CMN(gpr.R(rt), val, shift); - } else { - gpr.MapInIn(rs, rt); - CMP(gpr.R(rs), gpr.R(rt)); - } - - if (!likely) { - if (!branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_SAFE_FLUSH); - else - FlushAll(); - ptr = B(cc); - } else { - FlushAll(); - ptr = B(cc); - } - } - - if (likely && !branchInfo.delaySlotIsBranch) { - // Only executed when taking the branch. - CompileDelaySlot(DELAYSLOT_FLUSH); - } - - if (branchInfo.delaySlotIsBranch) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - FlushAll(); - } - - // Take the branch - WriteExit(targetAddr, js.nextExit++); - - SetJumpTarget(ptr); - // Not taken - WriteExit(notTakenTarget, js.nextExit++); - } - - js.compiling = false; -} - - -void Arm64Jit::BranchRSZeroComp(MIPSOpcode op, CCFlags cc, bool andLink, bool likely) -{ - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in RSZeroComp delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - MIPSGPReg rs = _RS; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), andLink, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceReg(op, branchInfo.delaySlotOp, rs); - CONDITIONAL_NICE_DELAYSLOT; - - bool immBranch = false; - bool immBranchTaken = false; - if (gpr.IsImm(rs) && !branchInfo.delaySlotIsBranch) { - // The cc flags are opposites: when NOT to take the branch. - bool immBranchNotTaken; - s32 imm = (s32)gpr.GetImm(rs); - - switch (cc) - { - case CC_GT: immBranchNotTaken = imm > 0; break; - case CC_GE: immBranchNotTaken = imm >= 0; break; - case CC_LT: immBranchNotTaken = imm < 0; break; - case CC_LE: immBranchNotTaken = imm <= 0; break; - default: immBranchNotTaken = false; _dbg_assert_msg_(false, "Bad cc flag in BranchRSZeroComp()."); - } - immBranch = true; - immBranchTaken = !immBranchNotTaken; - } - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - if (immBranch) { - // Continuing is handled above, this is just static jumping. - if (andLink) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - if (immBranchTaken || !likely) - CompileDelaySlot(DELAYSLOT_FLUSH); - else - FlushAll(); - - const u32 destAddr = immBranchTaken ? targetAddr : notTakenTarget; - WriteExit(destAddr, js.nextExit++); - } else { - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - gpr.MapReg(rs); - CMP(gpr.R(rs), 0); - - if (andLink) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - - Arm64Gen::FixupBranch ptr; - if (!likely) - { - if (!branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_SAFE_FLUSH); - else - FlushAll(); - ptr = B(cc); - } - else - { - FlushAll(); - ptr = B(cc); - if (!branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_FLUSH); - } - - if (branchInfo.delaySlotIsBranch) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - FlushAll(); - } - - // Take the branch - WriteExit(targetAddr, js.nextExit++); - - SetJumpTarget(ptr); - // Not taken - WriteExit(notTakenTarget, js.nextExit++); - } - js.compiling = false; -} - - -void Arm64Jit::Comp_RelBranch(MIPSOpcode op) -{ - // The CC flags here should be opposite of the actual branch becuase they skip the branching action. - switch (op >> 26) - { - case 4: BranchRSRTComp(op, CC_NEQ, false); break;//beq - case 5: BranchRSRTComp(op, CC_EQ, false); break;//bne - - case 6: BranchRSZeroComp(op, CC_GT, false, false); break;//blez - case 7: BranchRSZeroComp(op, CC_LE, false, false); break;//bgtz - - case 20: BranchRSRTComp(op, CC_NEQ, true); break;//beql - case 21: BranchRSRTComp(op, CC_EQ, true); break;//bnel - - case 22: BranchRSZeroComp(op, CC_GT, false, true); break;//blezl - case 23: BranchRSZeroComp(op, CC_LE, false, true); break;//bgtzl - - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } -} - -void Arm64Jit::Comp_RelBranchRI(MIPSOpcode op) -{ - switch ((op >> 16) & 0x1F) - { - case 0: BranchRSZeroComp(op, CC_GE, false, false); break; //if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 4; break;//bltz - case 1: BranchRSZeroComp(op, CC_LT, false, false); break; //if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 4; break;//bgez - case 2: BranchRSZeroComp(op, CC_GE, false, true); break; //if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 8; break;//bltzl - case 3: BranchRSZeroComp(op, CC_LT, false, true); break; //if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 8; break;//bgezl - case 16: BranchRSZeroComp(op, CC_GE, true, false); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 4; break;//bltzal - case 17: BranchRSZeroComp(op, CC_LT, true, false); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 4; break;//bgezal - case 18: BranchRSZeroComp(op, CC_GE, true, true); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) < 0) DelayBranchTo(addr); else SkipLikely(); break;//bltzall - case 19: BranchRSZeroComp(op, CC_LT, true, true); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) >= 0) DelayBranchTo(addr); else SkipLikely(); break;//bgezall - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } -} - -// If likely is set, discard the branch slot if NOT taken. -void Arm64Jit::BranchFPFlag(MIPSOpcode op, CCFlags cc, bool likely) { - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in FPFlag delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceFPU(op, branchInfo.delaySlotOp); - CONDITIONAL_NICE_DELAYSLOT; - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - gpr.MapReg(MIPS_REG_FPCOND); - Arm64Gen::FixupBranch ptr; - if (likely || branchInfo.delaySlotIsNice) { - // FlushAll() won't actually change the reg. - ARM64Reg ar = gpr.R(MIPS_REG_FPCOND); - FlushAll(); - if (cc == CC_EQ) { - ptr = TBZ(ar, 0); - } else { - ptr = TBNZ(ar, 0); - } - } else { - TSTI2R(gpr.R(MIPS_REG_FPCOND), 1, SCRATCH1); - if (!branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_SAFE_FLUSH); - ptr = B(cc); - } - - if (likely && !branchInfo.delaySlotIsBranch) { - CompileDelaySlot(DELAYSLOT_FLUSH); - } - - if (branchInfo.delaySlotIsBranch) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - FlushAll(); - } - - // Take the branch - WriteExit(targetAddr, js.nextExit++); - - SetJumpTarget(ptr); - // Not taken - WriteExit(ResolveNotTakenTarget(branchInfo), js.nextExit++); - js.compiling = false; -} - -void Arm64Jit::Comp_FPUBranch(MIPSOpcode op) { - switch((op >> 16) & 0x1f) { - case 0: BranchFPFlag(op, CC_NEQ, false); break; // bc1f - case 1: BranchFPFlag(op, CC_EQ, false); break; // bc1t - case 2: BranchFPFlag(op, CC_NEQ, true); break; // bc1fl - case 3: BranchFPFlag(op, CC_EQ, true); break; // bc1tl - default: - _dbg_assert_msg_( 0, "Trying to interpret instruction that can't be interpreted"); - break; - } -} - -// If likely is set, discard the branch slot if NOT taken. -void Arm64Jit::BranchVFPUFlag(MIPSOpcode op, CCFlags cc, bool likely) { - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in VFPU delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - // Sometimes there's a VFPU branch in a delay slot (Disgaea 2: Dark Hero Days, Zettai Hero Project, La Pucelle) - // The behavior is undefined - the CPU may take the second branch even if the first one passes. - // However, it does consistently try each branch, which these games seem to expect. - branchInfo.delaySlotIsNice = IsDelaySlotNiceVFPU(op, branchInfo.delaySlotOp); - CONDITIONAL_NICE_DELAYSLOT; - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - if (!likely && branchInfo.delaySlotIsNice) - CompileDelaySlot(DELAYSLOT_NICE); - - int imm3 = (op >> 18) & 7; - - gpr.MapReg(MIPS_REG_VFPUCC); - Arm64Gen::FixupBranch ptr; - if (likely || branchInfo.delaySlotIsNice || branchInfo.delaySlotIsBranch) { - // FlushAll() won't actually change the reg. - ARM64Reg ar = gpr.R(MIPS_REG_VFPUCC); - FlushAll(); - if (cc == CC_EQ) { - ptr = TBZ(ar, imm3); - } else { - ptr = TBNZ(ar, imm3); - } - } else { - TSTI2R(gpr.R(MIPS_REG_VFPUCC), 1ULL << imm3, SCRATCH1); - if (!branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_SAFE_FLUSH); - ptr = B(cc); - } - - if (likely && !branchInfo.delaySlotIsBranch) { - CompileDelaySlot(DELAYSLOT_FLUSH); - } - - if (branchInfo.delaySlotIsBranch) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - FlushAll(); - } - - // Take the branch - WriteExit(targetAddr, js.nextExit++); - - SetJumpTarget(ptr); - // Not taken - WriteExit(ResolveNotTakenTarget(branchInfo), js.nextExit++); - js.compiling = false; -} - -void Arm64Jit::Comp_VBranch(MIPSOpcode op) -{ - switch ((op >> 16) & 3) - { - case 0: BranchVFPUFlag(op, CC_NEQ, false); break; // bvf - case 1: BranchVFPUFlag(op, CC_EQ, false); break; // bvt - case 2: BranchVFPUFlag(op, CC_NEQ, true); break; // bvfl - case 3: BranchVFPUFlag(op, CC_EQ, true); break; // bvtl - } -} - -static void HitInvalidJump(uint32_t dest) { - Core_ExecException(dest, currentMIPS->pc - 8, ExecExceptionType::JUMP); -} - -void Arm64Jit::Comp_Jump(MIPSOpcode op) { - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in Jump delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - u32 off = TARGET26; - u32 targetAddr = (GetCompilerPC() & 0xF0000000) | off; - - // Might be a stubbed address or something? - if (!Memory::IsValidAddress(targetAddr) || (targetAddr & 3) != 0) { - if (js.nextExit == 0) { - ERROR_LOG(Log::JIT, "Jump to invalid address: %08x", targetAddr); - } else { - js.compiling = false; - } - // TODO: Mark this block dirty or something? May be indication it will be changed by imports. - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - MOVI2R(SCRATCH1, GetCompilerPC() + 8); - MovToPC(SCRATCH1); - MOVI2R(W0, targetAddr); - QuickCallFunction(SCRATCH1, (const void *)&HitInvalidJump); - WriteSyscallExit(); - return; - } - - switch (op >> 26) { - case 2: //j - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - WriteExit(targetAddr, js.nextExit++); - break; - - case 3: //jal - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - WriteExit(targetAddr, js.nextExit++); - break; - - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } - js.compiling = false; -} - -void Arm64Jit::Comp_JumpReg(MIPSOpcode op) -{ - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in JumpReg delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - bool andLink = (op & 0x3f) == 9 && rd != MIPS_REG_ZERO; - - MIPSOpcode delaySlotOp = GetOffsetInstruction(1); - js.downcountAmount += MIPSGetInstructionCycleEstimate(delaySlotOp); - bool delaySlotIsNice = IsDelaySlotNiceReg(op, delaySlotOp, rs); - if (andLink && rs == rd) - delaySlotIsNice = false; - CONDITIONAL_NICE_DELAYSLOT; - - ARM64Reg destReg = INVALID_REG; - if (IsSyscall(delaySlotOp)) { - gpr.MapReg(rs); - MovToPC(gpr.R(rs)); // For syscall to be able to return. - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_FLUSH); - return; // Syscall (delay slot) wrote exit code. - } else if (delaySlotIsNice) { - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - - if (!andLink && rs == MIPS_REG_RA && g_Config.bDiscardRegsOnJRRA) { - // According to the MIPS ABI, there are some regs we don't need to preserve. - // Let's discard them so we don't need to write them back. - // NOTE: Not all games follow the MIPS ABI! Tekken 6, for example, will crash - // with this enabled. - gpr.DiscardR(MIPS_REG_COMPILER_SCRATCH); - for (int i = MIPS_REG_A0; i <= MIPS_REG_T7; i++) - gpr.DiscardR((MIPSGPReg)i); - gpr.DiscardR(MIPS_REG_T8); - gpr.DiscardR(MIPS_REG_T9); - } - - gpr.MapReg(rs); - destReg = gpr.R(rs); // Safe because FlushAll doesn't change any regs - FlushAll(); - } else { - // Since we can't be in a delay slot, should be safe to steal FLAGTEMPREG for a temp reg. - // It will be saved, even if a function is called. - destReg = DecodeReg(FLAGTEMPREG); - gpr.MapReg(rs); - MOV(destReg, gpr.R(rs)); - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - } - - switch (op & 0x3f) - { - case 8: //jr - break; - case 9: //jalr - break; - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } - - WriteExitDestInR(destReg); - js.compiling = false; -} - -void Arm64Jit::Comp_Syscall(MIPSOpcode op) -{ - if (op.encoding == 0x03FFFFcc) { - WARN_LOG(Log::JIT, "Encountered bad syscall instruction at %08x (%08x)", js.compilerPC, op.encoding); - } - if (!g_Config.bSkipDeadbeefFilling) - { - // All of these will be overwritten with DEADBEEF anyway. - gpr.DiscardR(MIPS_REG_COMPILER_SCRATCH); - // We need to keep A0 - T3, which are used for args. - gpr.DiscardR(MIPS_REG_T4); - gpr.DiscardR(MIPS_REG_T5); - gpr.DiscardR(MIPS_REG_T6); - gpr.DiscardR(MIPS_REG_T7); - gpr.DiscardR(MIPS_REG_T8); - gpr.DiscardR(MIPS_REG_T9); - - gpr.DiscardR(MIPS_REG_HI); - gpr.DiscardR(MIPS_REG_LO); - } - - // If we're in a delay slot, this is off by one. - const int offset = js.inDelaySlot ? -1 : 0; - WriteDownCount(offset, false); - RestoreRoundingMode(); - js.downcountAmount = -offset; - - if (!js.inDelaySlot) { - MOVI2R(SCRATCH1, GetCompilerPC() + 4); - MovToPC(SCRATCH1); - } - - FlushAll(); - - SaveStaticRegisters(); -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - MOVI2R(W0, op.encoding); - QuickCallFunction(X1, (void *)&CallSyscall); -#else - // Skip the CallSyscall where possible. - const HLEFunction *func = GetSyscallFunctionData(op, js.compilerPC); - if (func) { - void *quickFunc = GetQuickSyscallFunc(func, op); - if (quickFunc) { - MOVI2R(X0, (uintptr_t)func); - // Already flushed, so X1 is safe. - QuickCallFunction(X1, quickFunc); - } else { - MOVI2R(W0, op.encoding); - QuickCallFunction(X1, (void *)&CallSyscall); - } - } else { - MOVI2R(W0, js.compilerPC); - QuickCallFunction(X1, (void *)&CallSyscallUnresolvedAtPC); - } -#endif - LoadStaticRegisters(); - ApplyRoundingMode(); - - WriteSyscallExit(); - js.compiling = false; -} - -void Arm64Jit::Comp_Break(MIPSOpcode op) -{ - Comp_Generic(op); - WriteSyscallExit(); - js.compiling = false; -} - -} // namespace Mipscomp - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompFPU.cpp b/Core/MIPS/ARM64/Arm64CompFPU.cpp deleted file mode 100644 index b2bc9083cf..0000000000 --- a/Core/MIPS/ARM64/Arm64CompFPU.cpp +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include "Core/Config.h" -#include "Core/MemMap.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/MIPSTables.h" - -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Common/CPUDetect.h" - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - - -// FPCR interesting bits: -// 24: FZ (flush-to-zero) -// 23:22: RMode (0 = nearest, 1 = +inf, 2 = -inf, 3 = zero) -// not much else is interesting for us, but should be preserved. -// To access: MRS Xt, FPCR ; MSR FPCR, Xt - - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE(flag) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } - -namespace MIPSComp { - using namespace Arm64Gen; - using namespace Arm64JitConstants; - -void Arm64Jit::Comp_FPU3op(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU); - - int ft = _FT; - int fs = _FS; - int fd = _FD; - - fpr.MapDirtyInIn(fd, fs, ft); - switch (op & 0x3f) { - case 0: fp.FADD(fpr.R(fd), fpr.R(fs), fpr.R(ft)); break; //F(fd) = F(fs) + F(ft); //add - case 1: fp.FSUB(fpr.R(fd), fpr.R(fs), fpr.R(ft)); break; //F(fd) = F(fs) - F(ft); //sub - case 2: fp.FMUL(fpr.R(fd), fpr.R(fs), fpr.R(ft)); break; //F(fd) = F(fs) * F(ft); //mul - case 3: fp.FDIV(fpr.R(fd), fpr.R(fs), fpr.R(ft)); break; //F(fd) = F(fs) / F(ft); //div - default: - DISABLE; - return; - } -} - -void Arm64Jit::Comp_FPULS(MIPSOpcode op) -{ - CONDITIONAL_DISABLE(LSU_FPU); - CheckMemoryBreakpoint(); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - s32 offset = SignExtend16ToS32(op & 0xFFFF); - int ft = _FT; - MIPSGPReg rs = _RS; - std::vector skips; - switch (op >> 26) { - case 49: // lwc1 - if (!gpr.IsImm(rs) && jo.cachePointers && g_Config.bFastMemory && (offset & 3) == 0 && offset <= 16380 && offset >= 0) { - gpr.MapRegAsPointer(rs); - fpr.MapReg(ft, MAP_NOINIT | MAP_DIRTY); - fp.LDR(32, INDEX_UNSIGNED, fpr.R(ft), gpr.RPtr(rs), offset); - break; - } - - fpr.SpillLock(ft); - fpr.MapReg(ft, MAP_NOINIT | MAP_DIRTY); - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (offset + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = offset + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1, addr); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, offset); - } else { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } - } - fp.LDR(32, fpr.R(ft), SCRATCH1_64, ArithOption(MEMBASEREG)); - for (auto skip : skips) { - SetJumpTarget(skip); - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - break; - - case 57: // swc1 - if (!gpr.IsImm(rs) && jo.cachePointers && g_Config.bFastMemory && (offset & 3) == 0 && offset <= 16380 && offset >= 0) { - gpr.MapRegAsPointer(rs); - fpr.MapReg(ft, 0); - fp.STR(32, INDEX_UNSIGNED, fpr.R(ft), gpr.RPtr(rs), offset); - break; - } - - fpr.MapReg(ft); - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (offset + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = offset + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1, addr); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, offset); - } else { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } - } - fp.STR(32, fpr.R(ft), SCRATCH1_64, ArithOption(MEMBASEREG)); - for (auto skip : skips) { - SetJumpTarget(skip); - } - break; - - default: - Comp_Generic(op); - return; - } -} - -void Arm64Jit::Comp_FPUComp(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU_COMP); - - int opc = op & 0xF; - if (opc >= 8) opc -= 8; // alias - if (opc == 0) { // f, sf (signalling false) - gpr.SetImm(MIPS_REG_FPCOND, 0); - return; - } - - int fs = _FS; - int ft = _FT; - gpr.MapReg(MIPS_REG_FPCOND, MAP_DIRTY | MAP_NOINIT); - fpr.MapInIn(fs, ft); - fp.FCMP(fpr.R(fs), fpr.R(ft)); - - switch (opc) { - case 1: // un, ngle (unordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_VS); - break; - case 2: // eq, seq (equal, ordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_EQ); - break; - case 3: // ueq, ngl (equal, unordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_EQ); - // If ordered, use the above result. If unordered, use ZR+1 (being 1.) - CSINC(gpr.R(MIPS_REG_FPCOND), gpr.R(MIPS_REG_FPCOND), WZR, CC_VC); - return; - case 4: // olt, lt (less than, ordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_LO); - break; - case 5: // ult, nge (less than, unordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_LT); - break; - case 6: // ole, le (less equal, ordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_LS); - break; - case 7: // ule, ngt (less equal, unordered) - CSET(gpr.R(MIPS_REG_FPCOND), CC_LE); - break; - default: - Comp_Generic(op); - return; - } -} - -void Arm64Jit::Comp_FPU2op(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU); - int fs = _FS; - int fd = _FD; - - switch (op & 0x3f) { - case 4: //F(fd) = sqrtf(F(fs)); break; //sqrt - fpr.MapDirtyIn(fd, fs); - fp.FSQRT(fpr.R(fd), fpr.R(fs)); - break; - case 5: //F(fd) = fabsf(F(fs)); break; //abs - fpr.MapDirtyIn(fd, fs); - fp.FABS(fpr.R(fd), fpr.R(fs)); - break; - case 6: //F(fd) = F(fs); break; //mov - fpr.MapDirtyIn(fd, fs); - fp.FMOV(fpr.R(fd), fpr.R(fs)); - break; - case 7: //F(fd) = -F(fs); break; //neg - fpr.MapDirtyIn(fd, fs); - fp.FNEG(fpr.R(fd), fpr.R(fs)); - break; - - case 12: //FsI(fd) = (int)floorf(F(fs)+0.5f); break; //round.w.s - { - fpr.MapDirtyIn(fd, fs); - fp.FCMP(fpr.R(fs), fpr.R(fs)); // Detect NaN - fp.FCVTS(fpr.R(fd), fpr.R(fs), ROUND_N); // to nearest, ties to even - FixupBranch skip = B(CC_VC); - MOVI2R(SCRATCH1, 0x7FFFFFFF); - fp.FMOV(fpr.R(fd), SCRATCH1); - SetJumpTarget(skip); - break; - } - - case 13: //FsI(fd) = Rto0(F(fs))); break; //trunc.w.s - { - fpr.MapDirtyIn(fd, fs); - fp.FCMP(fpr.R(fs), fpr.R(fs)); - fp.FCVTS(fpr.R(fd), fpr.R(fs), ROUND_Z); - FixupBranch skip = B(CC_VC); - MOVI2R(SCRATCH1, 0x7FFFFFFF); - fp.FMOV(fpr.R(fd), SCRATCH1); - SetJumpTarget(skip); - break; - } - - case 14://FsI(fd) = (int)ceilf (F(fs)); break; //ceil.w.s - { - fpr.MapDirtyIn(fd, fs); - fp.FCMP(fpr.R(fs), fpr.R(fs)); - fp.FCVTS(fpr.R(fd), fpr.R(fs), ROUND_P); // towards +inf - FixupBranch skip = B(CC_VC); - MOVI2R(SCRATCH1, 0x7FFFFFFF); - fp.FMOV(fpr.R(fd), SCRATCH1); - SetJumpTarget(skip); - break; - } - case 15: //FsI(fd) = (int)floorf(F(fs)); break; //floor.w.s - { - fpr.MapDirtyIn(fd, fs); - fp.FCMP(fpr.R(fs), fpr.R(fs)); - fp.FCVTS(fpr.R(fd), fpr.R(fs), ROUND_M); // towards -inf - FixupBranch skip = B(CC_VC); - MOVI2R(SCRATCH1, 0x7FFFFFFF); - fp.FMOV(fpr.R(fd), SCRATCH1); - SetJumpTarget(skip); - break; - } - - case 32: //F(fd) = (float)FsI(fs); break; //cvt.s.w - fpr.MapDirtyIn(fd, fs); - fp.SCVTF(fpr.R(fd), fpr.R(fs)); - break; - - case 36: //FsI(fd) = (int) F(fs); break; //cvt.w.s - fpr.MapDirtyIn(fd, fs); - if (js.hasSetRounding) { - // We're just going to defer to our cached func. Here's the arg. - fp.FMOV(S0, fpr.R(fs)); - - MOVP2R(SCRATCH1_64, &js.currentRoundingFunc); - LDR(INDEX_UNSIGNED, SCRATCH1_64, SCRATCH1_64, 0); - - BLR(SCRATCH1_64); - - fp.FMOV(fpr.R(fd), S0); - } else { - fp.FCMP(fpr.R(fs), fpr.R(fs)); - fp.FCVTS(fpr.R(fd), fpr.R(fs), ROUND_N); - FixupBranch skip_nan = B(CC_VC); - MOVI2R(SCRATCH1, 0x7FFFFFFF); - fp.FMOV(fpr.R(fd), SCRATCH1); - SetJumpTarget(skip_nan); - } - break; - - default: - DISABLE; - } -} - -void Arm64Jit::Comp_mxc1(MIPSOpcode op) -{ - CONDITIONAL_DISABLE(FPU_XFER); - - int fs = _FS; - MIPSGPReg rt = _RT; - - switch ((op >> 21) & 0x1f) { - case 0: // R(rt) = FI(fs); break; //mfc1 - if (rt == MIPS_REG_ZERO) { - return; - } - gpr.MapReg(rt, MAP_DIRTY | MAP_NOINIT); - if (fpr.IsMapped(fs)) { - fp.FMOV(gpr.R(rt), fpr.R(fs)); - } else { - LDR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, fpr.GetMipsRegOffset(fs)); - } - return; - - case 2: //cfc1 - if (rt == MIPS_REG_ZERO) { - return; - } - if (fs == 31) { - if (gpr.IsImm(MIPS_REG_FPCOND)) { - gpr.MapReg(rt, MAP_DIRTY | MAP_NOINIT); - LDR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, offsetof(MIPSState, fcr31)); - if (gpr.GetImm(MIPS_REG_FPCOND) & 1) { - ORRI2R(gpr.R(rt), gpr.R(rt), 0x1 << 23, SCRATCH2); - } else { - ANDI2R(gpr.R(rt), gpr.R(rt), ~(0x1 << 23), SCRATCH2); - } - } else { - gpr.MapDirtyIn(rt, MIPS_REG_FPCOND); - LDR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, offsetof(MIPSState, fcr31)); - BFI(gpr.R(rt), gpr.R(MIPS_REG_FPCOND), 23, 1); - } - } else if (fs == 0) { - gpr.SetImm(rt, MIPSState::FCR0_VALUE); - } else { - // Unsupported regs are always 0. - gpr.SetImm(rt, 0); - } - return; - - case 4: //FI(fs) = R(rt); break; //mtc1 - if (gpr.IsImm(rt)) { - // This can't be run on LO/HI. - uint32_t ival = (uint32_t)gpr.GetImm(rt); - float floatval; - memcpy(&floatval, &ival, sizeof(floatval)); - uint8_t imm8; - // If zero, just zero it. - fpr.MapReg(fs, MAP_NOINIT | MAP_DIRTY); - if (ival == 0) { - fp.FMOV(fpr.R(fs), WZR); // This is supposedly special cased in hardware to be fast. - } else if (FPImm8FromFloat(floatval, &imm8)) { - fp.FMOV(fpr.R(fs), imm8); - } else { - // Materialize the register and do a cross move. - gpr.MapReg(rt); - fp.FMOV(fpr.R(fs), gpr.R(rt)); - } - } else { - gpr.MapReg(rt); - fpr.MapReg(fs, MAP_NOINIT | MAP_DIRTY); - fp.FMOV(fpr.R(fs), gpr.R(rt)); - } - return; - - case 6: //ctc1 - if (fs == 31) { - // Must clear before setting, since ApplyRoundingMode() assumes it was cleared. - RestoreRoundingMode(); - bool wasImm = gpr.IsImm(rt); - u32 immVal = -1; - if (wasImm) { - immVal = gpr.GetImm(rt); - gpr.SetImm(MIPS_REG_FPCOND, (immVal >> 23) & 1); - gpr.MapReg(rt); - } else { - gpr.MapDirtyIn(MIPS_REG_FPCOND, rt); - } - - // Update MIPS state. Only these bits can be written (pspautotests cpu/fpu/fcr). - ANDI2R(SCRATCH1, gpr.R(rt), 0x0181FFFF, SCRATCH2); - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, fcr31)); - if (!wasImm) { - UBFX(gpr.R(MIPS_REG_FPCOND), gpr.R(rt), 23, 1); - // TODO: We do have the fcr31 value in a register here, could use that in UpdateRoundingMode to avoid reloading it. - UpdateRoundingMode(); - } else { - UpdateRoundingMode(immVal); - } - ApplyRoundingMode(); - } else { - Comp_Generic(op); - } - return; - default: - DISABLE; - break; - } -} - -} // namespace MIPSComp - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompLoadStore.cpp b/Core/MIPS/ARM64/Arm64CompLoadStore.cpp deleted file mode 100644 index f293b94329..0000000000 --- a/Core/MIPS/ARM64/Arm64CompLoadStore.cpp +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include "Core/MemMap.h" -#include "Core/Config.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -//#define CONDITIONAL_DISABLE(flag) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } - -namespace MIPSComp { - using namespace Arm64Gen; - using namespace Arm64JitConstants; - - // Destroys SCRATCH2 - void Arm64Jit::SetScratch1ToEffectiveAddress(MIPSGPReg rs, s16 offset) { - if (offset) { - ADDI2R(SCRATCH1, gpr.R(rs), offset, SCRATCH2); - } else { - MOV(SCRATCH1, gpr.R(rs)); - } -#ifdef MASKED_PSP_MEMORY - ANDI2R(SCRATCH1, SCRATCH1, 0x3FFFFFFF); -#endif - } - - std::vector Arm64Jit::SetScratch1ForSafeAddress(MIPSGPReg rs, s16 offset, ARM64Reg tempReg) { - std::vector skips; - - SetScratch1ToEffectiveAddress(rs, offset); - - // We can do this a little smarter by shifting out the lower 8 bits, since blocks are 0x100 aligned. - // PSP_GetUserMemoryEnd() is dynamic, but the others encode to imms just fine. - // So we only need to safety check the one value. - // This is because ARM64 immediates for many instructions like CMP can only encode - // immediates up to 12 bits, shifted by 12 or not. - - if ((PSP_GetUserMemoryEnd() & 0x000FFFFF) == 0) { - // In other words, shift right 8, and kill off the top 4 bits as we don't want them involved in the ocmpares. - UBFX(tempReg, SCRATCH1, 8, 24 - 4); - // Now check if we're higher than that. - CMPI2R(tempReg, PSP_GetUserMemoryEnd() >> 8); - } else { - // Compare first using the tempReg (need it because we have a full 28-bit value), then shift into it. - ANDI2R(SCRATCH1, SCRATCH1, 0x0FFFFFFF); - CMPI2R(SCRATCH1, PSP_GetUserMemoryEnd(), tempReg); - UBFX(tempReg, SCRATCH1, 8, 24); - } - skips.push_back(B(CC_HS)); - - // If its higher than memory start and we didn't skip yet, it must be good. Hurray. - CMPI2R(tempReg, PSP_GetKernelMemoryBase() >> 8); - FixupBranch inRAM = B(CC_HS); - - // If we got here and it's higher, then it's between VRAM and RAM - skip. - CMPI2R(tempReg, PSP_GetVidMemEnd() >> 8); - skips.push_back(B(CC_HS)); - - // And if it's higher the VRAM and we're still here again, it's in VRAM. - CMPI2R(tempReg, PSP_GetVidMemBase() >> 8); - FixupBranch inVRAM = B(CC_HS); - - // Last gap, this is between SRAM and VRAM. Skip it. - CMPI2R(tempReg, PSP_GetScratchpadMemoryEnd() >> 8); - skips.push_back(B(CC_HS)); - - // And for lower than SRAM, we just skip again. - CMPI2R(tempReg, PSP_GetScratchpadMemoryBase() >> 8); - skips.push_back(B(CC_LO)); - - // At this point, we're either in SRAM (above) or in RAM/VRAM. - SetJumpTarget(inRAM); - SetJumpTarget(inVRAM); - - return skips; - } - - void Arm64Jit::Comp_ITypeMemLR(MIPSOpcode op, bool load) { - CONDITIONAL_DISABLE(LSU); - CheckMemoryBreakpoint(); - int offset = SignExtend16ToS32(op & 0xFFFF); - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - int o = op >> 26; - - if (!js.inDelaySlot && !jo.Disabled(JitDisable::LSU_UNALIGNED)) { - // Optimisation: Combine to single unaligned load/store - bool isLeft = (o == 34 || o == 42); - CheckMemoryBreakpoint(1); - MIPSOpcode nextOp = GetOffsetInstruction(1); - // Find a matching shift in opposite direction with opposite offset. - if (nextOp == (isLeft ? (op.encoding + (4 << 26) - 3) : (op.encoding - (4 << 26) + 3))) { - EatInstruction(nextOp); - nextOp = MIPSOpcode(((load ? 35 : 43) << 26) | ((isLeft ? nextOp : op) & 0x03FFFFFF)); //lw, sw - Comp_ITypeMem(nextOp); - return; - } - } - - u32 iaddr = gpr.IsImm(rs) ? offset + gpr.GetImm(rs) : 0xFFFFFFFF; - std::vector skips; - - if (gpr.IsImm(rs) && Memory::IsValidAddress(iaddr)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = iaddr & 0x3FFFFFFF; -#else - u32 addr = iaddr; -#endif - // Need to initialize since this only loads part of the register. - // But rs no longer matters (even if rs == rt) since we have the address. - gpr.MapReg(rt, load ? MAP_DIRTY : 0); - MOVI2R(SCRATCH1, addr & ~3); - - u8 shift = (addr & 3) * 8; - - switch (o) { - case 34: // lwl - LDR(SCRATCH1, MEMBASEREG, SCRATCH1); - ANDI2R(gpr.R(rt), gpr.R(rt), 0x00ffffff >> shift, INVALID_REG); - ORR(gpr.R(rt), gpr.R(rt), SCRATCH1, ArithOption(gpr.R(rt), ST_LSL, 24 - shift)); - break; - - case 38: // lwr - LDR(SCRATCH1, MEMBASEREG, SCRATCH1); - ANDI2R(gpr.R(rt), gpr.R(rt), 0xffffff00 << (24 - shift), INVALID_REG); - ORR(gpr.R(rt), gpr.R(rt), SCRATCH1, ArithOption(gpr.R(rt), ST_LSR, shift)); - break; - - case 42: // swl - LDR(SCRATCH2, MEMBASEREG, SCRATCH1); - ANDI2R(SCRATCH2, SCRATCH2, 0xffffff00 << shift, INVALID_REG); - ORR(SCRATCH2, SCRATCH2, gpr.R(rt), ArithOption(gpr.R(rt), ST_LSR, 24 - shift)); - STR(SCRATCH2, MEMBASEREG, SCRATCH1); - break; - - case 46: // swr - LDR(SCRATCH2, MEMBASEREG, SCRATCH1); - ANDI2R(SCRATCH2, SCRATCH2, 0x00ffffff >> (24 - shift), INVALID_REG); - ORR(SCRATCH2, SCRATCH2, gpr.R(rt), ArithOption(gpr.R(rt), ST_LSL, shift)); - STR(SCRATCH2, MEMBASEREG, SCRATCH1); - break; - } - return; - } - - _dbg_assert_msg_(!gpr.IsImm(rs), "Invalid immediate address %08x? CPU bug?", iaddr); - if (load) { - gpr.MapDirtyIn(rt, rs, false); - } else { - gpr.MapInIn(rt, rs); - } - gpr.SpillLock(rt); - gpr.SpillLock(rs); - // Need to get temps before skipping safe mem. - ARM64Reg LR_SCRATCH3 = gpr.GetAndLockTempR(); - ARM64Reg LR_SCRATCH4 = o == 42 || o == 46 ? gpr.GetAndLockTempR() : INVALID_REG; - - if (!g_Config.bFastMemory && rs != MIPS_REG_SP) { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } else { - SetScratch1ToEffectiveAddress(rs, offset); - } - - // Here's our shift amount. - ANDI2R(SCRATCH2, SCRATCH1, 3); - LSL(SCRATCH2, SCRATCH2, 3); - - // Now align the address for the actual read. - ANDI2R(SCRATCH1, SCRATCH1, ~3U); - - switch (o) { - case 34: // lwl - MOVI2R(LR_SCRATCH3, 0x00ffffff); - LDR(SCRATCH1, MEMBASEREG, ArithOption(SCRATCH1)); - LSRV(LR_SCRATCH3, LR_SCRATCH3, SCRATCH2); - AND(gpr.R(rt), gpr.R(rt), LR_SCRATCH3); - NEG(SCRATCH2, SCRATCH2); - ADDI2R(SCRATCH2, SCRATCH2, 24); - LSLV(SCRATCH1, SCRATCH1, SCRATCH2); - ORR(gpr.R(rt), gpr.R(rt), SCRATCH1); - break; - - case 38: // lwr - MOVI2R(LR_SCRATCH3, 0xffffff00); - LDR(SCRATCH1, MEMBASEREG, ArithOption(SCRATCH1)); - LSRV(SCRATCH1, SCRATCH1, SCRATCH2); - NEG(SCRATCH2, SCRATCH2); - ADDI2R(SCRATCH2, SCRATCH2, 24); - LSLV(LR_SCRATCH3, LR_SCRATCH3, SCRATCH2); - AND(gpr.R(rt), gpr.R(rt), LR_SCRATCH3); - ORR(gpr.R(rt), gpr.R(rt), SCRATCH1); - break; - - case 42: // swl - MOVI2R(LR_SCRATCH3, 0xffffff00); - LDR(LR_SCRATCH4, MEMBASEREG, ArithOption(SCRATCH1)); - LSLV(LR_SCRATCH3, LR_SCRATCH3, SCRATCH2); - AND(LR_SCRATCH4, LR_SCRATCH4, LR_SCRATCH3); - NEG(SCRATCH2, SCRATCH2); - ADDI2R(SCRATCH2, SCRATCH2, 24); - - LSRV(LR_SCRATCH3, gpr.R(rt), SCRATCH2); - ORR(LR_SCRATCH4, LR_SCRATCH4, LR_SCRATCH3); - STR(LR_SCRATCH4, MEMBASEREG, ArithOption(SCRATCH1)); - break; - - case 46: // swr - MOVI2R(LR_SCRATCH3, 0x00ffffff); - LDR(LR_SCRATCH4, MEMBASEREG, ArithOption(SCRATCH1)); - NEG(SCRATCH2, SCRATCH2); - ADDI2R(SCRATCH2, SCRATCH2, 24); - LSRV(LR_SCRATCH3, LR_SCRATCH3, SCRATCH2); - AND(LR_SCRATCH4, LR_SCRATCH4, LR_SCRATCH3); - NEG(SCRATCH2, SCRATCH2); - ADDI2R(SCRATCH2, SCRATCH2, 24); - LSLV(LR_SCRATCH3, gpr.R(rt), SCRATCH2); - ORR(LR_SCRATCH4, LR_SCRATCH4, LR_SCRATCH3); - STR(LR_SCRATCH4, MEMBASEREG, ArithOption(SCRATCH1)); - break; - } - - for (auto skip : skips) { - SetJumpTarget(skip); - } - - gpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_ITypeMem(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU); - CheckMemoryBreakpoint(); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - int offset = SignExtend16ToS32(op & 0xFFFF); - bool load = false; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - int o = op >> 26; - // 16-bit accesses can go through the depth layout (Memory::DepthMirrored16). - if (Memory::DepthMirrorsActive() && (o == 33 || o == 37 || o == 41)) { - DISABLE; - } - if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) { - // Don't load anything into $zr - return; - } - - u32 iaddr = gpr.IsImm(rs) ? offset + gpr.GetImm(rs) : 0xFFFFFFFF; - std::vector skips; - ARM64Reg targetReg = INVALID_REG; - ARM64Reg addrReg = INVALID_REG; - - int dataSize = 4; - switch (o) { - case 37: - case 33: - dataSize = 2; - break; - case 36: - case 32: - dataSize = 1; - break; - // Store - case 41: - dataSize = 2; - break; - case 40: - dataSize = 1; - break; - } - - switch (o) { - case 32: //lb - case 33: //lh - case 35: //lw - case 36: //lbu - case 37: //lhu - load = true; - case 40: //sb - case 41: //sh - case 43: //sw -#ifndef MASKED_PSP_MEMORY - if (jo.cachePointers && g_Config.bFastMemory) { - // ARM has smaller load/store immediate displacements than MIPS, 12 bits - and some memory ops only have 8 bits. - int offsetRange = 0x3ff; - if (o == 41 || o == 33 || o == 37 || o == 32) - offsetRange = 0xff; // 8 bit offset only - if (!gpr.IsImm(rs) && rs != rt && (offset <= offsetRange) && offset >= 0 && - (dataSize == 1 || (offset & (dataSize - 1)) == 0)) { // Check that the offset is aligned to the access size as that's required for INDEX_UNSIGNED encodings. we can get here through fallback from lwl/lwr - gpr.SpillLock(rs, rt); - gpr.MapRegAsPointer(rs); - - // For a store, try to avoid mapping a reg if not needed. - targetReg = load ? INVALID_REG : gpr.TryMapTempImm(rt); - if (targetReg == INVALID_REG) { - gpr.MapReg(rt, load ? MAP_NOINIT : 0); - targetReg = gpr.R(rt); - } - - switch (o) { - case 35: LDR(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 37: LDRH(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 33: LDRSH(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 36: LDRB(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 32: LDRSB(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - // Store - case 43: STR(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 41: STRH(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - case 40: STRB(INDEX_UNSIGNED, targetReg, gpr.RPtr(rs), offset); break; - } - gpr.ReleaseSpillLocksAndDiscardTemps(); - break; - } - } -#endif - - if (!load && gpr.IsImm(rt) && gpr.TryMapTempImm(rt) != INVALID_REG) { - // We're storing an immediate value, let's see if we can optimize rt. - if (!gpr.IsImm(rs) || !Memory::IsValidAddress(iaddr) || offset == 0) { - // In this case, we're always going to need rs mapped, which may flush the temp imm. - // We handle that in the cases below since targetReg is INVALID_REG. - gpr.MapIn(rs); - } - - targetReg = gpr.TryMapTempImm(rt); - } - - if (gpr.IsImm(rs) && Memory::IsValidAddress(iaddr)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = iaddr & 0x3FFFFFFF; -#else - u32 addr = iaddr; -#endif - if (addr == iaddr && offset == 0) { - // It was already safe. Let's shove it into a reg and use it directly. - if (targetReg == INVALID_REG) { - load ? gpr.MapDirtyIn(rt, rs) : gpr.MapInIn(rt, rs); - targetReg = gpr.R(rt); - } - addrReg = gpr.R(rs); - } else { - // In this case, only map rt. rs+offset will be in SCRATCH1. - if (targetReg == INVALID_REG) { - gpr.MapReg(rt, load ? MAP_NOINIT : 0); - targetReg = gpr.R(rt); - } - MOVI2R(SCRATCH1, addr); - addrReg = SCRATCH1; - } - } else { - // This gets hit in a few games, as a result of never-taken delay slots (some branch types - // conditionally execute the delay slot instructions). Ignore in those cases. - if (!js.inDelaySlot) { - _dbg_assert_msg_(!gpr.IsImm(rs), "Invalid immediate address %08x? CPU bug?", iaddr); - } - - // If we already have a targetReg, we optimized an imm, and rs is already mapped. - if (targetReg == INVALID_REG) { - if (load) { - gpr.MapDirtyIn(rt, rs); - } else { - gpr.MapInIn(rt, rs); - } - targetReg = gpr.R(rt); - } - - if (!g_Config.bFastMemory && rs != MIPS_REG_SP) { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } else { - SetScratch1ToEffectiveAddress(rs, offset); - } - addrReg = SCRATCH1; - } - - switch (o) { - // Load - case 35: LDR(targetReg, MEMBASEREG, addrReg); break; - case 37: LDRH(targetReg, MEMBASEREG, addrReg); break; - case 33: LDRSH(targetReg, MEMBASEREG, addrReg); break; - case 36: LDRB(targetReg, MEMBASEREG, addrReg); break; - case 32: LDRSB(targetReg, MEMBASEREG, addrReg); break; - // Store - case 43: STR(targetReg, MEMBASEREG, addrReg); break; - case 41: STRH(targetReg, MEMBASEREG, addrReg); break; - case 40: STRB(targetReg, MEMBASEREG, addrReg); break; - } - for (auto skip : skips) { - SetJumpTarget(skip); - // TODO: Could clear to zero here on load, if skipping this for good reads. - } - break; - case 34: //lwl - case 38: //lwr - load = true; - case 42: //swl - case 46: //swr - Comp_ITypeMemLR(op, load); - break; - default: - Comp_Generic(op); - return; - } - } - - void Arm64Jit::Comp_StoreSync(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU); - CheckMemoryBreakpoint(); - - // ll loads a word and sets llBit. sc stores a word only if llBit is set, and sets rt to - // whether it did. Only the fast memory path, like the plain loads and stores mostly take. - if (js.kernelMode || !g_Config.bFastMemory) { - DISABLE; - } - - int offset = SignExtend16ToS32(op & 0xFFFF); - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - switch (op >> 26) { - case 48: // ll - if (rt != MIPS_REG_ZERO) { - gpr.MapDirtyIn(rt, rs); - SetScratch1ToEffectiveAddress(rs, offset); - LDR(gpr.R(rt), MEMBASEREG, SCRATCH1); - } - MOVI2R(SCRATCH2, 1); - STR(INDEX_UNSIGNED, SCRATCH2, CTXREG, offsetof(MIPSState, llBit)); - break; - - case 56: // sc - { - if (rt == MIPS_REG_ZERO) { - DISABLE; - } - // rt is both the value to store and the result. - gpr.MapDirtyIn(rt, rs, false); - SetScratch1ToEffectiveAddress(rs, offset); - LDR(INDEX_UNSIGNED, SCRATCH2, CTXREG, offsetof(MIPSState, llBit)); - FixupBranch failed = CBZ(SCRATCH2); - STR(gpr.R(rt), MEMBASEREG, SCRATCH1); - MOVI2R(gpr.R(rt), 1); - FixupBranch done = B(); - SetJumpTarget(failed); - MOVI2R(gpr.R(rt), 0); - SetJumpTarget(done); - break; - } - - default: - DISABLE; - } - } - - void Arm64Jit::Comp_Cache(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU); - - int func = (op >> 16) & 0x1F; - - // See Int_Cache for the definitions. - switch (func) { - case 24: break; - case 25: break; - case 27: break; - case 30: break; - default: - // Fall back to the interpreter. - DISABLE; - } - } -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompReplace.cpp b/Core/MIPS/ARM64/Arm64CompReplace.cpp deleted file mode 100644 index 937e2f5b67..0000000000 --- a/Core/MIPS/ARM64/Arm64CompReplace.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2013- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include "Common/CPUDetect.h" -#include "Core/MemMap.h" -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" - -namespace MIPSComp { - -int Arm64Jit::Replace_fabsf() { - fpr.MapDirtyIn(0, 12); - fp.FABS(fpr.R(0), fpr.R(12)); - return 4; // Number of instructions in the MIPS function -} - -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64CompVFPU.cpp b/Core/MIPS/ARM64/Arm64CompVFPU.cpp deleted file mode 100644 index a66d1e1532..0000000000 --- a/Core/MIPS/ARM64/Arm64CompVFPU.cpp +++ /dev/null @@ -1,2468 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include -#include -#include "Common/Arm64Emitter.h" -#include "Common/CPUDetect.h" -#include "Common/Data/Convert/SmallDataConvert.h" -#include "Common/Math/math_util.h" - -#include "Core/Compatibility.h" -#include "Core/Config.h" -#include "Core/MemMap.h" -#include "Core/Reporting.h" -#include "Core/System.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSTables.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE(flag) { fpr.ReleaseSpillLocksAndDiscardTemps(); Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { fpr.ReleaseSpillLocksAndDiscardTemps(); Comp_Generic(op); return; } - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -namespace MIPSComp { - using namespace Arm64Gen; - using namespace Arm64JitConstants; - - // Vector regs can overlap in all sorts of swizzled ways. - // This does allow a single overlap in sregs[i]. - // True if the prefix only touches lanes the op has. A position past the size may only be the - // identity, and a position within it may not name a lane past it (which zeroes the result lane - // on hardware, see cpu/vfpu/prefix_ctrl - the interpreter handles that). - static bool IsPrefixWithinSize(u32 prefix, VectorSize sz) { - int n = GetNumVectorElements(sz); - for (int i = 0; i < 4; i++) { - int regnum = (prefix >> (i * 2)) & 3; - int abs = (prefix >> (8 + i)) & 1; - int negate = (prefix >> (16 + i)) & 1; - int constants = (prefix >> (12 + i)) & 1; - if (constants) { - continue; - } - if (i >= n) { - if (abs || negate || regnum != i) - return false; - } else if (regnum >= n) { - return false; - } - } - - return true; - } - - static bool IsPrefixWithinSize(u32 prefix, MIPSOpcode op) { - return IsPrefixWithinSize(prefix, GetVecSize(op)); - } - - static bool IsOverlapSafeAllowS(int dreg, int di, int sn, u8 sregs[], int tn = 0, u8 tregs[] = NULL) - { - for (int i = 0; i < sn; ++i) { - if (sregs[i] == dreg && i != di) - return false; - } - for (int i = 0; i < tn; ++i) { - if (tregs[i] == dreg) - return false; - } - - // Hurray, no overlap, we can write directly. - return true; - } - - static bool IsOverlapSafe(int dreg, int di, int sn, u8 sregs[], int tn = 0, u8 tregs[] = NULL) - { - return IsOverlapSafeAllowS(dreg, di, sn, sregs, tn, tregs) && sregs[di] != dreg; - } - - void Arm64Jit::Comp_VPFX(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - int data = op & 0xFFFFF; - int regnum = (op >> 24) & 3; - switch (regnum) { - case 0: // S - js.prefixS = data; - js.prefixSFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - case 1: // T - js.prefixT = data; - js.prefixTFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - case 2: // D - js.prefixD = data & 0x00000FFF; - js.prefixDFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - default: - ERROR_LOG(Log::CPU, "VPFX - bad regnum %i : data=%08x", regnum, data); - break; - } - } - - void Arm64Jit::ApplyPrefixST(u8 *vregs, u32 prefix, VectorSize sz) { - if (prefix == 0xE4) - return; - - int n = GetNumVectorElements(sz); - u8 origV[4]; - static const float constantArray[8] = { 0.f, 1.f, 2.f, 0.5f, 3.f, 1.f / 3.f, 0.25f, 1.f / 6.f }; - - for (int i = 0; i < n; i++) - origV[i] = vregs[i]; - - for (int i = 0; i < n; i++) { - int regnum = (prefix >> (i * 2)) & 3; - int abs = (prefix >> (8 + i)) & 1; - int negate = (prefix >> (16 + i)) & 1; - int constants = (prefix >> (12 + i)) & 1; - - // Unchanged, hurray. - if (!constants && regnum == i && !abs && !negate) - continue; - - // This puts the value into a temp reg, so we won't write the modified value back. - vregs[i] = fpr.GetTempV(); - if (!constants) { - fpr.MapDirtyInV(vregs[i], origV[regnum]); - fpr.SpillLockV(vregs[i]); - - // Prefix may say "z, z, z, z" but if this is a pair, we force to x. - // TODO: But some ops seem to use const 0 instead? - if (regnum >= n) { - WARN_LOG(Log::CPU, "JIT: Invalid VFPU swizzle: %08x : %d / %d at PC = %08x (%s)", prefix, regnum, n, GetCompilerPC(), MIPSDisasmAt(GetCompilerPC()).c_str()); - regnum = 0; - } - - if (abs) { - fp.FABS(fpr.V(vregs[i]), fpr.V(origV[regnum])); - if (negate) - fp.FNEG(fpr.V(vregs[i]), fpr.V(vregs[i])); - } else { - if (negate) - fp.FNEG(fpr.V(vregs[i]), fpr.V(origV[regnum])); - else - fp.FMOV(fpr.V(vregs[i]), fpr.V(origV[regnum])); - } - } else { - fpr.MapRegV(vregs[i], MAP_DIRTY | MAP_NOINIT); - fpr.SpillLockV(vregs[i]); - fp.MOVI2F(fpr.V(vregs[i]), constantArray[regnum + (abs << 2)], SCRATCH1, (bool)negate); - } - } - } - - void Arm64Jit::GetVectorRegsPrefixD(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixDFlag & JitState::PREFIX_KNOWN); - - GetVectorRegs(regs, sz, vectorReg); - if (js.prefixD == 0) - return; - - int n = GetNumVectorElements(sz); - for (int i = 0; i < n; i++) { - // Hopefully this is rare, we'll just write it into a reg we drop. - if (js.VfpuWriteMask(i)) - regs[i] = fpr.GetTempV(); - } - } - - void Arm64Jit::ApplyPrefixD(const u8 *vregs, VectorSize sz) { - _assert_msg_(js.prefixDFlag & JitState::PREFIX_KNOWN, "Unexpected unknown prefix!"); - if (!js.prefixD) - return; - - int n = GetNumVectorElements(sz); - for (int i = 0; i < n; i++) { - if (js.VfpuWriteMask(i)) - continue; - - int sat = (js.prefixD >> (i * 2)) & 3; - if (sat == 1) { - // clamped = x < 0 ? (x > 1 ? 1 : x) : x [0, 1] - fpr.MapRegV(vregs[i], MAP_DIRTY); - - fp.MOVI2F(S0, 0.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - fp.FMIN(fpr.V(vregs[i]), fpr.V(vregs[i]), S1); - fp.FMAX(fpr.V(vregs[i]), fpr.V(vregs[i]), S0); - } else if (sat == 3) { - // clamped = x < -1 ? (x > 1 ? 1 : x) : x [-1, 1] - fpr.MapRegV(vregs[i], MAP_DIRTY); - - fp.MOVI2F(S0, -1.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - fp.FMIN(fpr.V(vregs[i]), fpr.V(vregs[i]), S1); - fp.FMAX(fpr.V(vregs[i]), fpr.V(vregs[i]), S0); - } - } - } - - void Arm64Jit::Comp_SV(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU_VFPU); - CheckMemoryBreakpoint(); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - // NOTE: This should be done before the $zr check in case there are reads with side effects. - DISABLE; - return; - } - - s32 offset = (signed short)(op & 0xFFFC); - int vt = ((op >> 16) & 0x1f) | ((op & 3) << 5); - MIPSGPReg rs = _RS; - - std::vector skips; - switch (op >> 26) { - case 50: // lv.s - { - if (!gpr.IsImm(rs) && jo.cachePointers && g_Config.bFastMemory && (offset & 3) == 0 && offset >= 0 && offset < 16384) { - gpr.MapRegAsPointer(rs); - fpr.MapRegV(vt, MAP_NOINIT | MAP_DIRTY); - fp.LDR(32, INDEX_UNSIGNED, fpr.V(vt), gpr.RPtr(rs), offset); - break; - } - - // CC might be set by slow path below, so load regs first. - fpr.MapRegV(vt, MAP_DIRTY | MAP_NOINIT); - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (offset + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = offset + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1, addr); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, offset); - } else { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } - } - fp.LDR(32, fpr.V(vt), SCRATCH1_64, ArithOption(MEMBASEREG)); - for (auto skip : skips) { - SetJumpTarget(skip); - } - } - break; - - case 58: // sv.s - { - if (!gpr.IsImm(rs) && jo.cachePointers && g_Config.bFastMemory && (offset & 3) == 0 && offset >= 0 && offset < 16384) { - gpr.MapRegAsPointer(rs); - fpr.MapRegV(vt, 0); - fp.STR(32, INDEX_UNSIGNED, fpr.V(vt), gpr.RPtr(rs), offset); - break; - } - - // CC might be set by slow path below, so load regs first. - fpr.MapRegV(vt); - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (offset + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = offset + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1, addr); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, offset); - } else { - skips = SetScratch1ForSafeAddress(rs, offset, SCRATCH2); - } - } - fp.STR(32, fpr.V(vt), SCRATCH1_64, ArithOption(MEMBASEREG)); - for (auto skip : skips) { - SetJumpTarget(skip); - } - } - break; - - - default: - DISABLE; - } - } - - void Arm64Jit::Comp_SVQ(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU_VFPU); - CheckMemoryBreakpoint(); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - int imm = (signed short)(op&0xFFFC); - int vt = (((op >> 16) & 0x1f)) | ((op&1) << 5); - MIPSGPReg rs = _RS; - - std::vector skips; - switch (op >> 26) - { - case 54: //lv.q - { - // CC might be set by slow path below, so load regs first. - u8 vregs[4]; - GetVectorRegs(vregs, V_Quad, vt); - fpr.MapRegsAndSpillLockV(vregs, V_Quad, MAP_DIRTY | MAP_NOINIT); - - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (imm + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = imm + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1_64, addr + (uintptr_t)Memory::base); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, imm); - } else { - skips = SetScratch1ForSafeAddress(rs, imm, SCRATCH2); - } - if (jo.enablePointerify) { - MOVK(SCRATCH1_64, ((uint64_t)Memory::base) >> 32, SHIFT_32); - } else { - ADD(SCRATCH1_64, SCRATCH1_64, MEMBASEREG); - } - } - - fp.LDP(32, INDEX_SIGNED, fpr.V(vregs[0]), fpr.V(vregs[1]), SCRATCH1_64, 0); - fp.LDP(32, INDEX_SIGNED, fpr.V(vregs[2]), fpr.V(vregs[3]), SCRATCH1_64, 8); - - for (auto skip : skips) { - SetJumpTarget(skip); - } - } - break; - - case 62: //sv.q - { - // CC might be set by slow path below, so load regs first. - u8 vregs[4]; - GetVectorRegs(vregs, V_Quad, vt); - fpr.MapRegsAndSpillLockV(vregs, V_Quad, 0); - - if (gpr.IsImm(rs)) { -#ifdef MASKED_PSP_MEMORY - u32 addr = (imm + gpr.GetImm(rs)) & 0x3FFFFFFF; -#else - u32 addr = imm + gpr.GetImm(rs); -#endif - MOVI2R(SCRATCH1_64, addr + (uintptr_t)Memory::base); - } else { - gpr.MapReg(rs); - if (g_Config.bFastMemory) { - SetScratch1ToEffectiveAddress(rs, imm); - } else { - skips = SetScratch1ForSafeAddress(rs, imm, SCRATCH2); - } - if (jo.enablePointerify) { - MOVK(SCRATCH1_64, ((uint64_t)Memory::base) >> 32, SHIFT_32); - } else { - ADD(SCRATCH1_64, SCRATCH1_64, MEMBASEREG); - } - } - fp.STP(32, INDEX_SIGNED, fpr.V(vregs[0]), fpr.V(vregs[1]), SCRATCH1_64, 0); - fp.STP(32, INDEX_SIGNED, fpr.V(vregs[2]), fpr.V(vregs[3]), SCRATCH1_64, 8); - - for (auto skip : skips) { - SetJumpTarget(skip); - } - } - break; - - default: - DISABLE; - break; - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VVectorInit(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - // vzero/vone are vmov with a constant forced into the S prefix, so a pending one changes them. - if (js.HasUnknownPrefix() || js.HasSPrefix()) { - DISABLE; - } - - switch ((op >> 16) & 0xF) { - case 6: // v=zeros; break; //vzero - fp.MOVI2F(S0, 0.0f, SCRATCH1); - break; - case 7: // v=ones; break; //vone - fp.MOVI2F(S0, 1.0f, SCRATCH1); - break; - default: - DISABLE; - break; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, _VD); - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - for (int i = 0; i < n; ++i) - fp.FMOV(fpr.V(dregs[i]), S0); - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VIdt(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - // Like vone, a pending S prefix changes the result. - if (js.HasUnknownPrefix() || js.HasSPrefix()) { - DISABLE; - } - - int vd = _VD; - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - // Only pairs and quads are handled here. - if (sz != V_Pair && sz != V_Quad) { - DISABLE; - } - fp.MOVI2F(S0, 0.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, _VD); - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - switch (sz) { - case V_Pair: - fp.FMOV(fpr.V(dregs[0]), (vd & 1) == 0 ? S1 : S0); - fp.FMOV(fpr.V(dregs[1]), (vd & 1) == 1 ? S1 : S0); - break; - case V_Quad: - fp.FMOV(fpr.V(dregs[0]), (vd & 3) == 0 ? S1 : S0); - fp.FMOV(fpr.V(dregs[1]), (vd & 3) == 1 ? S1 : S0); - fp.FMOV(fpr.V(dregs[2]), (vd & 3) == 2 ? S1 : S0); - fp.FMOV(fpr.V(dregs[3]), (vd & 3) == 3 ? S1 : S0); - break; - default: - _dbg_assert_msg_( 0, "Trying to interpret instruction that can't be interpreted"); - break; - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VMatrixInit(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - // They do care about prefixes, so leave those to the interpreter like the IR does. - if (!js.HasNoPrefix()) { - DISABLE; - } - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - u8 dregs[16]; - GetMatrixRegs(dregs, sz, _VD); - - switch ((op >> 16) & 0xF) { - case 3: // vmidt - fp.MOVI2F(S0, 0.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapRegV(dregs[a * 4 + b], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[a * 4 + b]), a == b ? S1 : S0); - } - } - break; - case 6: // vmzero - fp.MOVI2F(S0, 0.0f, SCRATCH1); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapRegV(dregs[a * 4 + b], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[a * 4 + b]), S0); - } - } - break; - case 7: // vmone - fp.MOVI2F(S1, 1.0f, SCRATCH1); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapRegV(dregs[a * 4 + b], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[a * 4 + b]), S1); - } - } - break; - } - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VHdp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - // The S prefix gets its last lane forced to 1, which only the interpreter does. - if (js.HasUnknownPrefix() || js.HasSPrefix() || !IsPrefixWithinSize(js.prefixT, op)) { - DISABLE; - } - - int vd = _VD; - int vs = _VS; - int vt = _VT; - VectorSize sz = GetVecSize(op); - - // TODO: Force read one of them into regs? probably not. - u8 sregs[4], tregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, vs); - GetVectorRegsPrefixT(tregs, sz, vt); - GetVectorRegsPrefixD(dregs, V_Single, vd); - - // TODO: applyprefixST here somehow (shuffle, etc...) - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - fpr.MapRegsAndSpillLockV(tregs, sz, 0); - fp.FMUL(S0, fpr.V(sregs[0]), fpr.V(tregs[0])); - - int n = GetNumVectorElements(sz); - for (int i = 1; i < n; i++) { - // sum += s[i]*t[i], rounding the product first like the other backends (no FMADD). - if (i == n - 1) { - fp.FADD(S0, S0, fpr.V(tregs[i])); - } else { - fp.FMUL(S1, fpr.V(sregs[i]), fpr.V(tregs[i])); - fp.FADD(S0, S0, S1); - } - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - - fpr.MapRegV(dregs[0], MAP_NOINIT | MAP_DIRTY); - - fp.FMOV(fpr.V(dregs[0]), S0); - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - alignas(16) static const float vavg_table[4] = { 1.0f, 1.0f / 2.0f, 1.0f / 3.0f, 1.0f / 4.0f }; - - void Arm64Jit::Comp_Vhoriz(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - int vd = _VD; - int vs = _VS; - int vt = _VT; - VectorSize sz = GetVecSize(op); - - // TODO: Force read one of them into regs? probably not. - u8 sregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, vs); - GetVectorRegsPrefixD(dregs, V_Single, vd); - - // TODO: applyprefixST here somehow (shuffle, etc...) - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - - int n = GetNumVectorElements(sz); - - bool is_vavg = ((op >> 16) & 0x1f) == 7; - if (is_vavg) { - fp.MOVI2F(S1, vavg_table[n - 1], SCRATCH1); - } - // Have to start at +0.000 for the correct sign. - fp.MOVI2F(S0, 0.0f, SCRATCH1); - for (int i = 0; i < n; i++) { - // sum += s[i]; - fp.FADD(S0, S0, fpr.V(sregs[i])); - } - - fpr.MapRegV(dregs[0], MAP_NOINIT | MAP_DIRTY); - if (is_vavg) { - fp.FMUL(fpr.V(dregs[0]), S0, S1); - } else { - fp.FMOV(fpr.V(dregs[0]), S0); - } - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VDot(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) { - DISABLE; - } - - int vd = _VD; - int vs = _VS; - int vt = _VT; - VectorSize sz = GetVecSize(op); - - // TODO: Force read one of them into regs? probably not. - u8 sregs[4], tregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, vs); - GetVectorRegsPrefixT(tregs, sz, vt); - GetVectorRegsPrefixD(dregs, V_Single, vd); - - // TODO: applyprefixST here somehow (shuffle, etc...) - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - fpr.MapRegsAndSpillLockV(tregs, sz, 0); - fp.FMUL(S0, fpr.V(sregs[0]), fpr.V(tregs[0])); - - int n = GetNumVectorElements(sz); - for (int i = 1; i < n; i++) { - // sum += s[i]*t[i], rounding the product first like the other backends (no FMADD). - fp.FMUL(S1, fpr.V(sregs[i]), fpr.V(tregs[i])); - fp.FADD(S0, S0, S1); - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - - fpr.MapRegV(dregs[0], MAP_NOINIT | MAP_DIRTY); - - fp.FMOV(fpr.V(dregs[0]), S0); - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VecDo3(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) { - DISABLE; - } - // vdiv applies the prefixes to its last lane only, from position 0. - if (((op >> 23) & 7) == 7 && GetVecSize(op) != V_Single && !js.HasNoPrefix()) { - DISABLE; - } - - int vd = _VD; - int vs = _VS; - int vt = _VT; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; i++) { - if (!IsOverlapSafe(dregs[i], i, n, sregs, n, tregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - // Map first, then work. This will allow us to use VLDMIA more often - // (when we add the appropriate map function) and the instruction ordering - // will improve. - // Note that mapping like this (instead of first all sregs, first all tregs etc) - // reduces the amount of continuous registers a lot :( - for (int i = 0; i < n; i++) { - fpr.MapDirtyInInV(tempregs[i], sregs[i], tregs[i]); - fpr.SpillLockV(tempregs[i]); - fpr.SpillLockV(sregs[i]); - fpr.SpillLockV(tregs[i]); - } - - const bool minmax = (op >> 26) == 27 && (((op >> 23) & 7) == 2 || ((op >> 23) & 7) == 3); - const ARM64Reg minmaxTemp = minmax ? gpr.GetAndLockTempR() : INVALID_REG; - - for (int i = 0; i < n; i++) { - switch (op >> 26) { - case 24: //VFPU0 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] + t[i]; break; //vadd - fp.FADD(fpr.V(tempregs[i]), fpr.V(sregs[i]), fpr.V(tregs[i])); - break; - case 1: // d[i] = s[i] - t[i]; break; //vsub - fp.FSUB(fpr.V(tempregs[i]), fpr.V(sregs[i]), fpr.V(tregs[i])); - break; - case 7: // d[i] = s[i] / t[i]; break; //vdiv - fp.FDIV(fpr.V(tempregs[i]), fpr.V(sregs[i]), fpr.V(tregs[i])); - break; - default: - DISABLE; - } - break; - case 25: //VFPU1 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] * t[i]; break; //vmul - fp.FMUL(fpr.V(tempregs[i]), fpr.V(sregs[i]), fpr.V(tregs[i])); - break; - default: - DISABLE; - } - break; - // Fortunately there is FMIN/FMAX on ARM64! - case 27: //VFPU3 - switch ((op >> 23) & 7) { - case 2: // vmin - case 3: // vmax - { - // Compare by vfpu_minmax_key, select the operand with its bits. A tie takes t. - auto key = [&](ARM64Reg r, MIPSReg v) { - fp.FMOV(r, fpr.V(v)); - TSTI2R(r, 0x7F800000); - ASR(minmaxTemp, r, 31); - EOR(r, r, minmaxTemp, ArithOption(minmaxTemp, ST_LSR, 1)); - CSEL(r, WZR, r, CC_EQ); - }; - key(SCRATCH1, sregs[i]); - key(SCRATCH2, tregs[i]); - CMP(SCRATCH1, SCRATCH2); - fp.FCSEL(fpr.V(tempregs[i]), fpr.V(sregs[i]), fpr.V(tregs[i]), ((op >> 23) & 7) == 2 ? CC_LT : CC_GT); - break; - } - case 6: // vsge - case 7: // vslt - if (i == 0) { - fp.MOVI2F(S0, 1.0f, SCRATCH1); - fp.MOVI2F(S1, 0.0f, SCRATCH1); - } - // GE and MI are both false for unordered, so NaN gives 0 either way. - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - fp.FCSEL(fpr.V(tempregs[i]), S0, S1, ((op >> 23) & 7) == 6 ? CC_GE : CC_MI); - break; - } - break; - - default: - DISABLE; - } - } - - for (int i = 0; i < n; i++) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - gpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VV2Op(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix()) { - DISABLE; - } - - // The prefix rules here follow the IR frontend, which follows the interpreter and the - // prefix tests in pspautotests. What's not handled goes to the interpreter. - int optype = (op >> 16) & 0x1f; - if (optype == 0) { - if (!IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - } else if (optype == 1 || optype == 2) { - // vabs and vneg are vmov with the abs/negate bit forced on, so a negate in the S prefix - // doesn't negate twice. D prefix is fine for these, and used sometimes. - if (js.HasSPrefix()) - DISABLE; - } else if (optype == 5 && js.HasDPrefix()) { - // vsat1 doesn't apply the D saturation. - DISABLE; - } - if (optype >= 16 && !js.HasNoPrefix()) { - // These apply the S and D prefixes to their last lane only, from prefix position 0. - // That's the whole vector for a single, so only that case is handled here. - if (GetVecSize(op) != V_Single) - DISABLE; - if (!IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - // The negative ones seem to use negate flags as a prefix hack. - if (optype >= 24 && (js.prefixS & 0x000F0000) != 0) - DISABLE; - } - - // Pre-processing: Eliminate silly no-op VMOVs, common in Wipeout Pure - if (optype == 0 && _VS == _VD && js.HasNoPrefix()) { - return; - } - - // The special functions call the exact C versions. - switch ((op >> 16) & 0x1f) { - case 16: // vrcp - case 17: // vrsq - case 18: // vsin - case 19: // vcos - case 20: // vexp2 - case 21: // vlog2 - case 22: // vsqrt - case 23: // vasin - case 24: // vnrcp - case 26: // vnsin - case 28: // vrexp2 - CompVV2OpCall(op); - return; - default: - ; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - // Pre map the registers to get better instruction ordering. - // Note that mapping like this (instead of first all sregs, first all tempregs etc) - // reduces the amount of continuous registers a lot :( - for (int i = 0; i < n; i++) { - fpr.MapDirtyInV(tempregs[i], sregs[i]); - fpr.SpillLockV(tempregs[i]); - fpr.SpillLockV(sregs[i]); - } - - // Warning: sregs[i] and tempxregs[i] may be the same reg. - // Helps for vmov, hurts for vrcp, etc. - for (int i = 0; i < n; i++) { - switch ((op >> 16) & 0x1f) { - case 0: // d[i] = s[i]; break; //vmov - // Probably for swizzle. - fp.FMOV(fpr.V(tempregs[i]), fpr.V(sregs[i])); - break; - case 1: // d[i] = fabsf(s[i]); break; //vabs - fp.FABS(fpr.V(tempregs[i]), fpr.V(sregs[i])); - break; - case 2: // d[i] = -s[i]; break; //vneg - fp.FNEG(fpr.V(tempregs[i]), fpr.V(sregs[i])); - break; - case 4: // if (s[i] < 0) d[i] = 0; else {if(s[i] > 1.0f) d[i] = 1.0f; else d[i] = s[i];} break; // vsat0 - if (i == 0) { - fp.MOVI2F(S0, 0.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - } - fp.FCMP(fpr.V(sregs[i]), S0); - fp.FMOV(fpr.V(tempregs[i]), fpr.V(sregs[i])); - fp.FMAX(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S0); - fp.FMIN(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S1); - break; - case 5: // if (s[i] < -1.0f) d[i] = -1.0f; else {if(s[i] > 1.0f) d[i] = 1.0f; else d[i] = s[i];} break; // vsat1 - if (i == 0) { - fp.MOVI2F(S0, -1.0f, SCRATCH1); - fp.MOVI2F(S1, 1.0f, SCRATCH1); - } - fp.FCMP(fpr.V(sregs[i]), S0); - fp.FMOV(fpr.V(tempregs[i]), fpr.V(sregs[i])); - fp.FMAX(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S0); - fp.FMIN(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S1); - break; - default: - ERROR_LOG(Log::JIT, "case missing in vfpu vv2op"); - DISABLE; - break; - } - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - // The VFPU special functions call the exact C versions. The lanes stay in S8-S11 across the - // calls (callee-saved), and the results are stored to the destinations' homes. - // After fpr.FlushBeforeCall(), a VFPU register is either still mapped (in S8-S15) or its value is - // in memory. Loads it into dest either way. - void Arm64Jit::LoadVAfterCallFlush(ARM64Reg dest, u8 vreg) { - if (fpr.IsMappedV(vreg)) { - fp.FMOV(dest, fpr.V(vreg)); - } else { - fp.LDR(32, INDEX_UNSIGNED, dest, CTXREG, fpr.GetMipsRegOffsetV(vreg)); - } - } - - void Arm64Jit::CompVV2OpCall(MIPSOpcode op) { - if (js.HasSPrefix()) { - DISABLE; - } - - const int optype = (op >> 16) & 0x1f; - // vnrcp and vnsin negate the result. - const bool negate = optype == 24 || optype == 26; - float (*func)(float) = nullptr; - switch (optype) { - case 16: case 24: func = &vfpu_rcp; break; - case 17: func = &vfpu_rsqrt; break; - case 18: case 26: func = &vfpu_sin; break; - case 19: func = &vfpu_cos; break; - case 20: func = &vfpu_exp2; break; - case 21: func = &vfpu_log2; break; - case 22: func = &vfpu_sqrt; break; - case 23: func = &vfpu_asin; break; - case 28: func = &vfpu_rexp2; break; - default: DISABLE; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - u8 sregs[4], dregs[4]; - GetVectorRegs(sregs, sz, _VS); - GetVectorRegs(dregs, sz, _VD); - - // Values in S8-S15 survive the calls, so only the rest is flushed. - gpr.FlushBeforeCall(); - fpr.FlushBeforeCall(); - - if (n == 1) { - LoadVAfterCallFlush(S0, sregs[0]); - QuickCallFunction(SCRATCH2_64, func); - } else { - // The lanes wait in S8 and up between the calls, so free those too. - for (int i = 0; i < n; i++) { - fpr.FlushArmReg((ARM64Reg)(S8 + i)); - } - for (int i = 0; i < n; i++) { - LoadVAfterCallFlush((ARM64Reg)(S8 + i), sregs[i]); - } - for (int i = 0; i < n; i++) { - fp.FMOV(S0, (ARM64Reg)(S8 + i)); - QuickCallFunction(SCRATCH2_64, func); - fp.FMOV((ARM64Reg)(S8 + i), S0); - } - // Into S0-S3, which the cache never allocates, before mapping the destinations. - for (int i = 0; i < n; i++) { - fp.FMOV((ARM64Reg)(S0 + i), (ARM64Reg)(S8 + i)); - } - } - - for (int i = 0; i < n; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - if (negate) { - fp.FNEG(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } else { - fp.FMOV(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vi2f(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op)) { - DISABLE; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - int imm = (op >> 16) & 0x1f; - const float mult = 1.0f / (float)(1UL << imm); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - if (mult != 1.0f) - fp.MOVI2F(S0, mult, SCRATCH1); - - // TODO: Use the SCVTF with builtin scaling where possible. - for (int i = 0; i < n; i++) { - fpr.MapDirtyInV(tempregs[i], sregs[i]); - fp.SCVTF(fpr.V(tempregs[i]), fpr.V(sregs[i])); - if (mult != 1.0f) - fp.FMUL(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S0); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vh2f(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix()) { - DISABLE; - } - - // Half to float, by calling vfpu_h2f: FCVTL neither flushes subnormal halves nor keeps inf/NaN - // mantissa bits unshifted. Sizes above pair act like pair. - VectorSize sz = GetVecSize(op); - const int nIn = sz == V_Single ? 1 : 2; - const int nOut = nIn * 2; - const VectorSize outSz = sz == V_Single ? V_Pair : V_Quad; - - u8 sregs[4], dregs[4]; - GetVectorRegs(sregs, sz, _VS); - GetVectorRegs(dregs, outSz, _VD); - - // The inputs wait in S8-S9 and the results in S10-S13 between the calls (callee-saved), so - // those are flushed along with the registers a call clobbers. - gpr.FlushBeforeCall(); - fpr.FlushBeforeCall(); - for (int i = 0; i < 6; i++) { - fpr.FlushArmReg((ARM64Reg)(S8 + i)); - } - for (int i = 0; i < nIn; i++) { - LoadVAfterCallFlush((ARM64Reg)(S8 + i), sregs[i]); - } - for (int i = 0; i < nOut; i++) { - fp.FMOV(S0, (ARM64Reg)(S8 + i / 2)); - QuickCallFunction(SCRATCH2_64, (i & 1) ? &vfpu_h2f_upper : &vfpu_h2f_lower); - fp.FMOV((ARM64Reg)(S10 + i), S0); - } - // Into S0-S3, which the cache never allocates, before mapping the destinations. - for (int i = 0; i < nOut; i++) { - fp.FMOV((ARM64Reg)(S0 + i), (ARM64Reg)(S10 + i)); - } - for (int i = 0; i < nOut; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } - - ApplyPrefixD(dregs, outSz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vf2i(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || (js.prefixD & 0xFF) != 0) { - DISABLE; - } - if (((op >> 21) & 0x1C) != 0x10) { - DISABLE; - } - - // Vector float to integer, d[N] = int(s[N] * 2^imm) in the rounding mode from the opcode. - // FCVT saturates like the PSP, but gives 0 for NaN where the PSP gives 0x7FFFFFFF. - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - const int imm = (op >> 16) & 0x1f; - static const RoundingMode modes[4] = { ROUND_N, ROUND_Z, ROUND_P, ROUND_M }; - const RoundingMode rm = modes[(op >> 21) & 3]; - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - // Invert 0x80000000 -> 0x7FFFFFFF for the NaN result. - fp.MVNI(32, EncodeRegToDouble(S1), 0x80, 24); - // Only the truncating conversion takes a scale, the others multiply first (exact). - if (imm != 0 && rm != ROUND_Z) { - fp.MOVI2F(S2, (float)(1UL << imm), SCRATCH1); - } - - for (int i = 0; i < n; i++) { - fpr.MapDirtyInV(tempregs[i], sregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(sregs[i])); - if (imm == 0) { - fp.FCVTS(fpr.V(tempregs[i]), fpr.V(sregs[i]), rm); - } else if (rm == ROUND_Z) { - fp.FCVTZS(fpr.V(tempregs[i]), fpr.V(sregs[i]), imm); - } else { - fp.FMUL(S0, fpr.V(sregs[i]), S2); - fp.FCVTS(fpr.V(tempregs[i]), S0, rm); - } - fp.FCSEL(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S1, CC_VC); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Mftv(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - int imm = op & 0xFF; - MIPSGPReg rt = _RT; - switch ((op >> 21) & 0x1f) { - case 3: //mfv / mfvc - // rt = 0, imm = 255 appears to be used as a CPU interlock by some games. - if (rt != 0) { - if (imm < 128) { //R(rt) = VI(imm); - if (!fpr.IsInRAMV(imm)) { - fpr.MapRegV(imm, 0); - gpr.MapReg(rt, MAP_NOINIT | MAP_DIRTY); - fp.FMOV(gpr.R(rt), fpr.V(imm)); - } else { - gpr.MapReg(rt, MAP_NOINIT | MAP_DIRTY); - LDR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, fpr.GetMipsRegOffsetV(imm)); - } - } else if (imm < 128 + VFPU_CTRL_MAX) { //mtvc - if (imm - 128 == VFPU_CTRL_CC) { - if (gpr.IsImm(MIPS_REG_VFPUCC)) { - gpr.SetImm(rt, gpr.GetImm(MIPS_REG_VFPUCC)); - } else { - gpr.MapDirtyIn(rt, MIPS_REG_VFPUCC); - MOV(gpr.R(rt), gpr.R(MIPS_REG_VFPUCC)); - } - } else { - // In case we have a saved prefix. - FlushPrefixV(); - gpr.MapReg(rt, MAP_NOINIT | MAP_DIRTY); - LDR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, offsetof(MIPSState, vfpuCtrl) + 4 * (imm - 128)); - } - } else { - //ERROR - maybe need to make this value too an "interlock" value? - ERROR_LOG(Log::CPU, "mfv - invalid register %i", imm); - } - } - break; - - case 7: // mtv - if (imm < 128) { - if (rt == MIPS_REG_ZERO) { - fpr.MapRegV(imm, MAP_DIRTY | MAP_NOINIT); - fp.MOVI2F(fpr.V(imm), 0.0f, SCRATCH1); - } else if (!gpr.IsInRAM(rt)) { - gpr.MapReg(rt); - fpr.MapRegV(imm, MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(imm), gpr.R(rt)); - } else { - fpr.MapRegV(imm, MAP_DIRTY | MAP_NOINIT); - fp.LDR(32, INDEX_UNSIGNED, fpr.V(imm), CTXREG, gpr.GetMipsRegOffset(rt)); - } - } else if (imm < 128 + VFPU_CTRL_MAX) { //mtvc //currentMIPS->vfpuCtrl[imm - 128] = R(rt); - if (imm - 128 == VFPU_CTRL_CC) { - // Six condition bits, the rest don't stick (cpu/vfpu/vbranch). - if (gpr.IsImm(rt)) { - gpr.SetImm(MIPS_REG_VFPUCC, gpr.GetImm(rt) & 0x3F); - } else { - gpr.MapDirtyIn(MIPS_REG_VFPUCC, rt); - ANDI2R(gpr.R(MIPS_REG_VFPUCC), gpr.R(rt), 0x3F, SCRATCH1); - } - } else { - // Only some of the bits stick (the low 20 of a prefix, say), same as the IR does it. - u32 mask; - u32 setBits = GetVFPUCtrlSetBits(imm - 128); - if (!GetVFPUCtrlMask(imm - 128, &mask)) { - // Read-only or unknown register: nothing is written. - } else if (mask != 0xFFFFFFFF || setBits != 0) { - gpr.MapReg(rt); - ANDI2R(SCRATCH1, gpr.R(rt), mask, SCRATCH2); - if (setBits != 0) - ORRI2R(SCRATCH1, SCRATCH1, setBits, SCRATCH2); - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, vfpuCtrl) + 4 * (imm - 128)); - } else { - gpr.MapReg(rt); - STR(INDEX_UNSIGNED, gpr.R(rt), CTXREG, offsetof(MIPSState, vfpuCtrl) + 4 * (imm - 128)); - } - } - - // TODO: Optimization if rt is Imm? - // Set these BEFORE disable! - if (imm - 128 == VFPU_CTRL_SPREFIX) { - js.prefixSFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm - 128 == VFPU_CTRL_TPREFIX) { - js.prefixTFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm - 128 == VFPU_CTRL_DPREFIX) { - js.prefixDFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } - } else { - //ERROR - _dbg_assert_msg_( 0, "mtv - invalid register"); - } - break; - - default: - DISABLE; - } - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vmfvc(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - int vd = _VD; - int imm = (op >> 8) & 0x7F; - if (imm < VFPU_CTRL_MAX) { - fpr.MapRegV(vd, MAP_DIRTY | MAP_NOINIT); - if (imm == VFPU_CTRL_CC) { - gpr.MapReg(MIPS_REG_VFPUCC, 0); - fp.FMOV(fpr.V(vd), gpr.R(MIPS_REG_VFPUCC)); - } else { - // In case we have a saved prefix. - FlushPrefixV(); - ADDI2R(SCRATCH1_64, CTXREG, offsetof(MIPSState, vfpuCtrl[0]) + imm * 4, SCRATCH2); - fp.LDR(32, INDEX_UNSIGNED, fpr.V(vd), SCRATCH1_64, 0); - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - } else { - fpr.MapRegV(vd, MAP_DIRTY | MAP_NOINIT); - fp.MOVI2F(fpr.V(vd), 0.0f, SCRATCH1); - } - } - - void Arm64Jit::Comp_Vmtvc(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - int vs = _VS; - int imm = op & 0x7F; - if (imm < VFPU_CTRL_MAX) { - fpr.MapRegV(vs); - if (imm == VFPU_CTRL_CC) { - gpr.MapReg(MIPS_REG_VFPUCC, MAP_DIRTY | MAP_NOINIT); - fp.FMOV(gpr.R(MIPS_REG_VFPUCC), fpr.V(vs)); - } else { - ADDI2R(SCRATCH1_64, CTXREG, offsetof(MIPSState, vfpuCtrl[0]) + imm * 4, SCRATCH2); - fp.STR(32, INDEX_UNSIGNED, fpr.V(vs), SCRATCH1_64, 0); - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - - if (imm == VFPU_CTRL_SPREFIX) { - js.prefixSFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm == VFPU_CTRL_TPREFIX) { - js.prefixTFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm == VFPU_CTRL_DPREFIX) { - js.prefixDFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } - } - } - - void Arm64Jit::Comp_Vmmov(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMMOV); - if (!js.HasNoPrefix()) { - DISABLE; - } - - if (_VS == _VD) { - // A lot of these no-op matrix moves in Wipeout... Just drop the instruction entirely. - return; - } - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - u8 sregs[16], dregs[16]; - GetMatrixRegs(sregs, sz, _VS); - GetMatrixRegs(dregs, sz, _VD); - - switch (GetMatrixOverlap(_VS, _VD, sz)) { - case OVERLAP_EQUAL: - // In-place transpose - DISABLE; - case OVERLAP_PARTIAL: - DISABLE; - case OVERLAP_NONE: - default: - break; - } - - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapDirtyInV(dregs[a * 4 + b], sregs[a * 4 + b]); - fp.FMOV(fpr.V(dregs[a * 4 + b]), fpr.V(sregs[a * 4 + b])); - } - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VScl(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - // The T prefix is applied oddly here, the interpreter knows how. - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4], treg; - GetVectorRegsPrefixS(sregs, sz, _VS); - // TODO: Prefixes seem strange... - GetVectorRegsPrefixT(&treg, V_Single, _VT); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Move to S0 early, so we don't have to worry about overlap with scale. - fpr.LoadToRegV(S0, treg); - - // For prefixes to work, we just have to ensure that none of the output registers spill - // and that there's no overlap. - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - // Need to use temp regs - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - // The meat of the function! - for (int i = 0; i < n; i++) { - fpr.MapDirtyInV(tempregs[i], sregs[i]); - fp.FMUL(fpr.V(tempregs[i]), fpr.V(sregs[i]), S0); - } - - for (int i = 0; i < n; i++) { - // All must be mapped for prefixes to work. - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vmmul(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMMUL); - if (!js.HasNoPrefix()) { - DISABLE; - } - - if (PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { - // Fall back to interpreter, which has the accurate implementation. - // Later we might do something more optimized here. - DISABLE; - } - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - u8 sregs[16], tregs[16], dregs[16]; - GetMatrixRegs(sregs, sz, _VS); - GetMatrixRegs(tregs, sz, _VT); - GetMatrixRegs(dregs, sz, _VD); - - MatrixOverlapType soverlap = GetMatrixOverlap(_VS, _VD, sz); - MatrixOverlapType toverlap = GetMatrixOverlap(_VT, _VD, sz); - - if (soverlap || toverlap) { - DISABLE; - } else { - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapDirtyInInV(dregs[a * 4 + b], sregs[b * 4], tregs[a * 4], true); - fp.FMUL(fpr.V(dregs[a * 4 + b]), fpr.V(sregs[b * 4]), fpr.V(tregs[a * 4])); - for (int c = 1; c < n; c++) { - fpr.MapDirtyInInV(dregs[a * 4 + b], sregs[b * 4 + c], tregs[a * 4 + c], false); - fp.FMUL(S0, fpr.V(sregs[b * 4 + c]), fpr.V(tregs[a * 4 + c])); - fp.FADD(fpr.V(dregs[a * 4 + b]), fpr.V(dregs[a * 4 + b]), S0); - } - } - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - } - - void Arm64Jit::Comp_Vmscl(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMSCL); - if (!js.HasNoPrefix()) { - DISABLE; - } - - // Matrix scale: d[N,M] = s[N,M] * t[0]. Element-wise, so transposition doesn't matter, but a - // source that partly overlaps the destination would be read after it's written. - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - if (_VS != _VD && GetMatrixOverlap(_VS, _VD, sz) != OVERLAP_NONE) { - DISABLE; - } - - u8 sregs[16], dregs[16], treg; - GetMatrixRegs(sregs, sz, _VS); - GetMatrixRegs(dregs, sz, _VD); - GetVectorRegs(&treg, V_Single, _VT); - - // The scale can be part of the destination, so keep a copy. - fpr.MapRegV(treg); - fp.FMOV(S0, fpr.V(treg)); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - fpr.MapDirtyInV(dregs[a * 4 + b], sregs[a * 4 + b]); - fp.FMUL(fpr.V(dregs[a * 4 + b]), fpr.V(sregs[a * 4 + b]), S0); - } - } - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vtfm(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VTFM); - if (!js.HasNoPrefix()) { - DISABLE; - } - - VectorSize sz = GetVecSize(op); - MatrixSize msz = GetMtxSize(op); - int n = GetNumVectorElements(sz); - int ins = (op >> 23) & 7; - - bool homogenous = false; - if (n == ins) { - n++; - sz = (VectorSize)((int)(sz)+1); - msz = (MatrixSize)((int)(msz)+1); - homogenous = true; - } - // Otherwise, n should already be ins + 1. - else if (n != ins + 1) { - DISABLE; - } - - u8 sregs[16], dregs[4], tregs[4]; - GetMatrixRegs(sregs, msz, _VS); - GetVectorRegs(tregs, sz, _VT); - GetVectorRegs(dregs, sz, _VD); - - MatrixOverlapType soverlap = GetMatrixOverlap(_VS, _VD, msz); - MatrixOverlapType toverlap = GetMatrixOverlap(_VT, _VD, msz); - - int tempregs[4]; - for (int i = 0; i < n; i++) { - if (soverlap || toverlap) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - fpr.SpillLockV(tempregs[i]); - } - for (int i = 0; i < n; i++) { - fpr.MapRegV(tempregs[i], MAP_NOINIT); - fpr.MapInInV(sregs[i * 4], tregs[0]); - fp.FMUL(fpr.V(tempregs[i]), fpr.V(sregs[i * 4]), fpr.V(tregs[0])); - for (int k = 1; k < n; k++) { - if (!homogenous || k != n - 1) { - fpr.MapInInV(sregs[i * 4 + k], tregs[k]); - // Unfused, like the other backends. - fp.FMUL(S0, fpr.V(sregs[i * 4 + k]), fpr.V(tregs[k])); - fp.FADD(fpr.V(tempregs[i]), fpr.V(tempregs[i]), S0); - } else { - fpr.MapRegV(sregs[i * 4 + k]); - fp.FADD(fpr.V(tempregs[i]), fpr.V(tempregs[i]), fpr.V(sregs[i * 4 + k])); - } - } - } - for (int i = 0; i < n; i++) { - u8 temp = tempregs[i]; - if (temp != dregs[i]) { - fpr.MapDirtyInV(dregs[i], temp, true); - fp.FMOV(fpr.V(dregs[i]), fpr.V(temp)); - } - } - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VCrs(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix() || js.HasTPrefix()) { - DISABLE; - } - - // Half a cross product: d[0] = s[y]*t[z], d[1] = s[z]*t[x], d[2] = s[x]*t[y]. - VectorSize sz = GetVecSize(op); - if (sz != V_Triple) { - DISABLE; - } - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Into S0-S2 first, since d may overlap s or t. - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - fpr.MapRegsAndSpillLockV(tregs, sz, 0); - fp.FMUL(S0, fpr.V(sregs[1]), fpr.V(tregs[2])); - fp.FMUL(S1, fpr.V(sregs[2]), fpr.V(tregs[0])); - fp.FMUL(S2, fpr.V(sregs[0]), fpr.V(tregs[1])); - - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT); - for (int i = 0; i < 3; i++) { - fp.FMOV(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VDet(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - // 2D determinant of two vectors: d[0] = s[0]*t[1] - s[1]*t[0]. - VectorSize sz = GetVecSize(op); - if (sz != V_Pair) { - DISABLE; - } - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, V_Single, _VD); - - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - fpr.MapRegsAndSpillLockV(tregs, sz, 0); - fp.FMUL(S1, fpr.V(sregs[1]), fpr.V(tregs[0])); - fp.FMUL(S0, fpr.V(sregs[0]), fpr.V(tregs[1])); - fp.FSUB(S0, S0, S1); - - fpr.MapRegV(dregs[0], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[0]), S0); - - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vi2x(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - int bits = ((op >> 16) & 2) == 0 ? 8 : 16; // vi2uc/vi2c (0/1), vi2us/vi2s (2/3) - bool unsignedOp = ((op >> 16) & 1) == 0; // vi2uc (0), vi2us (2) - - // These instructions pack pairs or quads of integers into 32 bits. - // The unsigned (u) versions skip the sign bit when packing. - VectorSize sz = GetVecSize(op); - VectorSize outsize; - if (bits == 8) { - outsize = V_Single; - if (sz != V_Quad) { - DISABLE; - } - } else { - switch (sz) { - case V_Pair: - outsize = V_Single; - break; - case V_Quad: - outsize = V_Pair; - break; - default: - DISABLE; - } - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, outsize, _VD); - - int n = GetNumVectorElements(sz); - int nOut = GetNumVectorElements(outsize); - - // Take the single registers and combine them to a D or Q register. - for (int i = 0; i < n; i++) { - fpr.MapRegV(sregs[i], sz); - fp.INS(32, Q0, i, fpr.V(sregs[i]), 0); - } - - if (unsignedOp) { - // What's the best way to zero a Q reg? - fp.EOR(Q1, Q1, Q1); - fp.SMAX(32, Q0, Q0, Q1); - } - - // At this point, we simply need to collect the high bits of each 32-bit lane into one register. - if (bits == 8) { - // Really want to do a SHRN(..., 23/24) but that can't be encoded. So we synthesize it. - fp.USHR(32, Q0, Q0, 16); - fp.SHRN(16, D0, Q0, unsignedOp ? 7 : 8); - fp.XTN(8, D0, Q0); - } else { - fp.SHRN(16, D0, Q0, unsignedOp ? 15 : 16); - } - - // Split apart again. - for (int i = 0; i < nOut; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - fp.INS(32, fpr.V(dregs[i]), 0, Q0, i); - } - - ApplyPrefixD(dregs, outsize); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vx2i(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - int bits = ((op >> 16) & 2) == 0 ? 8 : 16; // vuc2i/vc2i (0/1), vus2i/vs2i (2/3) - bool unsignedOp = ((op >> 16) & 1) == 0; // vuc2i (0), vus2i (2) - - // vs2i or vus2i unpack pairs of 16-bit integers into 32-bit integers, with the values - // at the top. vus2i shifts it an extra bit right afterward. - // vc2i and vuc2i unpack quads of 8-bit integers into 32-bit integers, with the values - // at the top too. vuc2i is a bit special (see below.) - // Let's do this similarly as h2f - we do a solution that works for both singles and pairs - // then use it for both. - - VectorSize sz = GetVecSize(op); - VectorSize outsize; - if (bits == 8) { - outsize = V_Quad; - } else { - switch (sz) { - case V_Single: - outsize = V_Pair; - break; - case V_Pair: - outsize = V_Quad; - break; - default: - DISABLE; - } - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, outsize, _VD); - - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - int n = 1; - if (sz == V_Single) { - n = 1; - } else if (sz == V_Pair) { - n = 2; - } else if (bits == 8) { - n = 1; - } - - // Take the single registers and combine them to a D or Q register. - for (int i = 0; i < n; i++) { - fpr.MapRegV(sregs[i], sz); - fp.INS(32, Q0, i, fpr.V(sregs[i]), 0); - } - - if (bits == 16) { - // Simply expand, to upper bits. - // Hm, can't find a USHLL equivalent that works with shift == size? - fp.UXTL(16, Q0, D0); - fp.SHL(32, Q0, Q0, 16); - } else if (bits == 8) { - fp.UXTL(8, Q0, D0); - fp.UXTL(16, Q0, D0); - fp.SHL(32, Q0, D0, 24); - if (unsignedOp) { - // vuc2i is a bit special. It spreads out the bits like this: - // s[0] = 0xDDCCBBAA -> d[0] = (0xAAAAAAAA >> 1), d[1] = (0xBBBBBBBB >> 1), etc. - fp.USHR(32, Q1, Q0, 8); - fp.ORR(Q0, Q0, Q1); - fp.USHR(32, Q1, Q0, 16); - fp.ORR(Q0, Q0, Q1); - } - } - - // At this point we have the regs in the 4 lanes. - // In the "u" mode, we need to shift it out of the sign bit. - if (unsignedOp) { - Arm64Gen::ARM64Reg reg = (outsize == V_Quad) ? Q0 : D0; - fp.USHR(32, reg, reg, 1); - } - - fpr.MapRegsAndSpillLockV(dregs, outsize, MAP_NOINIT); - - int nOut = 2; - if (outsize == V_Quad) - nOut = 4; - - // Split apart again. - for (int i = 0; i < nOut; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - fp.INS(32, fpr.V(dregs[i]), 0, Q0, i); - } - - ApplyPrefixD(dregs, outsize); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_VCrossQuat(MIPSOpcode op) { - // This op does not support prefixes anyway. - CONDITIONAL_DISABLE(VFPU_VEC); - if (!js.HasNoPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegs(sregs, sz, _VS); - GetVectorRegs(tregs, sz, _VT); - GetVectorRegs(dregs, sz, _VD); - - // Map everything into registers. - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - fpr.MapRegsAndSpillLockV(tregs, sz, 0); - - if (sz == V_Triple) { - MIPSReg temp3 = fpr.GetTempV(); - MIPSReg temp4 = fpr.GetTempV(); - fpr.MapRegV(temp3, MAP_DIRTY | MAP_NOINIT); - fpr.MapRegV(temp4, MAP_DIRTY | MAP_NOINIT); - // Cross product vcrsp.t - - // Note: using FMSUB here causes accuracy issues, see #18203. - // Compute X: s[1] * t[2] - s[2] * t[1] - fp.FMUL(fpr.V(temp3), fpr.V(sregs[1]), fpr.V(tregs[2])); - fp.FMUL(fpr.V(temp4), fpr.V(sregs[2]), fpr.V(tregs[1])); - fp.FSUB(S0, fpr.V(temp3), fpr.V(temp4)); - - // Compute Y: s[2] * t[0] - s[0] * t[2] - fp.FMUL(fpr.V(temp3), fpr.V(sregs[2]), fpr.V(tregs[0])); - fp.FMUL(fpr.V(temp4), fpr.V(sregs[0]), fpr.V(tregs[2])); - fp.FSUB(S1, fpr.V(temp3), fpr.V(temp4)); - - // Compute Z: s[0] * t[1] - s[1] * t[0] - fp.FMUL(fpr.V(temp3), fpr.V(sregs[0]), fpr.V(tregs[1])); - fp.FMUL(fpr.V(temp4), fpr.V(sregs[1]), fpr.V(tregs[0])); - fp.FSUB(fpr.V(temp3), fpr.V(temp3), fpr.V(temp4)); - - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT); - fp.FMOV(fpr.V(dregs[0]), S0); - fp.FMOV(fpr.V(dregs[1]), S1); - fp.FMOV(fpr.V(dregs[2]), fpr.V(temp3)); - } else if (sz == V_Quad) { - MIPSReg temp3 = fpr.GetTempV(); - MIPSReg temp4 = fpr.GetTempV(); - fpr.MapRegV(temp3, MAP_DIRTY | MAP_NOINIT); - fpr.MapRegV(temp4, MAP_DIRTY | MAP_NOINIT); - - // Quaternion product vqmul.q, unfused and summed in the interpreter's order. - // sum = a * b, then sum += or -= c * d for each further pair. - auto sumOfProducts = [&](ARM64Reg dest, const int (&terms)[4][3]) { - fp.FMUL(dest, fpr.V(sregs[terms[0][0]]), fpr.V(tregs[terms[0][1]])); - for (int i = 1; i < 4; i++) { - fp.FMUL(S2, fpr.V(sregs[terms[i][0]]), fpr.V(tregs[terms[i][1]])); - if (terms[i][2] < 0) { - fp.FSUB(dest, dest, S2); - } else { - fp.FADD(dest, dest, S2); - } - } - }; - // d[0] = s[0] * t[3] + s[1] * t[2] - s[2] * t[1] + s[3] * t[0]; - static const int x[4][3] = { { 0, 3, 1 }, { 1, 2, 1 }, { 2, 1, -1 }, { 3, 0, 1 } }; - sumOfProducts(S0, x); - // d[1] = s[1] * t[3] - s[0] * t[2] + s[2] * t[0] + s[3] * t[1]; - static const int y[4][3] = { { 1, 3, 1 }, { 0, 2, -1 }, { 2, 0, 1 }, { 3, 1, 1 } }; - sumOfProducts(S1, y); - // d[2] = s[0] * t[1] - s[1] * t[0] + s[2] * t[3] + s[3] * t[2]; - static const int z[4][3] = { { 0, 1, 1 }, { 1, 0, -1 }, { 2, 3, 1 }, { 3, 2, 1 } }; - sumOfProducts(fpr.V(temp3), z); - // d[3] = s[3] * t[3] - (s[0] * t[0] + s[1] * t[1] + s[2] * t[2]); - fp.FMUL(fpr.V(temp4), fpr.V(sregs[0]), fpr.V(tregs[0])); - for (int i = 1; i < 3; i++) { - fp.FMUL(S2, fpr.V(sregs[i]), fpr.V(tregs[i])); - fp.FADD(fpr.V(temp4), fpr.V(temp4), S2); - } - fp.FMUL(S2, fpr.V(sregs[3]), fpr.V(tregs[3])); - fp.FSUB(fpr.V(temp4), S2, fpr.V(temp4)); - - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT); - fp.FMOV(fpr.V(dregs[0]), S0); - fp.FMOV(fpr.V(dregs[1]), S1); - fp.FMOV(fpr.V(dregs[2]), fpr.V(temp3)); - fp.FMOV(fpr.V(dregs[3]), fpr.V(temp4)); - } - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vcmp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_COMP); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - VCondition cond = (VCondition)(op & 0xF); - - u8 sregs[4], tregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - - // Some, we just fall back to the interpreter. - // ES is just really equivalent to (value & 0x7F800000) == 0x7F800000. - - switch (cond) { - case VC_EZ: - case VC_NZ: - break; - default: - ; - } - - // First, let's get the trivial ones. - int affected_bits = (1 << 4) | (1 << 5); // 4 and 5 - - MOVI2R(SCRATCH1, 0); - for (int i = 0; i < n; ++i) { - // Let's only handle the easy ones, and fall back on the interpreter for the rest. - CCFlags flag = CC_AL; - switch (cond) { - case VC_FL: // c = 0; - break; - - case VC_TR: // c = 1 - if (i == 0) { - if (n == 1) { - MOVI2R(SCRATCH1, 0x31); - } else { - MOVI2R(SCRATCH1, 1ULL << i); - } - } else { - ORRI2R(SCRATCH1, SCRATCH1, 1ULL << i); - } - break; - - case VC_ES: // c = my_isnan(s[i]) || my_isinf(s[i]); break; // Tekken Dark Resurrection - case VC_NS: // c = !(my_isnan(s[i]) || my_isinf(s[i])); break; - // For these, we use the integer ALU as there is no support on ARM for testing for INF. - // Testing for nan or inf is the same as testing for &= 0x7F800000 == 0x7F800000. - // We need an extra temporary register so we store away SCRATCH1. - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, temp)); - fpr.MapRegV(sregs[i], 0); - MOVI2R(SCRATCH1, 0x7F800000); - fp.FMOV(SCRATCH2, fpr.V(sregs[i])); - AND(SCRATCH2, SCRATCH2, SCRATCH1); - CMP(SCRATCH2, SCRATCH1); // (SCRATCH2 & 0x7F800000) == 0x7F800000 - flag = cond == VC_ES ? CC_EQ : CC_NEQ; - LDR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, temp)); - break; - - case VC_EI: // c = my_isinf(s[i]); break; - case VC_NI: // c = !my_isinf(s[i]); break; - // |s| == inf, in the integer ALU like ES above. - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, temp)); - fpr.MapRegV(sregs[i], 0); - fp.FMOV(SCRATCH2, fpr.V(sregs[i])); - ANDI2R(SCRATCH2, SCRATCH2, 0x7FFFFFFF); - MOVI2R(SCRATCH1, 0x7F800000); - CMP(SCRATCH2, SCRATCH1); - flag = cond == VC_EI ? CC_EQ : CC_NEQ; - LDR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, temp)); - break; - - case VC_EN: // c = my_isnan(s[i]); break; // Tekken 6 - case VC_NN: // c = !my_isnan(s[i]); break; - // Only s counts: comparing it with itself is unordered exactly when it's NaN. - fpr.MapRegV(sregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(sregs[i])); - flag = cond == VC_EN ? CC_VS : CC_VC; // overflow = unordered - break; - - case VC_EQ: // c = s[i] == t[i] - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_EQ; - break; - - case VC_LT: // c = s[i] < t[i] - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_LO; - break; - - case VC_LE: // c = s[i] <= t[i]; - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_LS; - break; - - case VC_NE: // c = s[i] != t[i] - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_NEQ; - break; - - case VC_GE: // c = s[i] >= t[i] - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_GE; - break; - - case VC_GT: // c = s[i] > t[i] - fpr.MapInInV(sregs[i], tregs[i]); - fp.FCMP(fpr.V(sregs[i]), fpr.V(tregs[i])); - flag = CC_GT; - break; - - case VC_EZ: // c = s[i] == 0.0f || s[i] == -0.0f - fpr.MapRegV(sregs[i]); - fp.FCMP(fpr.V(sregs[i])); // vcmp(sregs[i], #0.0) - flag = CC_EQ; - break; - - case VC_NZ: // c = s[i] != 0 - fpr.MapRegV(sregs[i]); - fp.FCMP(fpr.V(sregs[i])); // vcmp(sregs[i], #0.0) - flag = CC_NEQ; - break; - - default: - DISABLE; - } - if (flag != CC_AL) { - FixupBranch b = B(InvertCond(flag)); - if (i == 0) { - if (n == 1) { - MOVI2R(SCRATCH1, 0x31); - } else { - MOVI2R(SCRATCH1, 1); // 1 << i, but i == 0 - } - } else { - ORRI2R(SCRATCH1, SCRATCH1, 1ULL << i); - } - SetJumpTarget(b); - } - - affected_bits |= 1 << i; - } - - // Aggregate the bits. Urgh, expensive. Can optimize for the case of one comparison, which is the most common - // after all. - if (n > 1) { - CMP(SCRATCH1, affected_bits & 0xF); - FixupBranch skip1 = B(CC_NEQ); - ORRI2R(SCRATCH1, SCRATCH1, 1 << 5); - SetJumpTarget(skip1); - - CMP(SCRATCH1, 0); - FixupBranch skip2 = B(CC_EQ); - ORRI2R(SCRATCH1, SCRATCH1, 1 << 4); - SetJumpTarget(skip2); - } - - gpr.MapReg(MIPS_REG_VFPUCC, MAP_DIRTY); - ANDI2R(gpr.R(MIPS_REG_VFPUCC), gpr.R(MIPS_REG_VFPUCC), ~affected_bits, SCRATCH2); - ORR(gpr.R(MIPS_REG_VFPUCC), gpr.R(MIPS_REG_VFPUCC), SCRATCH1); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vcmov(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_COMP); - // D is read as T, prefix included, which only the interpreter does. - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - int tf = (op >> 19) & 1; - int imm3 = (op >> 16) & 7; - - for (int i = 0; i < n; ++i) { - // Simplification: Disable if overlap unsafe - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs)) { - DISABLE; - } - } - - // 7 tests a CC bit that is always clear (cpu/vfpu/undefined). - if (imm3 != 6) { - // Test one bit of CC. This bit decides whether none or all subregisters are copied. - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_DIRTY); - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - gpr.MapReg(MIPS_REG_VFPUCC); - TSTI2R(gpr.R(MIPS_REG_VFPUCC), 1ULL << imm3); - // TODO: Use fsel? - FixupBranch b = B(tf ? CC_NEQ : CC_EQ); - for (int i = 0; i < n; i++) { - fp.FMOV(fpr.V(dregs[i]), fpr.V(sregs[i])); - } - SetJumpTarget(b); - } else { - // Look at the bottom four bits of CC to individually decide if the subregisters should be copied. - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_DIRTY); - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - gpr.MapReg(MIPS_REG_VFPUCC); - for (int i = 0; i < n; i++) { - TSTI2R(gpr.R(MIPS_REG_VFPUCC), 1ULL << i); - FixupBranch b = B(tf ? CC_NEQ : CC_EQ); - fp.FMOV(fpr.V(dregs[i]), fpr.V(sregs[i])); - SetJumpTarget(b); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Viim(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - if (js.HasUnknownPrefix()) { - DISABLE; - } - - u8 dreg; - GetVectorRegs(&dreg, V_Single, _VT); - - s32 imm = SignExtend16ToS32(op); - fpr.MapRegV(dreg, MAP_DIRTY | MAP_NOINIT); - fp.MOVI2F(fpr.V(dreg), (float)imm, SCRATCH1); - - ApplyPrefixD(&dreg, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vfim(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - if (js.HasUnknownPrefix()) { - DISABLE; - } - - u8 dreg; - GetVectorRegs(&dreg, V_Single, _VT); - - FP16 half; - half.u = op & 0xFFFF; - FP32 fval = half_to_float_fast5(half); - fpr.MapRegV(dreg, MAP_DIRTY | MAP_NOINIT); - fp.MOVI2F(fpr.V(dreg), fval.f, SCRATCH1); - - ApplyPrefixD(&dreg, V_Single); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vcst(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - if (js.HasUnknownPrefix()) { - DISABLE; - } - - int conNum = (op >> 16) & 0x1f; - int vd = _VD; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, _VD); - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - MOVP2R(SCRATCH1_64, (void *)&cst_constants[conNum]); - fp.LDR(32, INDEX_UNSIGNED, S0, SCRATCH1_64, 0); - for (int i = 0; i < n; ++i) - fp.FMOV(fpr.V(dregs[i]), S0); - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - static double SinCos(float angle) { - union { struct { float sin; float cos; }; double out; } sincos; - vfpu_sincos(angle, sincos.sin, sincos.cos); - return sincos.out; - } - - static double SinCosNegSin(float angle) { - union { struct { float sin; float cos; }; double out; } sincos; - vfpu_sincos(angle, sincos.sin, sincos.cos); - sincos.sin = -sincos.sin; - return sincos.out; - } - - void Arm64Jit::CompVrotShuffle(u8 *dregs, int imm, VectorSize sz, bool negSin) { - int n = GetNumVectorElements(sz); - char what[4] = { '0', '0', '0', '0' }; - if (((imm >> 2) & 3) == (imm & 3)) { - for (int i = 0; i < 4; i++) - what[i] = 'S'; - } - what[(imm >> 2) & 3] = 'S'; - what[imm & 3] = 'C'; - - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_DIRTY | MAP_NOINIT); - for (int i = 0; i < n; i++) { - switch (what[i]) { - case 'C': fp.FMOV(fpr.V(dregs[i]), S1); break; - case 'S': if (negSin) fp.FNEG(fpr.V(dregs[i]), S0); else fp.FMOV(fpr.V(dregs[i]), S0); break; - case '0': - { - fp.MOVI2F(fpr.V(dregs[i]), 0.0f); - break; - } - default: - ERROR_LOG(Log::JIT, "Bad what in vrot"); - break; - } - } - } - - // Very heavily used by FF:CC. Should be replaced by a fast approximation instead of - // calling the math library. - void Arm64Jit::Comp_VRot(MIPSOpcode op) { - // VRot probably doesn't accept prefixes anyway. - CONDITIONAL_DISABLE(VFPU_VEC); - // The prefixes apply to the sine but never to the cosine; leave that to the interpreter. - if (!js.HasNoPrefix()) { - DISABLE; - } - - int vd = _VD; - int vs = _VS; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 dregs[4]; - u8 dregs2[4]; - - MIPSOpcode nextOp = GetOffsetInstruction(1); - int vd2 = -1; - int imm2 = -1; - if ((nextOp >> 26) == 60 && ((nextOp >> 21) & 0x1F) == 29 && _VS == MIPS_GET_VS(nextOp)) { - // Pair of vrot. Let's join them. - vd2 = MIPS_GET_VD(nextOp); - imm2 = (nextOp >> 16) & 0x1f; - // NOTICE_LOG(Log::JIT, "Joint VFPU at %08x", js.blockStart); - } - u8 sreg; - GetVectorRegs(dregs, sz, vd); - if (vd2 >= 0) - GetVectorRegs(dregs2, sz, vd2); - GetVectorRegs(&sreg, V_Single, vs); - // With the angle in a destination lane, the cosine is taken of what was written there. - // The assembler refuses that, so leave it to the interpreter, and don't pair such a vrot. - for (int i = 0; i < n; i++) { - if (dregs[i] == sreg) { - DISABLE; - } - if (vd2 >= 0 && dregs2[i] == sreg) { - vd2 = -1; - } - } - - int imm = (op >> 16) & 0x1f; - - // Values in S8-S15 survive the call, so only the rest is flushed. - gpr.FlushBeforeCall(); - fpr.FlushBeforeCall(); - - // Don't need to SaveStaticRegs here as long as they are all in callee-save regs - this callee won't read them. - - bool negSin1 = (imm & 0x10) ? true : false; - - LoadVAfterCallFlush(S0, sreg); - QuickCallFunction(SCRATCH2_64, negSin1 ? (void *)&SinCosNegSin : (void *)&SinCos); - // Here, sin and cos are stored together in Q0.d. On ARM32 we could use it directly - // but with ARM64's register organization, we need to split it up. - fp.INS(32, Q1, 0, Q0, 1); - - CompVrotShuffle(dregs, imm, sz, false); - if (vd2 != -1) { - // If the negsin setting differs between the two joint invocations, we need to flip the second one. - bool negSin2 = (imm2 & 0x10) ? true : false; - CompVrotShuffle(dregs2, imm2, sz, negSin1 != negSin2); - EatInstruction(nextOp); - } - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vsgn(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - // Vector extract sign: +1 or -1 with the sign of s, or 0 when s is zero or denormal. NaN - // keeps its sign bit, like the rest. - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - fp.MOVI2F(S1, FLT_MIN, SCRATCH1); - fp.MOVI2F(S2, 0.0f, SCRATCH1); - for (int i = 0; i < n; i++) { - fpr.MapDirtyInV(tempregs[i], sregs[i]); - // MI (below the smallest normal) is false for NaN. - fp.FABS(S0, fpr.V(sregs[i])); - fp.FCMP(S0, S1); - fp.FMOV(SCRATCH1, fpr.V(sregs[i])); - ANDI2R(SCRATCH1, SCRATCH1, 0x80000000); - ORRI2R(SCRATCH1, SCRATCH1, 0x3F800000); - fp.FMOV(S3, SCRATCH1); - fp.FCSEL(fpr.V(tempregs[i]), S2, S3, CC_MI); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vocp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix() || (js.prefixS & VFPU_NEGATE(1, 1, 1, 1)) != 0) { - DISABLE; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - // This is a hack that modifies prefixes. We eat them later, so just overwrite. - // S prefix forces the negate flags. - js.prefixS |= 0x000F0000; - // T prefix forces constants on and regnum to 1. - // That means negate still works, and abs activates a different constant. - js.prefixT = (js.prefixT & ~0x000000FF) | 0x00000055 | 0x0000F000; - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - MIPSReg tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - fp.MOVI2F(S0, 1.0f, SCRATCH1); - for (int i = 0; i < n; ++i) { - fpr.MapDirtyInInV(tempregs[i], sregs[i], tregs[i]); - fp.FADD(fpr.V(tempregs[i]), fpr.V(tregs[i]), fpr.V(sregs[i])); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapDirtyInV(dregs[i], tempregs[i]); - fp.FMOV(fpr.V(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_ColorConv(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) { - DISABLE; - } - - // vt4444, vt5551, vt5650: four 8888 colors from a quad, whatever the size, into 16 bits each, - // two to a word. - const int type = (op >> 16) & 3; - if (type == 0) { - DISABLE; - } - VectorSize isz = GetVecSize(op); - VectorSize outSz = isz == V_Single ? V_Single : V_Pair; - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, V_Quad, _VS); - GetVectorRegsPrefixD(dregs, outSz, _VD); - - const ARM64Reg t = gpr.GetAndLockTempR(); - const ARM64Reg word = gpr.GetAndLockTempR(); - const ARM64Reg in = SCRATCH1, col = SCRATCH2; - // Into S0-S1 first, since d may overlap s. - for (int i = 0; i < 4; i++) { - fpr.MapRegV(sregs[i]); - fp.FMOV(in, fpr.V(sregs[i])); - switch (type) { - case 1: // 4444: the top four bits of each channel. - UBFX(col, in, 4, 4); - UBFX(t, in, 12, 4); - BFI(col, t, 4, 4); - UBFX(t, in, 20, 4); - BFI(col, t, 8, 4); - UBFX(t, in, 28, 4); - BFI(col, t, 12, 4); - break; - case 2: // 5551 - UBFX(col, in, 3, 5); - UBFX(t, in, 11, 5); - BFI(col, t, 5, 5); - UBFX(t, in, 19, 5); - BFI(col, t, 10, 5); - UBFX(t, in, 31, 1); - BFI(col, t, 15, 1); - break; - case 3: // 565, no alpha. - UBFX(col, in, 3, 5); - UBFX(t, in, 10, 6); - BFI(col, t, 5, 6); - UBFX(t, in, 19, 5); - BFI(col, t, 11, 5); - break; - } - if ((i & 1) == 0) { - MOV(word, col); - } else { - BFI(word, col, 16, 16); - fp.FMOV((ARM64Reg)(S0 + i / 2), word); - } - } - - const int nOut = outSz == V_Single ? 1 : 2; - for (int i = 0; i < nOut; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - fp.FMOV(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } - - ApplyPrefixD(dregs, outSz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - gpr.ReleaseSpillLocksAndDiscardTemps(); - } - - void Arm64Jit::Comp_Vbfy(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix() || (js.prefixS & VFPU_NEGATE(1, 1, 1, 1)) != 0) { - DISABLE; - } - - // Butterfly. vbfy1: d[2N] = s[2N] + s[2N+1], d[2N+1] = s[2N] - s[2N+1]. - // vbfy2: d[0] = s[0] + s[2], d[1] = s[1] + s[3], d[2] = s[0] - s[2], d[3] = s[1] - s[3]. - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - const int subop = (op >> 16) & 0x1F; - if (!(subop == 3 && n == 4) && !(subop == 2 && (n == 2 || n == 4))) { - DISABLE; - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Into S0-S3 first, since d may overlap s. - fpr.MapRegsAndSpillLockV(sregs, sz, 0); - if (subop == 3) { - fp.FADD(S0, fpr.V(sregs[0]), fpr.V(sregs[2])); - fp.FADD(S1, fpr.V(sregs[1]), fpr.V(sregs[3])); - fp.FSUB(S2, fpr.V(sregs[0]), fpr.V(sregs[2])); - fp.FSUB(S3, fpr.V(sregs[1]), fpr.V(sregs[3])); - } else { - fp.FADD(S0, fpr.V(sregs[0]), fpr.V(sregs[1])); - fp.FSUB(S1, fpr.V(sregs[0]), fpr.V(sregs[1])); - if (n == 4) { - fp.FADD(S2, fpr.V(sregs[2]), fpr.V(sregs[3])); - fp.FSUB(S3, fpr.V(sregs[2]), fpr.V(sregs[3])); - } - } - - fpr.MapRegsAndSpillLockV(dregs, sz, MAP_NOINIT); - for (int i = 0; i < n; i++) { - fp.FMOV(fpr.V(dregs[i]), (ARM64Reg)(S0 + i)); - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocksAndDiscardTemps(); - } -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64Jit.cpp b/Core/MIPS/ARM64/Arm64Jit.cpp deleted file mode 100644 index db3e7a48e4..0000000000 --- a/Core/MIPS/ARM64/Arm64Jit.cpp +++ /dev/null @@ -1,790 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" - -#if PPSSPP_ARCH(ARM64) - -#include "Common/Profiler/Profiler.h" -#include "Common/Log.h" -#include "Common/Serialize/Serializer.h" -#include "Common/Serialize/SerializeFuncs.h" -#include "Common/CPUDetect.h" -#include "Common/StringUtils.h" - -#include "Core/Reporting.h" -#include "Core/Core.h" -#include "Core/CoreTiming.h" -#include "Core/Debugger/Breakpoints.h" -#include "Core/Debugger/SymbolMap.h" -#include "Core/MemMap.h" - -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/Interpreter.h" -#include "Core/MIPS/MIPSTables.h" -#include "Core/HLE/ReplaceTables.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Core/MIPS/ARM64/Arm64RegCacheFPU.h" - -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/JitCommon/JitCommon.h" - -using namespace Arm64JitConstants; - -static void DisassembleArm64Print(const u8 *data, int size) { - std::vector lines = DisassembleArm64(data, size); - for (auto s : lines) { - INFO_LOG(Log::JIT, "%s", s.c_str()); - } - /* - INFO_LOG(Log::JIT, "+++"); - // A format friendly to Online Disassembler which gets endianness wrong - for (size_t i = 0; i < lines.size(); i++) { - uint32_t opcode = ((const uint32_t *)data)[i]; - INFO_LOG(Log::JIT, "%d/%d: %08x", (int)(i+1), (int)lines.size(), swap32(opcode)); - } - INFO_LOG(Log::JIT, "==="); - INFO_LOG(Log::JIT, "===");*/ -} - -static u32 JitBreakpoint(uint32_t addr) { - // Should we skip this breakpoint? - if (g_breakpoints.CheckSkipFirst() == currentMIPS->pc || g_breakpoints.CheckSkipFirst() == addr) - return 0; - - BreakAction result = g_breakpoints.ExecBreakPoint(addr); - if ((result & BREAK_ACTION_PAUSE) == 0) - return 0; - - return 1; -} - -static u32 JitMemCheck(u32 pc) { - if (g_breakpoints.CheckSkipFirst() == currentMIPS->pc) - return 0; - - // Note: pc may be the delay slot. - const auto op = Memory::Read_Instruction(pc); - s32 offset = SignExtend16ToS32(op & 0xFFFF); - if (MIPSGetInfo(op) & IS_VFPU) - offset &= 0xFFFC; - u32 addr = currentMIPS->r[MIPS_GET_RS(op)] + offset; - - g_breakpoints.ExecOpMemCheck(addr, pc); - return coreState == CORE_RUNNING_CPU || coreState == CORE_NEXTFRAME ? 0 : 1; -} - -namespace MIPSComp -{ -using namespace Arm64Gen; -using namespace Arm64JitConstants; - -Arm64Jit::Arm64Jit(MIPSState *mipsState) : blocks(mipsState, this), gpr(mipsState, &js, &jo), fpr(mipsState, &js, &jo), mips_(mipsState), fp(this) { - // Automatically disable incompatible options. - if (((intptr_t)Memory::base & 0x00000000FFFFFFFFUL) != 0) { - jo.enablePointerify = false; - } - -#ifdef MASKED_PSP_MEMORY - jo.enablePointerify = false; -#endif - - logBlocks = 0; - dontLogBlocks = 0; - blocks.Init(); - gpr.SetEmitter(this); - fpr.SetEmitter(this, &fp); - AllocCodeSpace(1024 * 1024 * 16); // 32MB is the absolute max because that's what an ARM branch instruction can reach, backwards and forwards. - GenerateFixedCode(jo); - js.startDefaultPrefix = mips_->HasDefaultPrefix(); - js.currentRoundingFunc = convertS0ToSCRATCH1[mips_->fcr31 & 3]; - - // The debugger sets this so that "go" on a breakpoint will actually... go. - // But if they reset, we can end up hitting it by mistake, since it's based on PC and ticks. - g_breakpoints.SetSkipFirst(0); -} - -Arm64Jit::~Arm64Jit() { -} - -void Arm64Jit::DoState(PointerWrap &p) { - auto s = p.Section("Jit", 1, 2); - if (!s) - return; - - Do(p, js.startDefaultPrefix); - if (p.mode == PointerWrap::MODE_READ && !js.startDefaultPrefix) { - WARN_LOG(Log::CPU, "Jit: An uneaten prefix was previously detected. Jitting in unknown-prefix mode."); - } - if (s >= 2) { - Do(p, js.hasSetRounding); - if (p.mode == PointerWrap::MODE_READ) { - js.lastSetRounding = 0; - } - } else { - js.hasSetRounding = 1; - } - - // Note: we can't update the currentRoundingFunc here because fcr31 wasn't loaded yet. - - // The debugger sets this so that "go" on a breakpoint will actually... go. - // But if they reset, we can end up hitting it by mistake, since it's based on PC and ticks. - g_breakpoints.SetSkipFirst(0); -} - -void Arm64Jit::UpdateFCR31() { - js.currentRoundingFunc = convertS0ToSCRATCH1[mips_->fcr31 & 3]; - // fcr31 can reach a non-default mode without a compiled ctc1 (an interpreted one, a savestate, - // a core switch, a thread switch). Blocks compiled before that don't switch modes around - // syscalls or conversions, so they're thrown away, at the next safe point. - if ((mips_->fcr31 & 0x01000003) && !js.hasSetRounding) { - js.hasSetRounding = true; - js.lastSetRounding = true; - mips_->ClearJitCacheDeferred(); - } -} - -void Arm64Jit::FlushAll() { - gpr.FlushAll(); - fpr.FlushAll(); - FlushPrefixV(); -} - -void Arm64Jit::FlushPrefixV() { - if (js.startDefaultPrefix && !js.blockWrotePrefixes && js.HasNoPrefix()) { - // They started default, we never modified in memory, and they're default now. - // No reason to modify memory. This is common at end of blocks. Just clear dirty. - js.prefixSFlag = (JitState::PrefixState)(js.prefixSFlag & ~JitState::PREFIX_DIRTY); - js.prefixTFlag = (JitState::PrefixState)(js.prefixTFlag & ~JitState::PREFIX_DIRTY); - js.prefixDFlag = (JitState::PrefixState)(js.prefixDFlag & ~JitState::PREFIX_DIRTY); - return; - } - - if ((js.prefixSFlag & JitState::PREFIX_DIRTY) != 0) { - MOVI2R(SCRATCH1, js.prefixS); - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, vfpuCtrl[VFPU_CTRL_SPREFIX])); - js.prefixSFlag = (JitState::PrefixState) (js.prefixSFlag & ~JitState::PREFIX_DIRTY); - } - - if ((js.prefixTFlag & JitState::PREFIX_DIRTY) != 0) { - MOVI2R(SCRATCH1, js.prefixT); - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, vfpuCtrl[VFPU_CTRL_TPREFIX])); - js.prefixTFlag = (JitState::PrefixState) (js.prefixTFlag & ~JitState::PREFIX_DIRTY); - } - - if ((js.prefixDFlag & JitState::PREFIX_DIRTY) != 0) { - MOVI2R(SCRATCH1, js.prefixD); - STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, vfpuCtrl[VFPU_CTRL_DPREFIX])); - js.prefixDFlag = (JitState::PrefixState) (js.prefixDFlag & ~JitState::PREFIX_DIRTY); - } - - // If we got here, we must've written prefixes to memory in this block. - js.blockWrotePrefixes = true; -} - -void Arm64Jit::ClearCache() { - INFO_LOG(Log::JIT, "ARM64Jit: Clearing the cache!"); - blocks.Clear(); - ClearCodeSpace(jitStartOffset); - FlushIcacheSection(region + jitStartOffset, region + region_size - jitStartOffset); -} - -void Arm64Jit::InvalidateCacheAt(u32 em_address, int length) { - if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { - blocks.InvalidateICache(em_address, length); - } -} - -void Arm64Jit::EatInstruction(MIPSOpcode op) { - MIPSInfo info = MIPSGetInfo(op); - if (info & DELAYSLOT) { - ERROR_LOG_REPORT_ONCE(ateDelaySlot, Log::JIT, "Ate a branch op."); - } - if (js.inDelaySlot) { - ERROR_LOG_REPORT_ONCE(ateInDelaySlot, Log::JIT, "Ate an instruction inside a delay slot."); - } - - CheckJitBreakpoint(GetCompilerPC() + 4, 0); - js.numInstructions++; - js.compilerPC += 4; - js.downcountAmount += MIPSGetInstructionCycleEstimate(op); -} - -void Arm64Jit::CompileDelaySlot(int flags) { - // Need to offset the downcount which was already incremented for the branch + delay slot. - CheckJitBreakpoint(GetCompilerPC() + 4, -2); - - // preserve flag around the delay slot! Maybe this is not always necessary on ARM where - // we can (mostly) control whether we set the flag or not. Of course, if someone puts an slt in to the - // delay slot, we're screwed. - if (flags & DELAYSLOT_SAFE) - MRS(FLAGTEMPREG, FIELD_NZCV); // Save flags register. FLAGTEMPREG is preserved through function calls and is not allocated. - - js.inDelaySlot = true; - MIPSOpcode op = GetOffsetInstruction(1); - MIPSCompileOp(op, this); - js.inDelaySlot = false; - - if (flags & DELAYSLOT_FLUSH) - FlushAll(); - if (flags & DELAYSLOT_SAFE) - _MSR(FIELD_NZCV, FLAGTEMPREG); // Restore flags register -} - -void Arm64Jit::Compile(u32 em_address) { - PROFILE_THIS_SCOPE("jitc"); - - if (!Memory::IsValid4AlignedAddress(em_address)) { - Core_ExecException(em_address, em_address, ExecExceptionType::JUMP); - return; - } - - if (GetSpaceLeft() < 0x10000 || blocks.IsFull()) { - INFO_LOG(Log::JIT, "Space left: %d", (int)GetSpaceLeft()); - ClearCache(); - } - - BeginWrite(JitBlockCache::MAX_BLOCK_INSTRUCTIONS * 16); - - // To debug really wacky JIT issues, it can be a good idea to bisect the block number, - // and disable parts of the jit using jo.disableFlags for certain ranges of blocks. - - int block_num = blocks.AllocateBlock(em_address); - JitBlock *b = blocks.GetBlock(block_num); - b->DoIntegrityCheck(em_address, block_num, "BeforeDoJit"); - DoJit(em_address, b); - b->DoIntegrityCheck(em_address, block_num, "BeforeFinalize"); - blocks.FinalizeBlock(block_num, jo.enableBlocklink); - b->DoIntegrityCheck(em_address, block_num, "AfterFinalize"); - EndWrite(); - _dbg_assert_(js.nextExit <= 2); - - // Don't forget to zap the newly written instructions in the instruction cache! - FlushIcache(); - - bool cleanSlate = false; - - if (js.hasSetRounding && !js.lastSetRounding) { - WARN_LOG(Log::JIT, "Detected rounding mode usage, rebuilding jit with checks"); - // Won't loop, since hasSetRounding is only ever set to 1. - js.lastSetRounding = js.hasSetRounding; - cleanSlate = true; - } - - // Drat. The VFPU hit an uneaten prefix at the end of a block. - if (js.startDefaultPrefix && js.MayHavePrefix()) { - WARN_LOG_REPORT(Log::JIT, "An uneaten prefix at end of block: %08x", GetCompilerPC() - 4); - js.LogPrefix(); - - // Let's try that one more time. We won't get back here because we toggled the value. - js.startDefaultPrefix = false; - // TODO ARM64: This crashes. - // TODO: Really? Need to fix that in such a case. Wonder what games are affected. - //cleanSlate = true; - } - - if (cleanSlate) { - // Our assumptions are all wrong so it's clean-slate time. - ClearCache(); - Compile(em_address); - } -} - -void Arm64Jit::RunLoopUntil(u64 globalticks) { - PROFILE_THIS_SCOPE("jit"); - ((void (*)())enterDispatcher)(); -} - -u32 Arm64Jit::GetCompilerPC() { - return js.compilerPC; -} - -MIPSOpcode Arm64Jit::GetOffsetInstruction(int offset) { - const u32 addr = GetCompilerPC() + 4 * offset; - return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); -} - -void Arm64Jit::DoJit(u32 em_address, JitBlock *b) { - js.Begin(b); - - // We add a downcount flag check before the block, used when entering from a linked block. - // The last block decremented downcounter, and the flag should still be available. - // Got three variants here of where we position the code, needs detailed benchmarking. - - FixupBranch bail; - if (jo.useBackJump) { - // Moves the MOVI2R and B *before* checkedEntry, and just branch backwards there. - // Speedup seems to be zero unfortunately but I guess it may vary from device to device. - // Not intrusive so keeping it around here to experiment with, may help on ARMv6 due to - // large/slow construction of 32-bit immediates? - const u8 *backJump = GetCodePtr(); - MOVI2R(SCRATCH1, js.blockStart); - B((const void *)outerLoopPCInSCRATCH1); - b->checkedEntry = GetCodePtr(); - B(CC_LT, backJump); - } else if (jo.useForwardJump) { - b->checkedEntry = GetCodePtr(); - bail = B(CC_LT); - } else if (jo.enableBlocklink) { - b->checkedEntry = GetCodePtr(); - MOVI2R(SCRATCH1, js.blockStart); - FixupBranch skip = B(CC_GE); - B((const void *)outerLoopPCInSCRATCH1); - SetJumpTarget(skip); - } else { - // No block linking, no need to add headers to blocks. - } - - b->normalEntry = GetCodePtr(); - - gpr.Start(); - fpr.Start(); - - js.numInstructions = 0; - while (js.compiling) { - gpr.SetCompilerPC(GetCompilerPC()); // Let it know for log messages - // Jit breakpoints are quite fast, so let's do them in release too. - CheckJitBreakpoint(GetCompilerPC(), 0); - - MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); - js.downcountAmount += MIPSGetInstructionCycleEstimate(inst); - - MIPSCompileOp(inst, this); - - js.compilerPC += 4; - js.numInstructions++; - - if (jo.Disabled(JitDisable::REGALLOC_GPR)) { - gpr.FlushAll(); - } - if (jo.Disabled(JitDisable::REGALLOC_FPR)) { - fpr.FlushAll(); - FlushPrefixV(); - } - - // Safety check, in case we get a bunch of really large jit ops without a lot of branching. - if (GetSpaceLeft() < 0x800 || js.numInstructions >= JitBlockCache::MAX_BLOCK_INSTRUCTIONS) { - FlushAll(); - WriteExit(GetCompilerPC(), js.nextExit++); - js.compiling = false; - } - } - - if (jo.useForwardJump) { - SetJumpTarget(bail); - MOVI2R(SCRATCH1, js.blockStart); - B((const void *)outerLoopPCInSCRATCH1); - } - - char temp[256]; - if (logBlocks > 0 && dontLogBlocks == 0) { - INFO_LOG(Log::JIT, "=============== mips %d ===============", blocks.GetNumBlocks()); - for (u32 cpc = b->originalAddress; cpc != GetCompilerPC() + 4; cpc += 4) { - MIPSDisAsm(Memory::ReadUnchecked_Instruction(cpc), cpc, temp, sizeof(temp), true); - INFO_LOG(Log::JIT, "M: %08x %s", cpc, temp); - } - } - - b->codeSize = GetCodePtr() - b->normalEntry; - if (logBlocks > 0 && dontLogBlocks == 0) { - INFO_LOG(Log::JIT, "=============== ARM (%d instructions -> %d bytes) ===============", js.numInstructions, b->codeSize); - DisassembleArm64Print(b->normalEntry, GetCodePtr() - b->normalEntry); - } - if (logBlocks > 0) - logBlocks--; - if (dontLogBlocks > 0) - dontLogBlocks--; - - b->originalSize = js.numInstructions; -} - -bool Arm64Jit::DescribeCodePtr(const u8 *ptr, std::string &name) { - // Used in disassembly viewer. - if (ptr == applyRoundingMode) - name = "applyRoundingMode"; - else if (ptr == updateRoundingMode) - name = "updateRoundingMode"; - else if (ptr == dispatcher) - name = "dispatcher"; - else if (ptr == dispatcherPCInSCRATCH1) - name = "dispatcher (PC in SCRATCH1)"; - else if (ptr == dispatcherNoCheck) - name = "dispatcherNoCheck"; - else if (ptr == enterDispatcher) - name = "enterDispatcher"; - else if (ptr == restoreRoundingMode) - name = "restoreRoundingMode"; - else if (ptr == saveStaticRegisters) - name = "saveStaticRegisters"; - else if (ptr == loadStaticRegisters) - name = "loadStaticRegisters"; - else { - u32 addr = blocks.GetAddressFromBlockPtr(ptr); - // Returns 0 when it's valid, but unknown. - if (addr == 0) { - name = "(unknown or deleted block)"; - return true; - } else if (addr != (u32)-1) { - name = "(outside space)"; - return true; - } - - int number = blocks.GetBlockNumberFromAddress(addr); - if (number != -1) { - const JitBlock *block = blocks.GetBlock(number); - if (block) { - name = StringFromFormat("(block %d at %08x)", number, block->originalAddress); - return true; - } - } - return false; - } - return true; -} - -void Arm64Jit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) { - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE); - } - ARM64XEmitter emit(GetCodePtrFromWritablePtr(exitPoint), exitPoint); - emit.B(checkedEntry); - // TODO: Write stuff after, convering up the now-unused instructions. - emit.FlushIcache(); - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_EXEC); - } -} - -void Arm64Jit::UnlinkBlock(u8 *checkedEntry, u32 originalAddress) { - // Send anyone who tries to run this block back to the dispatcher. - // Not entirely ideal, but .. works. - // Spurious entrances from previously linked blocks can only come through checkedEntry - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_WRITE); - } - - ARM64XEmitter emit(GetCodePtrFromWritablePtr(checkedEntry), checkedEntry); - emit.MOVI2R(SCRATCH1, originalAddress); - emit.STR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, pc)); - emit.B(MIPSComp::jit->GetDispatcher()); - emit.FlushIcache(); - - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_EXEC); - } -} - -void Arm64Jit::Comp_ReplacementFunc(MIPSOpcode op) -{ - // We get here if we execute the first instruction of a replaced function. This means - // that we do need to return to RA. - - // Inlined function calls (caught in jal) are handled differently. - - // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. - if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { - Comp_Generic(op); - return; - } - - int index = op.encoding & MIPS_REPLACEMENT_INDEX_MASK; - - const ReplacementTableEntry *entry = GetReplacementFunc(index); - if (!entry) { - ERROR_LOG_REPORT_ONCE(replFunc, Log::HLE, "Invalid replacement op %08x at %08x", op.encoding, js.compilerPC); - // TODO: What should we do here? We're way off in the weeds probably. - return; - } - - u32 funcSize = g_symbolMap->GetFunctionSize(GetCompilerPC()); - bool disabled = (entry->flags & REPFLAG_DISABLED) != 0; - if (!disabled && funcSize != SymbolMap::INVALID_ADDRESS && funcSize > sizeof(u32)) { - // We don't need to disable hooks, the code will still run. - if ((entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) == 0) { - // Any breakpoint at the func entry was already tripped, so we can still run the replacement. - // That's a common case - just to see how often the replacement hits. - disabled = g_breakpoints.RangeContainsBreakPoint(GetCompilerPC() + sizeof(u32), funcSize - sizeof(u32)); - } - } - - if (disabled) { - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); - } else if (entry->jitReplaceFunc) { - MIPSReplaceFunc repl = entry->jitReplaceFunc; - int cycles = (this->*repl)(); - - if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { - // Compile the original instruction at this address. We ignore cycles for hooks. - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); - } else { - FlushAll(); - // Flushed, so R1 is safe. - LDR(INDEX_UNSIGNED, SCRATCH1, CTXREG, MIPS_REG_RA * 4); - js.downcountAmount += cycles; - WriteExitDestInR(SCRATCH1); - js.compiling = false; - } - } else if (entry->replaceFunc) { - FlushAll(); - SaveStaticRegisters(); - RestoreRoundingMode(); - MOVI2R(SCRATCH1, GetCompilerPC()); - MovToPC(SCRATCH1); - - // Standard function call, nothing fancy. - // The function returns the number of cycles it took in EAX. - QuickCallFunction(SCRATCH1_64, (const void *)(entry->replaceFunc)); - - if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { - // Compile the original instruction at this address. We ignore cycles for hooks. - ApplyRoundingMode(); - LoadStaticRegisters(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); - } else { - ApplyRoundingMode(); - LoadStaticRegisters(); - - CMPI2R(W0, 0); - FixupBranch positive = B(CC_GE); - - NEG(W0, W0); - MovFromPC(W1); - FixupBranch done = B(); - - SetJumpTarget(positive); - LDR(INDEX_UNSIGNED, W1, CTXREG, MIPS_REG_RA * 4); - - SetJumpTarget(done); - WriteDownCountR(W0); - WriteExitDestInR(W1); - js.compiling = false; - } - } else { - ERROR_LOG(Log::HLE, "Replacement function %s has neither jit nor regular impl", entry->name); - } -} - -void Arm64Jit::Comp_Generic(MIPSOpcode op) { - FlushAll(); - MIPSInterpretFunc func = MIPSGetInterpretFunc(op); - const MIPSInfo info = MIPSGetInfo(op); - if (func) { - SaveStaticRegisters(); - // Interpreted ops run in the guest's rounding mode, like on the interpreter. A syscall is host code - // (like a compiled one), and only ctc1 to fcr31 changes the mode, so only those need it switched. - const bool isSyscall = (info & IS_SYSCALL) != 0; - const bool setsMode = MIPS_IS_CTC1_FCR31(op.encoding); - if (isSyscall) - RestoreRoundingMode(); - if (setsMode) - UpdateRoundingMode(); - MOVI2R(SCRATCH1, GetCompilerPC()); - MovToPC(SCRATCH1); - MOVI2R(W0, op.encoding); - QuickCallFunction(SCRATCH2_64, (void *)&MIPSInterpretTrampoline); - if (setsMode) - RestoreRoundingMode(true); - if (isSyscall || setsMode) - ApplyRoundingMode(setsMode); - LoadStaticRegisters(); - } - - if ((info & IS_VFPU) != 0 && (info & VFPU_NO_PREFIX) == 0) { - // If it does eat them, it'll happen in MIPSCompileOp(). - if ((info & OUT_EAT_PREFIX) == 0) - js.PrefixUnknown(); - - // Even if DISABLE'd, we want to set this flag so we overwrite. - if ((info & OUT_VFPU_PREFIX) != 0) - js.blockWrotePrefixes = true; - } -} - -void Arm64Jit::MovFromPC(ARM64Reg r) { - LDR(INDEX_UNSIGNED, r, CTXREG, offsetof(MIPSState, pc)); -} - -void Arm64Jit::MovToPC(ARM64Reg r) { - STR(INDEX_UNSIGNED, r, CTXREG, offsetof(MIPSState, pc)); -} - -// Should not really be necessary except when entering Advance -void Arm64Jit::SaveStaticRegisters() { - if (jo.useStaticAlloc) { - QuickCallFunction(SCRATCH2_64, saveStaticRegisters); - } else { - // Inline the single operation - STR(INDEX_UNSIGNED, DOWNCOUNTREG, CTXREG, offsetof(MIPSState, downcount)); - } -} - -void Arm64Jit::LoadStaticRegisters() { - if (jo.useStaticAlloc) { - QuickCallFunction(SCRATCH2_64, loadStaticRegisters); - } else { - LDR(INDEX_UNSIGNED, DOWNCOUNTREG, CTXREG, offsetof(MIPSState, downcount)); - } -} - -void Arm64Jit::WriteDownCount(int offset, bool updateFlags) { - int theDowncount = js.downcountAmount + offset; - if (updateFlags) { - SUBSI2R(DOWNCOUNTREG, DOWNCOUNTREG, theDowncount, SCRATCH1); - } else { - SUBI2R(DOWNCOUNTREG, DOWNCOUNTREG, theDowncount, SCRATCH1); - } -} - -void Arm64Jit::WriteDownCountR(ARM64Reg reg, bool updateFlags) { - if (updateFlags) { - SUBS(DOWNCOUNTREG, DOWNCOUNTREG, reg); - } else { - SUB(DOWNCOUNTREG, DOWNCOUNTREG, reg); - } -} - -// Destroys SCRATCH2 -void Arm64Jit::RestoreRoundingMode(bool force) { - // If the game has never set an interesting rounding mode, we can safely skip this. - if (force || js.hasSetRounding) { - QuickCallFunction(SCRATCH2_64, restoreRoundingMode); - } -} - -// Destroys SCRATCH1 and SCRATCH2 -void Arm64Jit::ApplyRoundingMode(bool force) { - // If the game has never set an interesting rounding mode, we can safely skip this. - if (force || js.hasSetRounding) { - QuickCallFunction(SCRATCH2_64, applyRoundingMode); - } -} - -// Destroys SCRATCH1 and SCRATCH2 -void Arm64Jit::UpdateRoundingMode(u32 fcr31) { - // We must set js.hasSetRounding at compile time, or this block will use the wrong rounding mode. - // The fcr31 parameter is -1 when not known at compile time, so we just assume it was changed. - if (fcr31 & 0x01000003) { - js.hasSetRounding = true; - } - QuickCallFunction(SCRATCH2_64, updateRoundingMode); -} - -// IDEA - could have a WriteDualExit that takes two destinations and two condition flags, -// and just have conditional that set PC "twice". This only works when we fall back to dispatcher -// though, as we need to have the SUBS flag set in the end. So with block linking in the mix, -// I don't think this gives us that much benefit. -void Arm64Jit::WriteExit(u32 destination, int exit_num) { - // NOTE: Can't blindly check for bad destination addresses here, sometimes exits with bad destinations are written intentionally (like breaks). - _assert_msg_(exit_num < MAX_JIT_BLOCK_EXITS, "Expected a valid exit_num. dest=%08x", destination); - - // NOTE: Can't blindly check for bad destination addresses here, sometimes exits with bad destinations are written intentionally (like breaks). - WriteDownCount(); - //If nobody has taken care of this yet (this can be removed when all branches are done) - JitBlock *b = js.curBlock; - b->exitAddress[exit_num] = destination; - b->exitPtrs[exit_num] = GetWritableCodePtr(); - - // Link opportunity! - int block = blocks.GetBlockNumberFromStartAddress(destination); - if (block >= 0 && jo.enableBlocklink) { - // The target block exists! Directly link to its checked entrypoint. - B(blocks.GetBlock(block)->checkedEntry); - b->linkStatus[exit_num] = true; - } else { - MOVI2R(SCRATCH1, destination); - B((const void *)dispatcherPCInSCRATCH1); - } -} - -void Arm64Jit::WriteExitDestInR(ARM64Reg Reg) { - // TODO: If not fast memory, check for invalid address in reg and trigger exception. - MovToPC(Reg); - WriteDownCount(); - // TODO: shouldn't need an indirect branch here... - B((const void *)dispatcher); -} - -void Arm64Jit::WriteSyscallExit() { - WriteDownCount(); - B((const void *)dispatcherCheckCoreState); -} - -bool Arm64Jit::CheckJitBreakpoint(u32 addr, int downcountOffset) { - if (g_breakpoints.NeedsBreakCheckAt(addr)) { - MRS(FLAGTEMPREG, FIELD_NZCV); - FlushAll(); - MOVI2R(SCRATCH1, GetCompilerPC()); - MovToPC(SCRATCH1); - SaveStaticRegisters(); - RestoreRoundingMode(); - MOVI2R(W0, addr); - QuickCallFunction(SCRATCH1_64, &JitBreakpoint); - - // If 0, the conditional breakpoint wasn't taken. - CMPI2R(W0, 0); - FixupBranch skip = B(CC_EQ); - WriteDownCount(downcountOffset); - ApplyRoundingMode(); - LoadStaticRegisters(); - B((const void *)dispatcherCheckCoreState); - SetJumpTarget(skip); - - ApplyRoundingMode(); - LoadStaticRegisters(); - _MSR(FIELD_NZCV, FLAGTEMPREG); - return true; - } - - return false; -} - -bool Arm64Jit::CheckMemoryBreakpoint(int instructionOffset) { - if (g_breakpoints.HasMemChecks()) { - int off = instructionOffset + (js.inDelaySlot ? 1 : 0); - - MRS(FLAGTEMPREG, FIELD_NZCV); - FlushAll(); - RestoreRoundingMode(); - MOVI2R(W0, GetCompilerPC()); - MovToPC(W0); - if (off != 0) - ADDI2R(W0, W0, off * 4); - QuickCallFunction(SCRATCH2_64, &JitMemCheck); - - // If 0, the breakpoint wasn't tripped. - CMPI2R(W0, 0); - FixupBranch skip = B(CC_EQ); - WriteDownCount(-1 - off); - ApplyRoundingMode(); - B((const void *)dispatcherCheckCoreState); - SetJumpTarget(skip); - - ApplyRoundingMode(); - _MSR(FIELD_NZCV, FLAGTEMPREG); - return true; - } - - return false; -} - -void Arm64Jit::Comp_DoNothing(MIPSOpcode op) { } - -} // namespace - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64Jit.h b/Core/MIPS/ARM64/Arm64Jit.h deleted file mode 100644 index ba7e7a19cd..0000000000 --- a/Core/MIPS/ARM64/Arm64Jit.h +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include "Common/CPUDetect.h" -#include "Common/ArmCommon.h" -#include "Common/Arm64Emitter.h" -#include "Core/MIPS/JitCommon/JitState.h" -#include "Core/MIPS/JitCommon/JitBlockCache.h" -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Core/MIPS/ARM64/Arm64RegCacheFPU.h" -#include "Core/MIPS/MIPSVFPUUtils.h" - -#ifndef offsetof -#include "stddef.h" -#endif - -namespace MIPSComp { - -class Arm64Jit : public Arm64Gen::ARM64CodeBlock, public JitInterface, public MIPSFrontendInterface { -public: - Arm64Jit(MIPSState *mipsState); - virtual ~Arm64Jit(); - - void DoState(PointerWrap &p) override; - - const JitOptions &GetJitOptions() { return jo; } - - // Compiled ops should ignore delay slots - // the compiler will take care of them by itself - // OR NOT - void Comp_Generic(MIPSOpcode op) override; - - void RunLoopUntil(u64 globalticks) override; - - void Compile(u32 em_address) override; // Compiles a block at current MIPS PC - void DoJit(u32 em_address, JitBlock *b); - - const u8 *GetCrashHandler() const override { return crashHandler; } - bool CodeInRange(const u8 *ptr) const override { return IsInSpace(ptr); } - bool DescribeCodePtr(const u8 *ptr, std::string &name) override; - - void Comp_ReplacementFunc(MIPSOpcode op) override; - - // Ops - void Comp_ITypeMem(MIPSOpcode op) override; - void Comp_StoreSync(MIPSOpcode op) override; - void Comp_Cache(MIPSOpcode op) override; - - void Comp_RelBranch(MIPSOpcode op) override; - void Comp_RelBranchRI(MIPSOpcode op) override; - void Comp_FPUBranch(MIPSOpcode op) override; - void Comp_FPULS(MIPSOpcode op) override; - void Comp_FPUComp(MIPSOpcode op) override; - void Comp_Jump(MIPSOpcode op) override; - void Comp_JumpReg(MIPSOpcode op) override; - void Comp_Syscall(MIPSOpcode op) override; - void Comp_Break(MIPSOpcode op) override; - - void Comp_IType(MIPSOpcode op) override; - void Comp_RType2(MIPSOpcode op) override; - void Comp_RType3(MIPSOpcode op) override; - void Comp_ShiftType(MIPSOpcode op) override; - void Comp_Allegrex(MIPSOpcode op) override; - void Comp_Allegrex2(MIPSOpcode op) override; - void Comp_VBranch(MIPSOpcode op) override; - void Comp_MulDivType(MIPSOpcode op) override; - void Comp_Special3(MIPSOpcode op) override; - - void Comp_FPU3op(MIPSOpcode op) override; - void Comp_FPU2op(MIPSOpcode op) override; - void Comp_mxc1(MIPSOpcode op) override; - - void Comp_DoNothing(MIPSOpcode op) override; - - void Comp_SV(MIPSOpcode op) override; - void Comp_SVQ(MIPSOpcode op) override; - void Comp_VPFX(MIPSOpcode op) override; - void Comp_VVectorInit(MIPSOpcode op) override; - void Comp_VMatrixInit(MIPSOpcode op) override; - void Comp_VDot(MIPSOpcode op) override; - void Comp_VecDo3(MIPSOpcode op) override; - void Comp_VV2Op(MIPSOpcode op) override; - void Comp_Mftv(MIPSOpcode op) override; - void Comp_Vmfvc(MIPSOpcode op) override; - void Comp_Vmtvc(MIPSOpcode op) override; - void Comp_Vmmov(MIPSOpcode op) override; - void Comp_VScl(MIPSOpcode op) override; - void Comp_Vmmul(MIPSOpcode op) override; - void Comp_Vmscl(MIPSOpcode op) override; - void Comp_Vtfm(MIPSOpcode op) override; - void Comp_VHdp(MIPSOpcode op) override; - void Comp_VCrs(MIPSOpcode op) override; - void Comp_VDet(MIPSOpcode op) override; - void Comp_Vi2x(MIPSOpcode op) override; - void Comp_Vx2i(MIPSOpcode op) override; - void Comp_Vf2i(MIPSOpcode op) override; - void Comp_Vi2f(MIPSOpcode op) override; - void Comp_Vh2f(MIPSOpcode op) override; - void Comp_Vcst(MIPSOpcode op) override; - void Comp_Vhoriz(MIPSOpcode op) override; - void Comp_VRot(MIPSOpcode op) override; - void Comp_VIdt(MIPSOpcode op) override; - void Comp_Vcmp(MIPSOpcode op) override; - void Comp_Vcmov(MIPSOpcode op) override; - void Comp_Viim(MIPSOpcode op) override; - void Comp_Vfim(MIPSOpcode op) override; - void Comp_VCrossQuat(MIPSOpcode op) override; - void Comp_Vsgn(MIPSOpcode op) override; - void Comp_Vocp(MIPSOpcode op) override; - void Comp_ColorConv(MIPSOpcode op) override; - void Comp_Vbfy(MIPSOpcode op) override; - - // Non-NEON: VPFX - - // NEON implementations of the VFPU ops. - void CompNEON_SV(MIPSOpcode op); - void CompNEON_SVQ(MIPSOpcode op); - void CompNEON_VVectorInit(MIPSOpcode op); - void CompNEON_VMatrixInit(MIPSOpcode op); - void CompNEON_VDot(MIPSOpcode op); - void CompNEON_VecDo3(MIPSOpcode op); - void CompNEON_VV2Op(MIPSOpcode op); - void CompNEON_Mftv(MIPSOpcode op); - void CompNEON_Vmfvc(MIPSOpcode op); - void CompNEON_Vmtvc(MIPSOpcode op); - void CompNEON_Vmmov(MIPSOpcode op); - void CompNEON_VScl(MIPSOpcode op); - void CompNEON_Vmmul(MIPSOpcode op); - void CompNEON_Vmscl(MIPSOpcode op); - void CompNEON_Vtfm(MIPSOpcode op); - void CompNEON_VHdp(MIPSOpcode op); - void CompNEON_VCrs(MIPSOpcode op); - void CompNEON_VDet(MIPSOpcode op); - void CompNEON_Vi2x(MIPSOpcode op); - void CompNEON_Vx2i(MIPSOpcode op); - void CompNEON_Vf2i(MIPSOpcode op); - void CompNEON_Vi2f(MIPSOpcode op); - void CompNEON_Vh2f(MIPSOpcode op); - void CompNEON_Vcst(MIPSOpcode op); - void CompNEON_Vhoriz(MIPSOpcode op); - void CompNEON_VRot(MIPSOpcode op); - void CompNEON_VIdt(MIPSOpcode op); - void CompNEON_Vcmp(MIPSOpcode op); - void CompNEON_Vcmov(MIPSOpcode op); - void CompNEON_Viim(MIPSOpcode op); - void CompNEON_Vfim(MIPSOpcode op); - void CompNEON_VCrossQuat(MIPSOpcode op); - void CompNEON_Vsgn(MIPSOpcode op); - void CompNEON_Vocp(MIPSOpcode op); - void CompNEON_ColorConv(MIPSOpcode op); - void CompNEON_Vbfy(MIPSOpcode op); - - int Replace_fabsf() override; - - JitBlockCache *GetBlockCache() override { return &blocks; } - JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - - void ClearCache() override; - void InvalidateCacheAt(u32 em_address, int length = 4) override; - void UpdateFCR31() override; - - void EatPrefix() override { js.EatPrefix(); } - - const u8 *GetDispatcher() const override { - return dispatcher; - } - u32 GetBlockShadowHookValue() const override { - return (u32)(blockShadowHook - GetBasePtr()); - } - bool IsAtDispatchFetch(const u8 *ptr) const override { - return ptr == dispatcherFetch; - } - - void LinkBlock(u8 *exitPoint, const u8 *checkedEntry) override; - void UnlinkBlock(u8 *checkedEntry, u32 originalAddress) override; - - const u8 *GetCodeBase() const override { return GetCodePtr(); } - -private: - void GenerateFixedCode(const JitOptions &jo); - void FlushAll(); - void FlushPrefixV(); - - u32 GetCompilerPC(); - void CompileDelaySlot(int flags); - void EatInstruction(MIPSOpcode op); - MIPSOpcode GetOffsetInstruction(int offset); - - void WriteDownCount(int offset = 0, bool updateFlags = true); - void WriteDownCountR(Arm64Gen::ARM64Reg reg, bool updateFlags = true); - void RestoreRoundingMode(bool force = false); - void ApplyRoundingMode(bool force = false); - void UpdateRoundingMode(u32 fcr31 = -1); - void MovFromPC(Arm64Gen::ARM64Reg r); - void MovToPC(Arm64Gen::ARM64Reg r); - - // Clobbers SCRATCH2. - void SaveStaticRegisters(); - // Clobbers SCRATCH2. - void LoadStaticRegisters(); - - void WriteExit(u32 destination, int exit_num); - void WriteExitDestInR(Arm64Gen::ARM64Reg Reg); - void WriteSyscallExit(); - bool CheckJitBreakpoint(u32 addr, int downcountOffset); - bool CheckMemoryBreakpoint(int instructionOffset = 0); - - // Utility compilation functions - void BranchFPFlag(MIPSOpcode op, CCFlags cc, bool likely); - void BranchVFPUFlag(MIPSOpcode op, CCFlags cc, bool likely); - void BranchRSZeroComp(MIPSOpcode op, CCFlags cc, bool andLink, bool likely); - void BranchRSRTComp(MIPSOpcode op, CCFlags cc, bool likely); - - // Utilities to reduce duplicated code - void CompImmLogic(MIPSGPReg rs, MIPSGPReg rt, u32 uimm, void (ARM64XEmitter::*arith)(Arm64Gen::ARM64Reg dst, Arm64Gen::ARM64Reg src, Arm64Gen::ARM64Reg src2), bool (ARM64XEmitter::*tryArithI2R)(Arm64Gen::ARM64Reg dst, Arm64Gen::ARM64Reg src, u64 val), u32 (*eval)(u32 a, u32 b)); - void CompType3(MIPSGPReg rd, MIPSGPReg rs, MIPSGPReg rt, void (ARM64XEmitter::*arithOp2)(Arm64Gen::ARM64Reg dst, Arm64Gen::ARM64Reg rm, Arm64Gen::ARM64Reg rn), bool (ARM64XEmitter::*tryArithI2R)(Arm64Gen::ARM64Reg dst, Arm64Gen::ARM64Reg rm, u64 val), u32 (*eval)(u32 a, u32 b), bool symmetric = false); - void CompShiftImm(MIPSOpcode op, Arm64Gen::ShiftType shiftType, int sa); - void CompShiftVar(MIPSOpcode op, Arm64Gen::ShiftType shiftType); - void CompVrotShuffle(u8 *dregs, int imm, VectorSize sz, bool negSin); - void CompVV2OpCall(MIPSOpcode op); - void LoadVAfterCallFlush(Arm64Gen::ARM64Reg dest, u8 vreg); - - void ApplyPrefixST(u8 *vregs, u32 prefix, VectorSize sz); - void ApplyPrefixD(const u8 *vregs, VectorSize sz); - void GetVectorRegsPrefixS(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixSFlag & JitState::PREFIX_KNOWN); - GetVectorRegs(regs, sz, vectorReg); - ApplyPrefixST(regs, js.prefixS, sz); - } - void GetVectorRegsPrefixT(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixTFlag & JitState::PREFIX_KNOWN); - GetVectorRegs(regs, sz, vectorReg); - ApplyPrefixST(regs, js.prefixT, sz); - } - void GetVectorRegsPrefixD(u8 *regs, VectorSize sz, int vectorReg); - - // Utils - void SetScratch1ToEffectiveAddress(MIPSGPReg rs, s16 offset); - std::vector SetScratch1ForSafeAddress(MIPSGPReg rs, s16 offset, Arm64Gen::ARM64Reg tempReg); - void Comp_ITypeMemLR(MIPSOpcode op, bool load); - - JitBlockCache blocks; - JitOptions jo; - JitState js; - - Arm64RegCache gpr; - Arm64RegCacheFPU fpr; - - Arm64Gen::ARM64FloatEmitter fp; - - MIPSState *mips_; - - int dontLogBlocks; - int logBlocks; - -public: - // Code pointers - const u8 *enterDispatcher; - - const u8 *outerLoop; - const u8 *outerLoopPCInSCRATCH1; - const u8 *dispatcherCheckCoreState; - const u8 *dispatcherPCInSCRATCH1; - const u8 *dispatcher; - const u8 *dispatcherNoCheck; - const u8 *dispatcherFetch; - - const u8 *saveStaticRegisters; - const u8 *loadStaticRegisters; - - const u8 *restoreRoundingMode; - const u8 *applyRoundingMode; - const u8 *updateRoundingMode; - - const u8 *crashHandler; - const u8 *blockShadowHook; - - int jitStartOffset; - - // Indexed by FPCR FZ:RN bits for convenience. Uses SCRATCH2. - const u8 *convertS0ToSCRATCH1[8]; -}; - -} // namespace MIPSComp - diff --git a/Core/MIPS/ARM64/Arm64RegCache.cpp b/Core/MIPS/ARM64/Arm64RegCache.cpp deleted file mode 100644 index 1904cc2286..0000000000 --- a/Core/MIPS/ARM64/Arm64RegCache.cpp +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include "Common/Log.h" -#include "Core/MemMap.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/Reporting.h" -#include "Common/Arm64Emitter.h" - -#ifndef offsetof -#include "stddef.h" -#endif - -using namespace Arm64Gen; -using namespace Arm64JitConstants; - -Arm64RegCache::Arm64RegCache(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo) : mips_(mipsState), js_(js), jo_(jo) { -} - -void Arm64RegCache::Init(ARM64XEmitter *emitter) { - emit_ = emitter; -} - -void Arm64RegCache::Start() { - for (int i = 0; i < NUM_ARMREG; i++) { - ar[i].mipsReg = MIPS_REG_INVALID; - ar[i].isDirty = false; - ar[i].pointerified = false; - ar[i].tempLocked = false; - } - for (int i = 0; i < NUM_MIPSREG; i++) { - mr[i].loc = ML_MEM; - mr[i].reg = INVALID_REG; - mr[i].imm = -1; - mr[i].spillLock = false; - mr[i].isStatic = false; - } - int numStatics; - const StaticAllocation *statics = GetStaticAllocations(numStatics); - for (int i = 0; i < numStatics; i++) { - ar[statics[i].ar].mipsReg = statics[i].mr; - ar[statics[i].ar].pointerified = statics[i].pointerified && jo_->enablePointerify; - mr[statics[i].mr].loc = ML_ARMREG; - mr[statics[i].mr].reg = statics[i].ar; - mr[statics[i].mr].isStatic = true; - mr[statics[i].mr].spillLock = true; - } -} - -const ARM64Reg *Arm64RegCache::GetMIPSAllocationOrder(int &count) { - // See register alloc remarks in Arm64Asm.cpp - - // W19-W23 are most suitable for static allocation. Those that are chosen for static allocation - // should be omitted here and added in GetStaticAllocations. - static const ARM64Reg allocationOrder[] = { - W19, W20, W21, W22, W23, W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, - }; - static const ARM64Reg allocationOrderStaticAlloc[] = { - W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, - }; - - if (jo_->useStaticAlloc) { - count = ARRAY_SIZE(allocationOrderStaticAlloc); - return allocationOrderStaticAlloc; - } else { - count = ARRAY_SIZE(allocationOrder); - return allocationOrder; - } -} - -const Arm64RegCache::StaticAllocation *Arm64RegCache::GetStaticAllocations(int &count) { - static const StaticAllocation allocs[] = { - {MIPS_REG_SP, W19, true}, - {MIPS_REG_V0, W20}, - {MIPS_REG_V1, W22}, - {MIPS_REG_A0, W21}, - {MIPS_REG_RA, W23}, - }; - - if (jo_->useStaticAlloc) { - count = ARRAY_SIZE(allocs); - return allocs; - } else { - count = 0; - return nullptr; - } -} - -void Arm64RegCache::EmitLoadStaticRegisters() { - int count; - const StaticAllocation *allocs = GetStaticAllocations(count); - // TODO: Use LDP when possible. - for (int i = 0; i < count; i++) { - int offset = GetMipsRegOffset(allocs[i].mr); - emit_->LDR(INDEX_UNSIGNED, allocs[i].ar, CTXREG, offset); - if (allocs[i].pointerified && jo_->enablePointerify) { - emit_->MOVK(EncodeRegTo64(allocs[i].ar), ((uint64_t)Memory::base) >> 32, SHIFT_32); - } - } -} - -void Arm64RegCache::EmitSaveStaticRegisters() { - int count; - const StaticAllocation *allocs = GetStaticAllocations(count); - // TODO: Use STP when possible. - // This only needs to run once (by Asm) so checks don't need to be fast. - for (int i = 0; i < count; i++) { - int offset = GetMipsRegOffset(allocs[i].mr); - emit_->STR(INDEX_UNSIGNED, allocs[i].ar, CTXREG, offset); - } -} - -void Arm64RegCache::FlushBeforeCall() { - // These registers are not preserved by function calls. - for (int i = 0; i < 19; ++i) { - FlushArmReg(ARM64Reg(W0 + i)); - } - FlushArmReg(W30); -} - -bool Arm64RegCache::IsInRAM(MIPSGPReg reg) { - return mr[reg].loc == ML_MEM; -} - -bool Arm64RegCache::IsMapped(MIPSGPReg mipsReg) { - return mr[mipsReg].loc == ML_ARMREG || mr[mipsReg].loc == ML_ARMREG_IMM; -} - -bool Arm64RegCache::IsMappedAsPointer(MIPSGPReg mipsReg) { - if (mr[mipsReg].loc == ML_ARMREG) { - return ar[mr[mipsReg].reg].pointerified; - } else if (mr[mipsReg].loc == ML_ARMREG_IMM) { - if (ar[mr[mipsReg].reg].pointerified) { - ERROR_LOG(Log::JIT, "Really shouldn't be pointerified here"); - } - } else if (mr[mipsReg].loc == ML_ARMREG_AS_PTR) { - return true; - } - return false; -} - -void Arm64RegCache::MarkDirty(ARM64Reg reg) { - ar[reg].isDirty = true; -} - -void Arm64RegCache::SetRegImm(ARM64Reg reg, u64 imm) { - if (reg == INVALID_REG || reg == W0 || reg == X0) { - ERROR_LOG(Log::JIT, "SetRegImm to invalid register: at %08x", js_->compilerPC); - return; - } - _dbg_assert_(!ar[reg].tempLocked); - // On ARM64, at least Cortex A57, good old MOVT/MOVW (MOVK in 64-bit) is really fast. - emit_->MOVI2R(reg, imm); - // ar[reg].pointerified = false; -} - -void Arm64RegCache::MapRegTo(ARM64Reg reg, MIPSGPReg mipsReg, int mapFlags) { - if (mr[mipsReg].isStatic) { - ERROR_LOG(Log::JIT, "Cannot MapRegTo static register %d", mipsReg); - return; - } - ar[reg].isDirty = (mapFlags & MAP_DIRTY) ? true : false; - if ((mapFlags & MAP_NOINIT) != MAP_NOINIT) { - if (mipsReg == MIPS_REG_ZERO) { - // If we get a request to map the zero register, at least we won't spend - // time on a memory access... - emit_->MOVI2R(reg, 0); - - // This way, if we SetImm() it, we'll keep it. - mr[mipsReg].loc = ML_ARMREG_IMM; - mr[mipsReg].imm = 0; - } else { - switch (mr[mipsReg].loc) { - case ML_MEM: - { - int offset = GetMipsRegOffset(mipsReg); - ARM64Reg loadReg = reg; - // INFO_LOG(Log::JIT, "MapRegTo %d mips: %d offset %d", (int)reg, mipsReg, offset); - if (mipsReg == MIPS_REG_LO) { - loadReg = EncodeRegTo64(loadReg); - } - // TODO: Scan ahead / hint when loading multiple regs? - // We could potentially LDP if mipsReg + 1 or mipsReg - 1 is needed. - emit_->LDR(INDEX_UNSIGNED, loadReg, CTXREG, offset); - mr[mipsReg].loc = ML_ARMREG; - break; - } - case ML_IMM: - emit_->MOVI2R(reg, mr[mipsReg].imm); - ar[reg].isDirty = true; // IMM is always dirty. - - // If we are mapping dirty, it means we're gonna overwrite. - // So the imm value is no longer valid. - if (mapFlags & MAP_DIRTY) - mr[mipsReg].loc = ML_ARMREG; - else - mr[mipsReg].loc = ML_ARMREG_IMM; - break; - default: - _assert_msg_(mr[mipsReg].loc != ML_ARMREG_AS_PTR, "MapRegTo with a pointer?"); - mr[mipsReg].loc = ML_ARMREG; - break; - } - } - } else { - mr[mipsReg].loc = ML_ARMREG; - } - ar[reg].mipsReg = mipsReg; - ar[reg].pointerified = false; - mr[mipsReg].reg = reg; -} - -ARM64Reg Arm64RegCache::AllocateReg() { - int allocCount; - const ARM64Reg *allocOrder = GetMIPSAllocationOrder(allocCount); - -allocate: - for (int i = 0; i < allocCount; i++) { - ARM64Reg reg = allocOrder[i]; - - if (ar[reg].mipsReg == MIPS_REG_INVALID && !ar[reg].tempLocked) { - return reg; - } - } - - // Still nothing. Let's spill a reg and goto 10. - // TODO: Use age or something to choose which register to spill? - // TODO: Spill dirty regs first? or opposite? - bool clobbered; - ARM64Reg bestToSpill = FindBestToSpill(true, &clobbered); - if (bestToSpill == INVALID_REG) { - bestToSpill = FindBestToSpill(false, &clobbered); - } - - if (bestToSpill != INVALID_REG) { - if (clobbered) { - DiscardR(ar[bestToSpill].mipsReg); - } else { - FlushArmReg(bestToSpill); - } - // Now one must be free. - goto allocate; - } - - // Uh oh, we have all of them spilllocked.... - ERROR_LOG_REPORT(Log::JIT, "Out of spillable registers at PC %08x!!!", mips_->pc); - return INVALID_REG; -} - -ARM64Reg Arm64RegCache::FindBestToSpill(bool unusedOnly, bool *clobbered) { - int allocCount; - const ARM64Reg *allocOrder = GetMIPSAllocationOrder(allocCount); - - static const int UNUSED_LOOKAHEAD_OPS = 30; - - *clobbered = false; - for (int i = 0; i < allocCount; i++) { - ARM64Reg reg = allocOrder[i]; - if (ar[reg].mipsReg != MIPS_REG_INVALID && mr[ar[reg].mipsReg].spillLock) - continue; - if (ar[reg].tempLocked) - continue; - - // As it's in alloc-order, we know it's not static so we don't need to check for that. - - // Awesome, a clobbered reg. Let's use it. - if (MIPSAnalyst::IsRegisterClobbered(ar[reg].mipsReg, compilerPC_, UNUSED_LOOKAHEAD_OPS)) { - bool canClobber = true; - // HI is stored inside the LO reg. They both have to clobber at the same time. - if (ar[reg].mipsReg == MIPS_REG_LO) { - canClobber = MIPSAnalyst::IsRegisterClobbered(MIPS_REG_HI, compilerPC_, UNUSED_LOOKAHEAD_OPS); - } - if (canClobber) { - *clobbered = true; - return reg; - } - } - - // Not awesome. A used reg. Let's try to avoid spilling. - if (unusedOnly && MIPSAnalyst::IsRegisterUsed(ar[reg].mipsReg, compilerPC_, UNUSED_LOOKAHEAD_OPS)) { - continue; - } - - return reg; - } - - return INVALID_REG; -} - -ARM64Reg Arm64RegCache::TryMapTempImm(MIPSGPReg r) { - // If already mapped, no need for a temporary. - if (IsMapped(r)) { - return R(r); - } - - if (mr[r].loc == ML_IMM) { - if (mr[r].imm == 0) { - return WZR; - } - - // Try our luck - check for an exact match in another armreg. - for (int i = 0; i < NUM_MIPSREG; ++i) { - if (mr[i].loc == ML_ARMREG_IMM && mr[i].imm == mr[r].imm) { - // Awesome, let's just use this reg. - return mr[i].reg; - } - } - } - - return INVALID_REG; -} - -ARM64Reg Arm64RegCache::GetAndLockTempR() { - ARM64Reg reg = AllocateReg(); - if (reg != INVALID_REG) { - ar[reg].tempLocked = true; - } - return reg; -} - -// TODO: Somewhat smarter spilling - currently simply spills the first available, should do -// round robin or FIFO or something. -ARM64Reg Arm64RegCache::MapReg(MIPSGPReg mipsReg, int mapFlags) { - if (mipsReg == MIPS_REG_HI) { - ERROR_LOG_REPORT(Log::JIT, "Cannot map HI in Arm64RegCache"); - return INVALID_REG; - } - - if (mipsReg == MIPS_REG_INVALID) { - ERROR_LOG(Log::JIT, "Cannot map invalid register"); - return INVALID_REG; - } - - ARM64Reg armReg = mr[mipsReg].reg; - - if (mr[mipsReg].isStatic) { - if (armReg == INVALID_REG) { - ERROR_LOG(Log::JIT, "MapReg on statically mapped reg %d failed - armReg got lost", mipsReg); - } - if (mr[mipsReg].loc == ML_IMM) { - // Back into the register, with or without the imm value. - // If noinit, the MAP_DIRTY check below will take care of the rest. - if ((mapFlags & MAP_NOINIT) != MAP_NOINIT) { - SetRegImm(armReg, mr[mipsReg].imm); - mr[mipsReg].loc = ML_ARMREG_IMM; - ar[armReg].pointerified = false; - } - } else if (mr[mipsReg].loc == ML_ARMREG_AS_PTR) { - // Was mapped as pointer, now we want it mapped as a value, presumably to - // add or subtract stuff to it. - if ((mapFlags & MAP_NOINIT) != MAP_NOINIT) { - emit_->SUB(EncodeRegTo64(armReg), EncodeRegTo64(armReg), MEMBASEREG); - } - mr[mipsReg].loc = ML_ARMREG; - } - // Erasing the imm on dirty (necessary since otherwise we will still think it's ML_ARMREG_IMM and return - // true for IsImm and calculate crazily wrong things). /unknown - if (mapFlags & MAP_DIRTY) { - mr[mipsReg].loc = ML_ARMREG; // As we are dirty, can't keep ARMREG_IMM, we will quickly drift out of sync - ar[armReg].pointerified = false; - ar[armReg].isDirty = true; // Not that it matters - } - return mr[mipsReg].reg; - } - - // Let's see if it's already mapped. If so we just need to update the dirty flag. - // We don't need to check for ML_NOINIT because we assume that anyone who maps - // with that flag immediately writes a "known" value to the register. - if (mr[mipsReg].loc == ML_ARMREG || mr[mipsReg].loc == ML_ARMREG_IMM) { - if (ar[armReg].mipsReg != mipsReg) { - ERROR_LOG_REPORT(Log::JIT, "Register mapping out of sync! %i", mipsReg); - } - if (mapFlags & MAP_DIRTY) { - // Mapping dirty means the old imm value is invalid. - mr[mipsReg].loc = ML_ARMREG; - ar[armReg].isDirty = true; - // If reg is written to, pointerification is lost. - ar[armReg].pointerified = false; - } - - return mr[mipsReg].reg; - } else if (mr[mipsReg].loc == ML_ARMREG_AS_PTR) { - // Was mapped as pointer, now we want it mapped as a value, presumably to - // add or subtract stuff to it. - if ((mapFlags & MAP_NOINIT) != MAP_NOINIT) { - emit_->SUB(EncodeRegTo64(armReg), EncodeRegTo64(armReg), MEMBASEREG); - } - mr[mipsReg].loc = ML_ARMREG; - if (mapFlags & MAP_DIRTY) { - ar[armReg].isDirty = true; - } - return (ARM64Reg)mr[mipsReg].reg; - } - - // Okay, not mapped, so we need to allocate an ARM register. - ARM64Reg reg = AllocateReg(); - if (reg != INVALID_REG) { - // Grab it, and load the value into it (if requested). - MapRegTo(reg, mipsReg, mapFlags); - } - - return reg; -} - -Arm64Gen::ARM64Reg Arm64RegCache::MapRegAsPointer(MIPSGPReg reg) { - // Already mapped. - if (mr[reg].loc == ML_ARMREG_AS_PTR) { - return mr[reg].reg; - } - - ARM64Reg retval = INVALID_REG; - if (mr[reg].loc != ML_ARMREG && mr[reg].loc != ML_ARMREG_IMM) { - retval = MapReg(reg); - } else { - retval = mr[reg].reg; - } - - if (mr[reg].loc == ML_ARMREG || mr[reg].loc == ML_ARMREG_IMM) { - // If there was an imm attached, discard it. - mr[reg].loc = ML_ARMREG; - ARM64Reg a = DecodeReg(mr[reg].reg); - if (!jo_->enablePointerify) { - // Convert to a pointer by adding the base and clearing off the top bits. - // If SP, we can probably avoid the top bit clear, let's play with that later. -#ifdef MASKED_PSP_MEMORY - emit_->ANDI2R(EncodeRegTo64(a), EncodeRegTo64(a), 0x3FFFFFFF); -#endif - emit_->ADD(EncodeRegTo64(a), EncodeRegTo64(a), MEMBASEREG); - mr[reg].loc = ML_ARMREG_AS_PTR; - } else if (!ar[a].pointerified) { - emit_->MOVK(EncodeRegTo64(a), ((uint64_t)Memory::base) >> 32, SHIFT_32); - ar[a].pointerified = true; - } - } else { - ERROR_LOG(Log::JIT, "MapRegAsPointer : MapReg failed to allocate a register?"); - } - return retval; -} - -void Arm64RegCache::MapIn(MIPSGPReg rs) { - MapReg(rs); -} - -void Arm64RegCache::MapInIn(MIPSGPReg rd, MIPSGPReg rs) { - SpillLock(rd, rs); - MapReg(rd); - MapReg(rs); - ReleaseSpillLock(rd, rs); -} - -void Arm64RegCache::MapDirtyIn(MIPSGPReg rd, MIPSGPReg rs, bool avoidLoad) { - SpillLock(rd, rs); - bool load = !avoidLoad || rd == rs; - MapReg(rd, load ? MAP_DIRTY : MAP_NOINIT); - MapReg(rs); - ReleaseSpillLock(rd, rs); -} - -void Arm64RegCache::MapDirtyInIn(MIPSGPReg rd, MIPSGPReg rs, MIPSGPReg rt, bool avoidLoad) { - SpillLock(rd, rs, rt); - bool load = !avoidLoad || (rd == rs || rd == rt); - MapReg(rd, load ? MAP_DIRTY : MAP_NOINIT); - MapReg(rt); - MapReg(rs); - ReleaseSpillLock(rd, rs, rt); -} - -void Arm64RegCache::MapDirtyDirtyIn(MIPSGPReg rd1, MIPSGPReg rd2, MIPSGPReg rs, bool avoidLoad) { - SpillLock(rd1, rd2, rs); - bool load1 = !avoidLoad || rd1 == rs; - bool load2 = !avoidLoad || rd2 == rs; - MapReg(rd1, load1 ? MAP_DIRTY : MAP_NOINIT); - MapReg(rd2, load2 ? MAP_DIRTY : MAP_NOINIT); - MapReg(rs); - ReleaseSpillLock(rd1, rd2, rs); -} - -void Arm64RegCache::MapDirtyDirtyInIn(MIPSGPReg rd1, MIPSGPReg rd2, MIPSGPReg rs, MIPSGPReg rt, bool avoidLoad) { - SpillLock(rd1, rd2, rs, rt); - bool load1 = !avoidLoad || (rd1 == rs || rd1 == rt); - bool load2 = !avoidLoad || (rd2 == rs || rd2 == rt); - MapReg(rd1, load1 ? MAP_DIRTY : MAP_NOINIT); - MapReg(rd2, load2 ? MAP_DIRTY : MAP_NOINIT); - MapReg(rt); - MapReg(rs); - ReleaseSpillLock(rd1, rd2, rs, rt); -} - -void Arm64RegCache::FlushArmReg(ARM64Reg r) { - if (r == INVALID_REG) { - ERROR_LOG(Log::JIT, "FlushArmReg called on invalid register %d", r); - return; - } - if (ar[r].mipsReg == MIPS_REG_INVALID) { - // Nothing to do, reg not mapped. - if (ar[r].isDirty) { - ERROR_LOG_REPORT(Log::JIT, "Dirty but no mipsreg?"); - } - return; - } - if (mr[ar[r].mipsReg].isStatic) { - ERROR_LOG(Log::JIT, "Cannot FlushArmReg a statically mapped register"); - return; - } - auto &mreg = mr[ar[r].mipsReg]; - if (mreg.loc == ML_ARMREG_IMM || ar[r].mipsReg == MIPS_REG_ZERO) { - // We know its immediate value, no need to STR now. - mreg.loc = ML_IMM; - mreg.reg = INVALID_REG; - } else { - if (mreg.loc == ML_IMM || ar[r].isDirty) { - if (mreg.loc == ML_ARMREG_AS_PTR) { - // Unpointerify, in case dirty. - emit_->SUB(EncodeRegTo64(r), EncodeRegTo64(r), MEMBASEREG); - mreg.loc = ML_ARMREG; - } - // Note: may be a 64-bit reg. - ARM64Reg storeReg = ARM64RegForFlush(ar[r].mipsReg); - if (storeReg != INVALID_REG) - emit_->STR(INDEX_UNSIGNED, storeReg, CTXREG, GetMipsRegOffset(ar[r].mipsReg)); - } - mreg.loc = ML_MEM; - mreg.reg = INVALID_REG; - mreg.imm = 0; - } - ar[r].isDirty = false; - ar[r].mipsReg = MIPS_REG_INVALID; - ar[r].pointerified = false; -} - -void Arm64RegCache::DiscardR(MIPSGPReg mipsReg) { - if (mr[mipsReg].isStatic) { - // Simply do nothing unless it's an IMM/ARMREG_IMM/ARMREG_AS_PTR, in case we just switch it over to ARMREG, losing the value. - ARM64Reg armReg = mr[mipsReg].reg; - if (mr[mipsReg].loc == ML_ARMREG_IMM || mr[mipsReg].loc == ML_IMM || mr[mipsReg].loc == ML_ARMREG_AS_PTR) { - // Ignore the imm value, restore sanity - mr[mipsReg].loc = ML_ARMREG; - ar[armReg].pointerified = false; - ar[armReg].isDirty = false; - } - return; - } - const RegMIPSLoc prevLoc = mr[mipsReg].loc; - if (prevLoc == ML_ARMREG || prevLoc == ML_ARMREG_IMM || prevLoc == ML_ARMREG_AS_PTR) { - ARM64Reg armReg = mr[mipsReg].reg; - ar[armReg].isDirty = false; - ar[armReg].mipsReg = MIPS_REG_INVALID; - ar[armReg].pointerified = false; - mr[mipsReg].reg = INVALID_REG; - if (mipsReg == MIPS_REG_ZERO) { - mr[mipsReg].loc = ML_IMM; - } else { - mr[mipsReg].loc = ML_MEM; - } - mr[mipsReg].imm = 0; - } - if (prevLoc == ML_IMM && mipsReg != MIPS_REG_ZERO) { - mr[mipsReg].loc = ML_MEM; - mr[mipsReg].imm = 0; - } -} - -ARM64Reg Arm64RegCache::ARM64RegForFlush(MIPSGPReg r) { - if (mr[r].isStatic) - return INVALID_REG; // No flushing needed - - switch (mr[r].loc) { - case ML_IMM: - if (r == MIPS_REG_ZERO) { - return INVALID_REG; - } - // Zero is super easy. - if (mr[r].imm == 0) { - return WZR; - } - // Could we get lucky? Check for an exact match in another armreg. - for (int i = 0; i < NUM_MIPSREG; ++i) { - if (mr[i].loc == ML_ARMREG_IMM && mr[i].imm == mr[r].imm) { - // Awesome, let's just store this reg. - return mr[i].reg; - } - } - return INVALID_REG; - - case ML_ARMREG: - case ML_ARMREG_IMM: - if (mr[r].reg == INVALID_REG) { - ERROR_LOG_REPORT(Log::JIT, "ARM64RegForFlush: MipsReg %d had bad ArmReg", r); - return INVALID_REG; - } - // No need to flush if it's zero or not dirty. - if (r == MIPS_REG_ZERO || !ar[mr[r].reg].isDirty) { - return INVALID_REG; - } - if (r == MIPS_REG_LO) { - return EncodeRegTo64(mr[r].reg); - } - return mr[r].reg; - - case ML_ARMREG_AS_PTR: - return INVALID_REG; - - case ML_MEM: - return INVALID_REG; - - default: - ERROR_LOG_REPORT(Log::JIT, "ARM64RegForFlush: MipsReg %d with invalid location %d", r, mr[r].loc); - return INVALID_REG; - } -} - -void Arm64RegCache::FlushR(MIPSGPReg r) { - if (mr[r].isStatic) { - ERROR_LOG(Log::JIT, "Cannot flush static reg %d", r); - return; - } - - switch (mr[r].loc) { - case ML_IMM: - // IMM is always "dirty". - if (r == MIPS_REG_LO) { - emit_->MOVI2R(SCRATCH1_64, mr[r].imm); - emit_->STR(INDEX_UNSIGNED, SCRATCH1_64, CTXREG, GetMipsRegOffset(r)); - } else if (r != MIPS_REG_ZERO) { - // Try to optimize using a different reg. - ARM64Reg storeReg = ARM64RegForFlush(r); - if (storeReg == INVALID_REG) { - emit_->MOVI2R(SCRATCH1, mr[r].imm); - storeReg = SCRATCH1; - } - emit_->STR(INDEX_UNSIGNED, storeReg, CTXREG, GetMipsRegOffset(r)); - } - break; - - case ML_ARMREG: - case ML_ARMREG_IMM: - if (ar[mr[r].reg].isDirty) { - // Note: might be a 64-bit reg. - ARM64Reg storeReg = ARM64RegForFlush(r); - if (storeReg != INVALID_REG) { - emit_->STR(INDEX_UNSIGNED, storeReg, CTXREG, GetMipsRegOffset(r)); - } - ar[mr[r].reg].isDirty = false; - } - ar[mr[r].reg].mipsReg = MIPS_REG_INVALID; - ar[mr[r].reg].pointerified = false; - break; - - case ML_ARMREG_AS_PTR: - if (ar[mr[r].reg].isDirty) { - emit_->SUB(EncodeRegTo64(mr[r].reg), EncodeRegTo64(mr[r].reg), MEMBASEREG); - // We set this so ARM64RegForFlush knows it's no longer a pointer. - mr[r].loc = ML_ARMREG; - ARM64Reg storeReg = ARM64RegForFlush(r); - if (storeReg != INVALID_REG) { - emit_->STR(INDEX_UNSIGNED, storeReg, CTXREG, GetMipsRegOffset(r)); - } - ar[mr[r].reg].isDirty = false; - } - ar[mr[r].reg].mipsReg = MIPS_REG_INVALID; - break; - - case ML_MEM: - // Already there, nothing to do. - break; - - default: - ERROR_LOG_REPORT(Log::JIT, "FlushR: MipsReg %d with invalid location %d", r, mr[r].loc); - break; - } - if (r == MIPS_REG_ZERO) { - mr[r].loc = ML_IMM; - } else { - mr[r].loc = ML_MEM; - } - mr[r].reg = INVALID_REG; - mr[r].imm = 0; -} - -void Arm64RegCache::FlushAll() { - // Note: make sure not to change the registers when flushing: - // Branching code expects the armreg to retain its value. - - // LO can't be included in a 32-bit pair, since it's 64 bit. - // Flush it first so we don't get it confused. - FlushR(MIPS_REG_LO); - - // Try to flush in pairs when possible. - // 1 because MIPS_REG_ZERO isn't flushable anyway. - // 31 because 30 and 31 are the last possible pair - MIPS_REG_FPCOND, etc. are too far away. - for (int i = 1; i < 31; i++) { - MIPSGPReg mreg1 = MIPSGPReg(i); - MIPSGPReg mreg2 = MIPSGPReg(i + 1); - ARM64Reg areg1 = ARM64RegForFlush(mreg1); - ARM64Reg areg2 = ARM64RegForFlush(mreg2); - - // If either one doesn't have a reg yet, try flushing imms to scratch regs. - if (areg1 == INVALID_REG && IsPureImm(mreg1) && !mr[i].isStatic) { - areg1 = SCRATCH1; - } - if (areg2 == INVALID_REG && IsPureImm(mreg2) && !mr[i + 1].isStatic) { - areg2 = SCRATCH2; - } - - if (areg1 != INVALID_REG && areg2 != INVALID_REG) { - // Actually put the imms in place now that we know we can do the STP. - // We didn't do it before in case the other wouldn't work. - if (areg1 == SCRATCH1) { - SetRegImm(areg1, GetImm(mreg1)); - } - if (areg2 == SCRATCH2) { - SetRegImm(areg2, GetImm(mreg2)); - } - - // We can use a paired store, awesome. - emit_->STP(INDEX_SIGNED, areg1, areg2, CTXREG, GetMipsRegOffset(mreg1)); - - // Now we mark them as stored by discarding. - DiscardR(mreg1); - DiscardR(mreg2); - } - } - - // Final pass to grab any that were left behind. - for (int i = 0; i < NUM_MIPSREG; i++) { - MIPSGPReg mipsReg = MIPSGPReg(i); - if (mr[i].isStatic) { - Arm64Gen::ARM64Reg armReg = mr[i].reg; - // Cannot leave any IMMs in registers, not even ML_ARMREG_IMM, can confuse the regalloc later if this flush is mid-block - // due to an interpreter fallback that changes the register. - if (mr[i].loc == ML_IMM) { - emit_->MOVI2R(mr[i].reg, mr[i].imm); - mr[i].loc = ML_ARMREG; - ar[armReg].pointerified = false; - } else if (mr[i].loc == ML_ARMREG_IMM) { - // The register already contains the immediate. - if (ar[armReg].pointerified) { - ERROR_LOG(Log::JIT, "ML_ARMREG_IMM but pointerified. Wrong."); - ar[armReg].pointerified = false; - } - mr[i].loc = ML_ARMREG; - } else if (mr[i].loc == ML_ARMREG_AS_PTR) { - emit_->SUB(EncodeRegTo64(armReg), EncodeRegTo64(armReg), MEMBASEREG); - mr[i].loc = ML_ARMREG; - } - if (i != MIPS_REG_ZERO && mr[i].reg == INVALID_REG) { - ERROR_LOG(Log::JIT, "ARM reg of static %i is invalid", i); - continue; - } - } else { - FlushR(mipsReg); - } - } - - int count = 0; - const StaticAllocation *allocs = GetStaticAllocations(count); - for (int i = 0; i < count; i++) { - if (allocs[i].pointerified && !ar[allocs[i].ar].pointerified && jo_->enablePointerify) { - // Re-pointerify - emit_->MOVK(EncodeRegTo64(allocs[i].ar), ((uint64_t)Memory::base) >> 32, SHIFT_32); - ar[allocs[i].ar].pointerified = true; - } else if (!allocs[i].pointerified) { - // If this register got pointerified on the way, mark it as not, so that after save/reload (like in an interpreter fallback), it won't be regarded as such, as it simply won't be. - ar[allocs[i].ar].pointerified = false; - } - } - // Sanity check - for (int i = 0; i < NUM_ARMREG; i++) { - if (ar[i].mipsReg != MIPS_REG_INVALID && mr[ar[i].mipsReg].isStatic == false) { - ERROR_LOG_REPORT(Log::JIT, "Flush fail: ar[%i].mipsReg=%i", i, ar[i].mipsReg); - } - } -} - -void Arm64RegCache::SetImm(MIPSGPReg r, u64 immVal) { - if (r == MIPS_REG_HI) { - ERROR_LOG_REPORT(Log::JIT, "Cannot set HI imm in Arm64RegCache"); - return; - } - if (r == MIPS_REG_ZERO && immVal != 0) { - ERROR_LOG_REPORT(Log::JIT, "Trying to set immediate %08x to r0 at %08x", (u32)immVal, compilerPC_); - return; - } - - if (mr[r].loc == ML_ARMREG_IMM && mr[r].imm == immVal) { - // Already have that value, let's keep it in the reg. - return; - } - - if (r != MIPS_REG_LO) { - // All regs on the PSP are 32 bit, but LO we treat as HI:LO so is 64 full bits. - immVal = immVal & 0xFFFFFFFF; - } - - if (mr[r].isStatic) { - mr[r].loc = ML_IMM; - mr[r].imm = immVal; - ar[mr[r].reg].pointerified = false; - // We do not change reg to INVALID_REG for obvious reasons.. - } else { - // Zap existing value if cached in a reg - if (mr[r].reg != INVALID_REG) { - ar[mr[r].reg].mipsReg = MIPS_REG_INVALID; - ar[mr[r].reg].isDirty = false; - ar[mr[r].reg].pointerified = false; - } - mr[r].loc = ML_IMM; - mr[r].imm = immVal; - mr[r].reg = INVALID_REG; - } -} - -bool Arm64RegCache::IsImm(MIPSGPReg r) const { - if (r == MIPS_REG_ZERO) - return true; - else - return mr[r].loc == ML_IMM || mr[r].loc == ML_ARMREG_IMM; -} - -bool Arm64RegCache::IsPureImm(MIPSGPReg r) const { - if (r == MIPS_REG_ZERO) - return true; - else - return mr[r].loc == ML_IMM; -} - -u64 Arm64RegCache::GetImm(MIPSGPReg r) const { - if (r == MIPS_REG_ZERO) - return 0; - if (mr[r].loc != ML_IMM && mr[r].loc != ML_ARMREG_IMM) { - ERROR_LOG_REPORT(Log::JIT, "Trying to get imm from non-imm register %i", r); - } - return mr[r].imm; -} - -int Arm64RegCache::GetMipsRegOffset(MIPSGPReg r) { - if (r < 32) - return r * 4; - switch (r) { - case MIPS_REG_HI: - return offsetof(MIPSState, hi); - case MIPS_REG_LO: - return offsetof(MIPSState, lo); - case MIPS_REG_FPCOND: - return offsetof(MIPSState, fpcond); - case MIPS_REG_VFPUCC: - return offsetof(MIPSState, vfpuCtrl[VFPU_CTRL_CC]); - default: - ERROR_LOG_REPORT(Log::JIT, "bad mips register %i", r); - return 0; // or what? - } -} - -void Arm64RegCache::SpillLock(MIPSGPReg r1, MIPSGPReg r2, MIPSGPReg r3, MIPSGPReg r4) { - mr[r1].spillLock = true; - if (r2 != MIPS_REG_INVALID) mr[r2].spillLock = true; - if (r3 != MIPS_REG_INVALID) mr[r3].spillLock = true; - if (r4 != MIPS_REG_INVALID) mr[r4].spillLock = true; -} - -void Arm64RegCache::ReleaseSpillLocksAndDiscardTemps() { - for (int i = 0; i < NUM_MIPSREG; i++) { - if (!mr[i].isStatic) - mr[i].spillLock = false; - } - for (int i = 0; i < NUM_ARMREG; i++) { - ar[i].tempLocked = false; - } -} - -void Arm64RegCache::ReleaseSpillLock(MIPSGPReg r1, MIPSGPReg r2, MIPSGPReg r3, MIPSGPReg r4) { - if (!mr[r1].isStatic) - mr[r1].spillLock = false; - if (r2 != MIPS_REG_INVALID && !mr[r2].isStatic) - mr[r2].spillLock = false; - if (r3 != MIPS_REG_INVALID && !mr[r3].isStatic) - mr[r3].spillLock = false; - if (r4 != MIPS_REG_INVALID && !mr[r4].isStatic) - mr[r4].spillLock = false; -} - -ARM64Reg Arm64RegCache::R(MIPSGPReg mipsReg) { - if (mr[mipsReg].loc == ML_ARMREG || mr[mipsReg].loc == ML_ARMREG_IMM) { - return mr[mipsReg].reg; - } else { - ERROR_LOG_REPORT(Log::JIT, "Reg %i not in arm reg. compilerPC = %08x", mipsReg, compilerPC_); - return INVALID_REG; // BAAAD - } -} - -ARM64Reg Arm64RegCache::RPtr(MIPSGPReg mipsReg) { - if (mr[mipsReg].loc == ML_ARMREG_AS_PTR) { - return (ARM64Reg)mr[mipsReg].reg; - } else if (mr[mipsReg].loc == ML_ARMREG || mr[mipsReg].loc == ML_ARMREG_IMM) { - int a = mr[mipsReg].reg; - if (ar[a].pointerified) { - return (ARM64Reg)mr[mipsReg].reg; - } else { - ERROR_LOG(Log::JIT, "Tried to use a non-pointer register as a pointer"); - return INVALID_REG; - } - } else { - ERROR_LOG_REPORT(Log::JIT, "Reg %i not in arm reg. compilerPC = %08x", mipsReg, compilerPC_); - return INVALID_REG; // BAAAD - } -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64RegCache.h b/Core/MIPS/ARM64/Arm64RegCache.h deleted file mode 100644 index 640fd80e26..0000000000 --- a/Core/MIPS/ARM64/Arm64RegCache.h +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include "Core/MIPS/MIPS.h" -#include "Common/Arm64Emitter.h" - -namespace Arm64JitConstants { - -const Arm64Gen::ARM64Reg DOWNCOUNTREG = Arm64Gen::W24; -const Arm64Gen::ARM64Reg FLAGTEMPREG = Arm64Gen::X25; -const Arm64Gen::ARM64Reg JITBASEREG = Arm64Gen::X26; -const Arm64Gen::ARM64Reg CTXREG = Arm64Gen::X27; -const Arm64Gen::ARM64Reg MEMBASEREG = Arm64Gen::X28; -const Arm64Gen::ARM64Reg SCRATCH1_64 = Arm64Gen::X16; -const Arm64Gen::ARM64Reg SCRATCH2_64 = Arm64Gen::X17; -const Arm64Gen::ARM64Reg SCRATCH1 = Arm64Gen::W16; -const Arm64Gen::ARM64Reg SCRATCH2 = Arm64Gen::W17; - -enum { - TOTAL_MAPPABLE_MIPSREGS = 36, -}; - -enum RegMIPSLoc { - ML_IMM, - ML_ARMREG, - // In an arm reg, but an adjusted pointer (not pointerified - unaligned.) - ML_ARMREG_AS_PTR, - // In an arm reg, but also has a known immediate value. - ML_ARMREG_IMM, - ML_MEM, -}; - -// Initing is the default so the flag is reversed. -enum { - MAP_DIRTY = 1, - MAP_NOINIT = 2 | MAP_DIRTY, -}; - -} // namespace - -typedef int MIPSReg; - -struct RegARM64 { - MIPSGPReg mipsReg; // if -1, no mipsreg attached. - bool isDirty; // Should the register be written back? - bool pointerified; // Has used movk to move the memory base into the top part of the reg. Note - still usable as 32-bit reg! - bool tempLocked; // Reserved for a temp register. -}; - -struct RegMIPS { - // Where is this MIPS register? - Arm64JitConstants::RegMIPSLoc loc; - // Data (both or only one may be used, depending on loc.) - u64 imm; - Arm64Gen::ARM64Reg reg; // reg index - bool spillLock; // if true, this register cannot be spilled. - bool isStatic; // if true, this register will not be written back to ram by the regcache - // If loc == ML_MEM, it's back in its location in the CPU context struct. -}; - -namespace MIPSComp { - struct JitOptions; - struct JitState; -} - -class Arm64RegCache { -public: - Arm64RegCache(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo); - ~Arm64RegCache() {} - - void Init(Arm64Gen::ARM64XEmitter *emitter); - void Start(); - - // Protect the arm register containing a MIPS register from spilling, to ensure that - // it's being kept allocated. - void SpillLock(MIPSGPReg reg, MIPSGPReg reg2 = MIPS_REG_INVALID, MIPSGPReg reg3 = MIPS_REG_INVALID, MIPSGPReg reg4 = MIPS_REG_INVALID); - void ReleaseSpillLock(MIPSGPReg reg, MIPSGPReg reg2 = MIPS_REG_INVALID, MIPSGPReg reg3 = MIPS_REG_INVALID, MIPSGPReg reg4 = MIPS_REG_INVALID); - void ReleaseSpillLocksAndDiscardTemps(); - - void SetImm(MIPSGPReg reg, u64 immVal); - bool IsImm(MIPSGPReg reg) const; - bool IsPureImm(MIPSGPReg reg) const; - u64 GetImm(MIPSGPReg reg) const; - void SetRegImm(Arm64Gen::ARM64Reg reg, u64 imm); - - // May fail and return INVALID_REG if it needs flushing. - Arm64Gen::ARM64Reg TryMapTempImm(MIPSGPReg); - - // Returns an ARM register containing the requested MIPS register. - Arm64Gen::ARM64Reg MapReg(MIPSGPReg reg, int mapFlags = 0); - Arm64Gen::ARM64Reg MapRegAsPointer(MIPSGPReg reg); - - bool IsMapped(MIPSGPReg reg); - bool IsMappedAsPointer(MIPSGPReg reg); - bool IsInRAM(MIPSGPReg reg); - - void MarkDirty(Arm64Gen::ARM64Reg reg); - void MapIn(MIPSGPReg rs); - void MapInIn(MIPSGPReg rd, MIPSGPReg rs); - void MapDirtyIn(MIPSGPReg rd, MIPSGPReg rs, bool avoidLoad = true); - void MapDirtyInIn(MIPSGPReg rd, MIPSGPReg rs, MIPSGPReg rt, bool avoidLoad = true); - void MapDirtyDirtyIn(MIPSGPReg rd1, MIPSGPReg rd2, MIPSGPReg rs, bool avoidLoad = true); - void MapDirtyDirtyInIn(MIPSGPReg rd1, MIPSGPReg rd2, MIPSGPReg rs, MIPSGPReg rt, bool avoidLoad = true); - void FlushArmReg(Arm64Gen::ARM64Reg r); - void FlushBeforeCall(); - void FlushAll(); - void FlushR(MIPSGPReg r); - void DiscardR(MIPSGPReg r); - - Arm64Gen::ARM64Reg GetAndLockTempR(); - - Arm64Gen::ARM64Reg R(MIPSGPReg preg); // Returns a cached register, while checking that it's NOT mapped as a pointer - Arm64Gen::ARM64Reg RPtr(MIPSGPReg preg); // Returns a cached register, if it has been mapped as a pointer - - void SetEmitter(Arm64Gen::ARM64XEmitter *emitter) { emit_ = emitter; } - - // For better log output only. - void SetCompilerPC(u32 compilerPC) { compilerPC_ = compilerPC; } - - int GetMipsRegOffset(MIPSGPReg r); - - // These are called once on startup to generate functions, that you should then call. - void EmitLoadStaticRegisters(); - void EmitSaveStaticRegisters(); - -private: - struct StaticAllocation { - MIPSGPReg mr; - Arm64Gen::ARM64Reg ar; - bool pointerified; - }; - const StaticAllocation *GetStaticAllocations(int &count); - const Arm64Gen::ARM64Reg *GetMIPSAllocationOrder(int &count); - void MapRegTo(Arm64Gen::ARM64Reg reg, MIPSGPReg mipsReg, int mapFlags); - Arm64Gen::ARM64Reg AllocateReg(); - Arm64Gen::ARM64Reg FindBestToSpill(bool unusedOnly, bool *clobbered); - Arm64Gen::ARM64Reg ARM64RegForFlush(MIPSGPReg r); - - MIPSState *mips_; - Arm64Gen::ARM64XEmitter *emit_; - MIPSComp::JitState *js_; - MIPSComp::JitOptions *jo_; - u32 compilerPC_; - - enum { - NUM_ARMREG = 32, // 31 actual registers, plus the zero/sp register which is not mappable. - NUM_MIPSREG = Arm64JitConstants::TOTAL_MAPPABLE_MIPSREGS, - }; - - RegARM64 ar[NUM_ARMREG]; - RegMIPS mr[NUM_MIPSREG]; -}; diff --git a/Core/MIPS/ARM64/Arm64RegCacheFPU.cpp b/Core/MIPS/ARM64/Arm64RegCacheFPU.cpp deleted file mode 100644 index cfb787618d..0000000000 --- a/Core/MIPS/ARM64/Arm64RegCacheFPU.cpp +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(ARM64) - -#include - -#include "Common/CPUDetect.h" -#include "Common/Log.h" -#include "Core/Reporting.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/ARM64/Arm64RegCacheFPU.h" -#include "Core/MIPS/ARM64/Arm64Jit.h" -#include "Core/MIPS/MIPSTables.h" - -using namespace Arm64Gen; -using namespace Arm64JitConstants; - -Arm64RegCacheFPU::Arm64RegCacheFPU(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo) : mips_(mipsState), vr(mr + 32), js_(js), jo_(jo) { - numARMFpuReg_ = 32; -} - -void Arm64RegCacheFPU::Init(Arm64Gen::ARM64XEmitter *emit, Arm64Gen::ARM64FloatEmitter *fp) { - emit_ = emit; - fp_ = fp; -} - -void Arm64RegCacheFPU::Start() { - if (!initialReady) { - SetupInitialRegs(); - initialReady = true; - } - - memcpy(ar, arInitial, sizeof(ar)); - memcpy(mr, mrInitial, sizeof(mr)); - pendingFlush = false; -} - -void Arm64RegCacheFPU::SetupInitialRegs() { - for (int i = 0; i < numARMFpuReg_; i++) { - arInitial[i].mipsReg = -1; - arInitial[i].isDirty = false; - } - for (int i = 0; i < NUM_MIPSFPUREG; i++) { - mrInitial[i].loc = ML_MEM; - mrInitial[i].reg = INVALID_REG; - mrInitial[i].spillLock = false; - mrInitial[i].tempLock = false; - } -} - -const ARM64Reg *Arm64RegCacheFPU::GetMIPSAllocationOrder(int &count) { - // On ARM64, all 32 registers are fully 128-bit and fully interchangable so we don't - // have to care about upper or lower registers. However, only S8-S15 are callee-save, and - // only the bottom 64 bits of those. So we should allocate into these when we call - // C functions, although we don't currently do so... - - static const ARM64Reg allocationOrder[] = { - // Reserve four full 128-bit temp registers, should be plenty. - S4, S5, S6, S7, - S8, S9, S10, S11, // Partially callee-save (bottom 64 bits) - S12, S13, S14, S15, // Partially callee-save (bottom 64 bits) - S16, S17, S18, S19, - S20, S21, S22, S23, - S24, S25, S26, S27, - S28, S29, S30, S31, - }; - - count = sizeof(allocationOrder) / sizeof(const ARM64Reg); - return allocationOrder; -} - -bool Arm64RegCacheFPU::IsMapped(MIPSReg r) { - return mr[r].loc == ML_ARMREG; -} - -bool Arm64RegCacheFPU::IsInRAM(MIPSReg r) { - return mr[r].loc == ML_MEM; -} - -ARM64Reg Arm64RegCacheFPU::MapReg(MIPSReg mipsReg, int mapFlags) { - pendingFlush = true; - // Let's see if it's already mapped. If so we just need to update the dirty flag. - // We don't need to check for ML_NOINIT because we assume that anyone who maps - // with that flag immediately writes a "known" value to the register. - if (mr[mipsReg].loc == ML_ARMREG) { - if (ar[mr[mipsReg].reg].mipsReg != mipsReg) { - ERROR_LOG(Log::JIT, "Reg mapping out of sync! MR %i", mipsReg); - } - if (mapFlags & MAP_DIRTY) { - ar[mr[mipsReg].reg].isDirty = true; - } - //INFO_LOG(Log::JIT, "Already mapped %i to %i", mipsReg, mr[mipsReg].reg); - return (ARM64Reg)(mr[mipsReg].reg + S0); - } - - // Okay, not mapped, so we need to allocate an ARM register. - - int allocCount; - const ARM64Reg *allocOrder = GetMIPSAllocationOrder(allocCount); - -allocate: - for (int i = 0; i < allocCount; i++) { - int reg = DecodeReg(allocOrder[i]); - - if (ar[reg].mipsReg == -1) { - // That means it's free. Grab it, and load the value into it (if requested). - ar[reg].isDirty = (mapFlags & MAP_DIRTY) ? true : false; - if ((mapFlags & MAP_NOINIT) != MAP_NOINIT) { - if (mr[mipsReg].loc == ML_MEM && mipsReg < TEMP0) { - fp_->LDR(32, INDEX_UNSIGNED, (ARM64Reg)(reg + S0), CTXREG, GetMipsRegOffset(mipsReg)); - } - } - ar[reg].mipsReg = mipsReg; - mr[mipsReg].loc = ML_ARMREG; - mr[mipsReg].reg = reg; - //INFO_LOG(Log::JIT, "Mapped %i to %i", mipsReg, mr[mipsReg].reg); - return (ARM64Reg)(reg + S0); - } - } - - - // Still nothing. Let's spill a reg and goto 10. - // TODO: Use age or something to choose which register to spill? - // TODO: Spill dirty regs first? or opposite? - int bestToSpill = -1; - for (int i = 0; i < allocCount; i++) { - int reg = allocOrder[i] - S0; - if (ar[reg].mipsReg != -1 && (mr[ar[reg].mipsReg].spillLock || mr[ar[reg].mipsReg].tempLock)) - continue; - bestToSpill = reg; - break; - } - - if (bestToSpill != -1) { - FlushArmReg((ARM64Reg)(S0 + bestToSpill)); - goto allocate; - } - - // Uh oh, we have all them spilllocked.... - ERROR_LOG(Log::JIT, "Out of spillable registers at PC %08x!!!", js_->compilerPC); - return INVALID_REG; -} - -void Arm64RegCacheFPU::MapInIn(MIPSReg rd, MIPSReg rs) { - SpillLock(rd, rs); - MapReg(rd); - MapReg(rs); - ReleaseSpillLock(rd); - ReleaseSpillLock(rs); -} - -void Arm64RegCacheFPU::MapDirtyIn(MIPSReg rd, MIPSReg rs, bool avoidLoad) { - SpillLock(rd, rs); - bool load = !avoidLoad || rd == rs; - MapReg(rd, load ? MAP_DIRTY : MAP_NOINIT); - MapReg(rs); - ReleaseSpillLock(rd); - ReleaseSpillLock(rs); -} - -void Arm64RegCacheFPU::MapDirtyInIn(MIPSReg rd, MIPSReg rs, MIPSReg rt, bool avoidLoad) { - SpillLock(rd, rs, rt); - bool load = !avoidLoad || (rd == rs || rd == rt); - MapReg(rd, load ? MAP_DIRTY : MAP_NOINIT); - MapReg(rt); - MapReg(rs); - ReleaseSpillLock(rd); - ReleaseSpillLock(rs); - ReleaseSpillLock(rt); -} - -void Arm64RegCacheFPU::SpillLockV(const u8 *v, VectorSize sz) { - for (int i = 0; i < GetNumVectorElements(sz); i++) { - vr[v[i]].spillLock = true; - } -} - -void Arm64RegCacheFPU::SpillLockV(int vec, VectorSize sz) { - u8 v[4]; - GetVectorRegs(v, sz, vec); - SpillLockV(v, sz); -} - -void Arm64RegCacheFPU::MapRegV(int vreg, int flags) { - MapReg(vreg + 32, flags); -} - -void Arm64RegCacheFPU::LoadToRegV(ARM64Reg armReg, int vreg) { - if (vr[vreg].loc == ML_ARMREG) { - fp_->FMOV(armReg, (ARM64Reg)(S0 + vr[vreg].reg)); - } else { - MapRegV(vreg); - fp_->FMOV(armReg, V(vreg)); - } -} - -void Arm64RegCacheFPU::MapRegsAndSpillLockV(int vec, VectorSize sz, int flags) { - u8 v[4]; - GetVectorRegs(v, sz, vec); - SpillLockV(v, sz); - for (int i = 0; i < GetNumVectorElements(sz); i++) { - MapRegV(v[i], flags); - } -} - -void Arm64RegCacheFPU::MapRegsAndSpillLockV(const u8 *v, VectorSize sz, int flags) { - SpillLockV(v, sz); - for (int i = 0; i < GetNumVectorElements(sz); i++) { - MapRegV(v[i], flags); - } -} - -void Arm64RegCacheFPU::MapInInV(int vs, int vt) { - SpillLockV(vs); - SpillLockV(vt); - MapRegV(vs); - MapRegV(vt); - ReleaseSpillLockV(vs); - ReleaseSpillLockV(vt); -} - -void Arm64RegCacheFPU::MapDirtyInV(int vd, int vs, bool avoidLoad) { - bool load = !avoidLoad || (vd == vs); - SpillLockV(vd); - SpillLockV(vs); - MapRegV(vd, load ? MAP_DIRTY : MAP_NOINIT); - MapRegV(vs); - ReleaseSpillLockV(vd); - ReleaseSpillLockV(vs); -} - -void Arm64RegCacheFPU::MapDirtyInInV(int vd, int vs, int vt, bool avoidLoad) { - bool load = !avoidLoad || (vd == vs || vd == vt); - SpillLockV(vd); - SpillLockV(vs); - SpillLockV(vt); - MapRegV(vd, load ? MAP_DIRTY : MAP_NOINIT); - MapRegV(vs); - MapRegV(vt); - ReleaseSpillLockV(vd); - ReleaseSpillLockV(vs); - ReleaseSpillLockV(vt); -} - -void Arm64RegCacheFPU::FlushBeforeCall() { - // Only the bottom 64 bits of D8-D15 are callee-saved, which is all a single needs. - for (int i = 0; i < 32; i++) { - if (i < 8 || i > 15) { - FlushArmReg((ARM64Reg)(S0 + i)); - } - } -} - -void Arm64RegCacheFPU::FlushArmReg(ARM64Reg r) { - if (r >= S0 && r <= S31) { - int reg = r - S0; - if (ar[reg].mipsReg == -1) { - // Nothing to do, reg not mapped. - return; - } - if (ar[reg].mipsReg != -1) { - if (ar[reg].isDirty && mr[ar[reg].mipsReg].loc == ML_ARMREG){ - //INFO_LOG(Log::JIT, "Flushing ARM reg %i", reg); - fp_->STR(32, INDEX_UNSIGNED, r, CTXREG, GetMipsRegOffset(ar[reg].mipsReg)); - } - // IMMs won't be in an ARM reg. - mr[ar[reg].mipsReg].loc = ML_MEM; - mr[ar[reg].mipsReg].reg = INVALID_REG; - } else { - ERROR_LOG(Log::JIT, "Dirty but no mipsreg?"); - } - ar[reg].mipsReg = -1; - ar[reg].isDirty = false; - } -} - -void Arm64RegCacheFPU::FlushV(MIPSReg r) { - FlushR(r + 32); -} - -void Arm64RegCacheFPU::FlushR(MIPSReg r) { - switch (mr[r].loc) { - case ML_IMM: - // IMM is always "dirty". - // IMM is not allowed for FP (yet). - ERROR_LOG(Log::JIT, "Imm in FP register?"); - break; - - case ML_ARMREG: - if (mr[r].reg == INVALID_REG) { - ERROR_LOG(Log::JIT, "FlushR: MipsReg had bad ArmReg"); - } - FlushArmReg((ARM64Reg)(S0 + mr[r].reg)); - break; - - case ML_MEM: - // Already there, nothing to do. - break; - - default: - //BAD - break; - } -} - -Arm64Gen::ARM64Reg Arm64RegCacheFPU::ARM64RegForFlush(int r) { - switch (mr[r].loc) { - case ML_IMM: - // IMM is always "dirty". - // IMM is not allowed for FP (yet). - ERROR_LOG(Log::JIT, "Imm in FP register?"); - return INVALID_REG; - - case ML_ARMREG: - if (mr[r].reg == INVALID_REG) { - ERROR_LOG_REPORT(Log::JIT, "ARM64RegForFlush: MipsReg %d had bad ArmReg", r); - return INVALID_REG; - } - // No need to flush if it's not dirty. - if (!ar[mr[r].reg].isDirty) { - return INVALID_REG; - } - return (ARM64Reg)(S0 + mr[r].reg); - - case ML_MEM: - return INVALID_REG; - - default: - ERROR_LOG_REPORT(Log::JIT, "ARM64RegForFlush: MipsReg %d with invalid location %d", r, mr[r].loc); - return INVALID_REG; - } -} - -void Arm64RegCacheFPU::FlushAll() { - if (!pendingFlush) { - // Nothing allocated. FPU regs are not nearly as common as GPR. - return; - } - - // Discard temps! - for (int i = TEMP0; i < TEMP0 + NUM_TEMPS; i++) { - DiscardR(i); - } - - int numArmRegs = 0; - - const ARM64Reg *order = GetMIPSAllocationOrder(numArmRegs); - - // Flush pairs first when possible. Note that STP's offset can't reach more than 256 bytes so - // most VFPU registers cannot be flushed this way, unless we are willing to generate another offset pointer - // (which we could actually do right here, point right in the middle of the VFPU stuff and would reach it all)... TODO - for (int i = 0; i < 31; i++) { - int mr1 = i; - int mr2 = i + 1; - ARM64Reg ar1 = ARM64RegForFlush(mr1); - ARM64Reg ar2 = ARM64RegForFlush(mr2); - - if (ar1 != INVALID_REG && ar2 != INVALID_REG) { - fp_->STP(32, INDEX_SIGNED, ar1, ar2, CTXREG, GetMipsRegOffset(mr1)); - DiscardR(mr1); - DiscardR(mr2); - } - } - - // Then flush one by one. - - for (int i = 0; i < numArmRegs; i++) { - int a = DecodeReg(order[i]); - int m = ar[a].mipsReg; - - if (ar[a].isDirty) { - if (m == -1) { - INFO_LOG(Log::JIT, "ARM reg %d is dirty but has no mipsreg", a); - continue; - } - - fp_->STR(32, INDEX_UNSIGNED, (ARM64Reg)(a + S0), CTXREG, GetMipsRegOffset(m)); - - mr[m].loc = ML_MEM; - mr[m].reg = (int)INVALID_REG; - ar[a].mipsReg = -1; - ar[a].isDirty = false; - } else { - if (m != -1) { - mr[m].loc = ML_MEM; - mr[m].reg = (int)INVALID_REG; - } - ar[a].mipsReg = -1; - // already not dirty - } - } - - // Sanity check - for (int i = 0; i < numARMFpuReg_; i++) { - if (ar[i].mipsReg != -1) { - ERROR_LOG(Log::JIT, "Flush fail: ar[%d].mipsReg=%d", i, ar[i].mipsReg); - } - } - pendingFlush = false; -} - -void Arm64RegCacheFPU::DiscardR(MIPSReg r) { - switch (mr[r].loc) { - case ML_IMM: - // IMM is always "dirty". - // IMM is not allowed for FP (yet). - ERROR_LOG(Log::JIT, "Imm in FP register?"); - break; - - case ML_ARMREG: - if (mr[r].reg == INVALID_REG) { - ERROR_LOG(Log::JIT, "DiscardR: MipsReg had bad ArmReg"); - } else { - // Note that we DO NOT write it back here. That's the whole point of Discard. - ar[mr[r].reg].isDirty = false; - ar[mr[r].reg].mipsReg = -1; - } - break; - - case ML_MEM: - // Already there, nothing to do. - break; - - default: - //BAD - break; - } - mr[r].loc = ML_MEM; - mr[r].reg = (int)INVALID_REG; - mr[r].tempLock = false; - mr[r].spillLock = false; -} - -bool Arm64RegCacheFPU::IsTempX(ARM64Reg r) const { - return ar[r - S0].mipsReg >= TEMP0; -} - -int Arm64RegCacheFPU::GetTempR() { - pendingFlush = true; - for (int r = TEMP0; r < TEMP0 + NUM_TEMPS; ++r) { - if (mr[r].loc == ML_MEM && !mr[r].tempLock) { - mr[r].tempLock = true; - return r; - } - } - - ERROR_LOG(Log::CPU, "Out of temp regs! Might need to DiscardR() some"); - _assert_msg_(false, "Regcache ran out of temp regs, might need to DiscardR() some."); - return -1; -} - -int Arm64RegCacheFPU::GetMipsRegOffset(MIPSReg r) { - // These are offsets within the MIPSState structure. First there are the GPRS, then FPRS, then the "VFPURs", then the VFPU ctrls. - if (r < 0 || r > 32 + 128 + NUM_TEMPS) { - ERROR_LOG(Log::JIT, "bad mips register %i, out of range", r); - return 0; // or what? - } - - if (r < 32 || r >= 32 + 128) { - return (32 + r) << 2; - } else { - // r is between 32 and 128 + 32 - return (32 + 32 + voffset[r - 32]) << 2; - } -} - -void Arm64RegCacheFPU::SpillLock(MIPSReg r1, MIPSReg r2, MIPSReg r3, MIPSReg r4) { - mr[r1].spillLock = true; - if (r2 != -1) mr[r2].spillLock = true; - if (r3 != -1) mr[r3].spillLock = true; - if (r4 != -1) mr[r4].spillLock = true; -} - -// This is actually pretty slow with all the 160 regs... -void Arm64RegCacheFPU::ReleaseSpillLocksAndDiscardTemps() { - for (int i = 0; i < NUM_MIPSFPUREG; i++) { - mr[i].spillLock = false; - } - for (int i = TEMP0; i < TEMP0 + NUM_TEMPS; ++i) { - DiscardR(i); - } -} - -ARM64Reg Arm64RegCacheFPU::R(int mipsReg) { - if (mr[mipsReg].loc == ML_ARMREG) { - return (ARM64Reg)(mr[mipsReg].reg + S0); - } else { - if (mipsReg < 32) { - ERROR_LOG(Log::JIT, "FReg %i not in ARM reg. compilerPC = %08x : %s", mipsReg, js_->compilerPC, MIPSDisasmAt(js_->compilerPC).c_str()); - } else if (mipsReg < 32 + 128) { - ERROR_LOG(Log::JIT, "VReg %i not in ARM reg. compilerPC = %08x : %s", mipsReg - 32, js_->compilerPC, MIPSDisasmAt(js_->compilerPC).c_str()); - } else { - ERROR_LOG(Log::JIT, "Tempreg %i not in ARM reg. compilerPC = %08x : %s", mipsReg - 128 - 32, js_->compilerPC, MIPSDisasmAt(js_->compilerPC).c_str()); - } - return INVALID_REG; // BAAAD - } -} - -#endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64RegCacheFPU.h b/Core/MIPS/ARM64/Arm64RegCacheFPU.h deleted file mode 100644 index 67cc3220f0..0000000000 --- a/Core/MIPS/ARM64/Arm64RegCacheFPU.h +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/ARM64/Arm64RegCache.h" -#include "Core/MIPS/MIPSVFPUUtils.h" -#include "Common/Arm64Emitter.h" - -namespace Arm64JitConstants { - -enum { - NUM_TEMPS = 16, - TEMP0 = 32 + 128, - TOTAL_MAPPABLE_MIPSFPUREGS = 32 + 128 + NUM_TEMPS, -}; - -enum { - MAP_READ = 0, - MAP_MTX_TRANSPOSED = 16, - MAP_PREFER_LOW = 16, - MAP_PREFER_HIGH = 32, - - // Force is not yet correctly implemented, if the reg is already mapped it will not move - MAP_FORCE_LOW = 64, // Only map Q0-Q7 (and probably not Q0-Q3 as they are S registers so that leaves Q8-Q15) - MAP_FORCE_HIGH = 128, // Only map Q8-Q15 -}; - -} - -struct FPURegARM64 { - int mipsReg; // if -1, no mipsreg attached. - bool isDirty; // Should the register be written back? -}; - -struct FPURegQuad64 { - int mipsVec; - VectorSize sz; - u8 vregs[4]; - bool isDirty; - bool spillLock; - bool isTemp; -}; - -struct FPURegMIPS { - // Where is this MIPS register? - Arm64JitConstants::RegMIPSLoc loc; - // Data (only one of these is used, depending on loc. Could make a union). - u32 reg; - int lane; - - bool spillLock; // if true, this register cannot be spilled. - bool tempLock; - // If loc == ML_MEM, it's back in its location in the CPU context struct. -}; - -namespace MIPSComp { - struct JitOptions; - struct JitState; -} - -class Arm64RegCacheFPU { -public: - Arm64RegCacheFPU(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo); - ~Arm64RegCacheFPU() {} - - void Init(Arm64Gen::ARM64XEmitter *emitter, Arm64Gen::ARM64FloatEmitter *fp); - - void Start(); - - // Protect the arm register containing a MIPS register from spilling, to ensure that - // it's being kept allocated. - void SpillLock(MIPSReg reg, MIPSReg reg2 = -1, MIPSReg reg3 = -1, MIPSReg reg4 = -1); - void SpillLockV(MIPSReg r) { SpillLock(r + 32); } - - void ReleaseSpillLocksAndDiscardTemps(); - void ReleaseSpillLock(int mipsreg) { - mr[mipsreg].spillLock = false; - } - void ReleaseSpillLockV(int mipsreg) { - ReleaseSpillLock(mipsreg + 32); - } - - void SetImm(MIPSReg reg, u32 immVal); - bool IsImm(MIPSReg reg) const; - u32 GetImm(MIPSReg reg) const; - - // Returns an ARM register containing the requested MIPS register. - Arm64Gen::ARM64Reg MapReg(MIPSReg reg, int mapFlags = 0); - void MapInIn(MIPSReg rd, MIPSReg rs); - void MapDirty(MIPSReg rd); - void MapDirtyIn(MIPSReg rd, MIPSReg rs, bool avoidLoad = true); - void MapDirtyInIn(MIPSReg rd, MIPSReg rs, MIPSReg rt, bool avoidLoad = true); - bool IsMapped(MIPSReg r); - bool IsMappedV(MIPSReg r) { return IsMapped((MIPSReg)(r + 32)); } - bool IsInRAM(MIPSReg r); - bool IsInRAMV(MIPSReg r) { return IsInRAM((MIPSReg)(r + 32)); } - void FlushArmReg(Arm64Gen::ARM64Reg r); - void FlushR(MIPSReg r); - void DiscardR(MIPSReg r); - Arm64Gen::ARM64Reg R(int preg); // Returns a cached register - - void MapRegV(int vreg, int flags = 0); - void LoadToRegV(Arm64Gen::ARM64Reg armReg, int vreg); - void MapInInV(int rt, int rs); - void MapDirtyInV(int rd, int rs, bool avoidLoad = true); - void MapDirtyInInV(int rd, int rs, int rt, bool avoidLoad = true); - - bool IsTempX(Arm64Gen::ARM64Reg r) const; - MIPSReg GetTempV() { return GetTempR() - 32; } - // VFPU registers as single VFP registers. - Arm64Gen::ARM64Reg V(int vreg) { return R(vreg + 32); } - - void FlushAll(); - // Flushes only the registers a call doesn't preserve. S8-S15 stay mapped. - void FlushBeforeCall(); - - // This one is allowed at any point. - void FlushV(MIPSReg r); - - // NOTE: These require you to release spill locks manually! - void MapRegsAndSpillLockV(int vec, VectorSize vsz, int flags); - void MapRegsAndSpillLockV(const u8 *v, VectorSize vsz, int flags); - - void SpillLockV(const u8 *v, VectorSize vsz); - void SpillLockV(int vec, VectorSize vsz); - - void SetEmitter(Arm64Gen::ARM64XEmitter *emitter, Arm64Gen::ARM64FloatEmitter *fp) { emit_ = emitter; fp_ = fp; } - - int GetMipsRegOffset(MIPSReg r); - int GetMipsRegOffsetV(MIPSReg r) { - return GetMipsRegOffset(r + 32); - } - -private: - Arm64Gen::ARM64Reg ARM64RegForFlush(int r); - MIPSReg GetTempR(); - const Arm64Gen::ARM64Reg *GetMIPSAllocationOrder(int &count); - - void SetupInitialRegs(); - - MIPSState *mips_; - Arm64Gen::ARM64XEmitter *emit_; - Arm64Gen::ARM64FloatEmitter *fp_; - MIPSComp::JitState *js_; - MIPSComp::JitOptions *jo_; - - int numARMFpuReg_; - - enum { - // On ARM64, each of the 32 registers are full 128-bit. No sharing of components! - MAX_ARMFPUREG = 32, - NUM_MIPSFPUREG = Arm64JitConstants::TOTAL_MAPPABLE_MIPSFPUREGS, - }; - - FPURegARM64 ar[MAX_ARMFPUREG]; - FPURegMIPS mr[NUM_MIPSFPUREG]; - FPURegMIPS *vr; - - bool pendingFlush; - bool initialReady = false; - FPURegARM64 arInitial[MAX_ARMFPUREG]; - FPURegMIPS mrInitial[NUM_MIPSFPUREG]; -}; diff --git a/Core/MIPS/JitCommon/JitCommon.cpp b/Core/MIPS/JitCommon/JitCommon.cpp index 37b3e2744d..f54802e3da 100644 --- a/Core/MIPS/JitCommon/JitCommon.cpp +++ b/Core/MIPS/JitCommon/JitCommon.cpp @@ -41,10 +41,8 @@ #if PPSSPP_ARCH(ARM) #include "../ARM/ArmJit.h" #elif PPSSPP_ARCH(ARM64) -#include "../ARM64/Arm64Jit.h" #include "../ARM64/Arm64IRJit.h" #elif PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) -#include "../x86/Jit.h" #include "../x86/X64IRJit.h" #elif PPSSPP_ARCH(RISCV64) #include "../RiscV/RiscVJit.h" @@ -105,19 +103,13 @@ namespace MIPSComp { return notTakenTarget; } - JitInterface *CreateNativeJit(MIPSState *mipsState, bool useIR) { + JitInterface *CreateNativeJit(MIPSState *mipsState) { #if PPSSPP_ARCH(ARM) return new MIPSComp::ArmJit(mipsState); #elif PPSSPP_ARCH(ARM64) - if (useIR) - return new MIPSComp::Arm64IRJit(mipsState); - else - return new MIPSComp::Arm64Jit(mipsState); + return new MIPSComp::Arm64IRJit(mipsState); #elif PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - if (useIR) - return new MIPSComp::X64IRJit(mipsState); - else - return new MIPSComp::Jit(mipsState); + return new MIPSComp::X64IRJit(mipsState); #elif PPSSPP_ARCH(RISCV64) return new MIPSComp::RiscVJit(mipsState); #elif PPSSPP_ARCH(LOONGARCH64) diff --git a/Core/MIPS/JitCommon/JitCommon.h b/Core/MIPS/JitCommon/JitCommon.h index 3a9b69d8f6..4dc09a892c 100644 --- a/Core/MIPS/JitCommon/JitCommon.h +++ b/Core/MIPS/JitCommon/JitCommon.h @@ -169,5 +169,5 @@ namespace MIPSComp { void DoDummyJitState(PointerWrap &p); - JitInterface *CreateNativeJit(MIPSState *mipsState, bool useIR); + JitInterface *CreateNativeJit(MIPSState *mipsState); } diff --git a/Core/MIPS/MIPS.cpp b/Core/MIPS/MIPS.cpp index 96745550b7..f975cbc95c 100644 --- a/Core/MIPS/MIPS.cpp +++ b/Core/MIPS/MIPS.cpp @@ -360,7 +360,7 @@ void MIPSState::Init() { memset(vcmpResult, 0, sizeof(vcmpResult)); if (PSP_CoreParameter().cpuCore == CPUCore::JIT || PSP_CoreParameter().cpuCore == CPUCore::JIT_IR) { - MIPSComp::jit = MIPSComp::CreateNativeJit(this, PSP_CoreParameter().cpuCore == CPUCore::JIT_IR); + MIPSComp::jit = MIPSComp::CreateNativeJit(this); } else if (PSP_CoreParameter().cpuCore == CPUCore::IR_INTERPRETER) { MIPSComp::jit = new MIPSComp::IRJit(this, false); } else { @@ -393,7 +393,7 @@ void MIPSState::UpdateCore(CPUCore desired) { case CPUCore::JIT: case CPUCore::JIT_IR: INFO_LOG(Log::CPU, "Switching to JIT%s", PSP_CoreParameter().cpuCore == CPUCore::JIT_IR ? " IR" : ""); - newjit = MIPSComp::CreateNativeJit(this, PSP_CoreParameter().cpuCore == CPUCore::JIT_IR); + newjit = MIPSComp::CreateNativeJit(this); break; case CPUCore::IR_INTERPRETER: diff --git a/Core/MIPS/MIPS.h b/Core/MIPS/MIPS.h index 6fec58da17..d38090cc39 100644 --- a/Core/MIPS/MIPS.h +++ b/Core/MIPS/MIPS.h @@ -148,22 +148,6 @@ extern u8 fromvoffset[128]; enum class CPUCore; -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -// Note that CTXREG is offset to point at the first floating point register, intentionally. This is so that a byte offset -// can reach both GPR and FPR regs. -#define MIPSSTATE_VAR(x) MDisp(X64JitConstants::CTXREG, \ - (int)(offsetof(MIPSState, x) - offsetof(MIPSState, f[0]))) - -// Workaround for compilers that don't like dynamic indexing in offsetof -#define MIPSSTATE_VAR_ELEM32(x, i) MDisp(X64JitConstants::CTXREG, \ - (int)(offsetof(MIPSState, x) - offsetof(MIPSState, f[0]) + (i) * 4)) - -// To get RIP/relative addressing (requires tight memory control so generated code isn't too far from the binary, and a reachable variable called mips): -// #define MIPSSTATE_VAR(x) M(&mips_->x) - -#endif - // Only icache invalidations currently, so no type field. struct PendingCacheOperation { u32 addr; diff --git a/Core/MIPS/x86/Asm.cpp b/Core/MIPS/x86/Asm.cpp deleted file mode 100644 index d404438712..0000000000 --- a/Core/MIPS/x86/Asm.cpp +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright (C) 2003 Dolphin Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official SVN repository and contact information can be found at -// http://code.google.com/p/dolphin-emu/ - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Common/Math/math_util.h" - -#include "ABI.h" -#include "x64Emitter.h" - -#include "Core/Core.h" -#include "Core/MemMap.h" -#include "Core/System.h" -#include "Core/MIPS/MIPS.h" -#include "Core/CoreTiming.h" -#include "Common/MemoryUtil.h" - -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/x86/Jit.h" - -using namespace Gen; -using namespace X64JitConstants; - -extern volatile CoreState coreState; - -namespace MIPSComp -{ - -//TODO - make an option -//#if _DEBUG -static bool enableDebug = false; - -//#else -// bool enableDebug = false; -//#endif - -//static bool enableStatistics = false; //unused? - -//GLOBAL STATIC ALLOCATIONS x86 -//EAX - ubiquitous scratch register - EVERYBODY scratches this -//EBP - Pointer to fpr/gpr regs - -//GLOBAL STATIC ALLOCATIONS x64 -//EAX - ubiquitous scratch register - EVERYBODY scratches this -//RBX - Base pointer of memory -//R14 - Pointer to fpr/gpr regs -//R15 - Pointer to array of block pointers - -void ImHere() { - DEBUG_LOG(Log::CPU, "JIT Here: %08x", currentMIPS->pc); -} - -void Jit::GenerateFixedCode(JitOptions &jo) { - BeginWrite(GetMemoryProtectPageSize()); - AlignCodePage(); - - restoreRoundingMode = AlignCode16(); { - STMXCSR(MIPSSTATE_VAR(temp)); - // Clear the rounding mode, flush-to-zero and denormals-are-zero bits back to 0. - AND(32, MIPSSTATE_VAR(temp), Imm32(~((7 << 13) | (1 << 6)))); - LDMXCSR(MIPSSTATE_VAR(temp)); - RET(); - } - - applyRoundingMode = AlignCode16(); { - MOV(32, R(EAX), MIPSSTATE_VAR(fcr31)); - AND(32, R(EAX), Imm32(0x01000003)); - - // If it's 0 (nearest + no flush0), we don't actually bother setting - we cleared the rounding - // mode out in restoreRoundingMode anyway. This is the most common. - FixupBranch skip = J_CC(CC_Z); - STMXCSR(MIPSSTATE_VAR(temp)); - - // The MIPS bits don't correspond exactly, so we have to adjust. - // 0 -> 0 (skip2), 1 -> 3, 2 -> 2 (skip2), 3 -> 1 - TEST(8, R(AL), Imm8(1)); - FixupBranch skip2 = J_CC(CC_Z); - XOR(32, R(EAX), Imm8(2)); - SetJumpTarget(skip2); - - // Adjustment complete, now reconstruct MXCSR - SHL(32, R(EAX), Imm8(13)); - // Before setting new bits, we must clear the old ones. - // Clearing bits 13-14 (rounding mode), 15 (flush to zero) and 6 (denormals are zero). - AND(32, MIPSSTATE_VAR(temp), Imm32(~((7 << 13) | (1 << 6)))); - OR(32, MIPSSTATE_VAR(temp), R(EAX)); - - TEST(32, MIPSSTATE_VAR(fcr31), Imm32(1 << 24)); - FixupBranch skip3 = J_CC(CC_Z); - // FTZ and DAZ together, which flushes like ARM64's FZ: inputs as well as results. - OR(32, MIPSSTATE_VAR(temp), Imm32((1 << 15) | (1 << 6))); - SetJumpTarget(skip3); - - LDMXCSR(MIPSSTATE_VAR(temp)); - SetJumpTarget(skip); - RET(); - } - - enterDispatcher = AlignCode16(); - ABI_PushAllCalleeSavedRegsAndAdjustStack(); -#if PPSSPP_ARCH(AMD64) - // Two statically allocated registers. - MOV(64, R(MEMBASEREG), ImmPtr(Memory::base)); - uintptr_t jitbase = (uintptr_t)GetBasePtr(); - if (jitbase > 0x7FFFFFFFULL) { - MOV(64, R(JITBASEREG), ImmPtr(GetBasePtr())); - jo.reserveR15ForAsm = true; - } -#endif - // From the start of the FP reg, a single byte offset can reach all GPR + all FPR (but no VFPUR) - MOV(PTRBITS, R(CTXREG), ImmPtr(&mips_->f[0])); - - outerLoop = GetCodePtr(); - RestoreRoundingMode(true); - LEA(PTRBITS, ECX, MDisp(CTXREG, -(s32)offsetof(MIPSState, f[0]))); // Adjust to get the real pointer. - ABI_CallFunctionR(reinterpret_cast(&CoreTiming::Advance), ECX); - ApplyRoundingMode(true); - FixupBranch skipToCoreStateCheck = J(); //skip the downcount check - - dispatcherCheckCoreState = GetCodePtr(); - - // The result of slice decrementation should be in flags if somebody jumped here - // IMPORTANT - We jump on negative, not carry!!! - FixupBranch bailCoreState = J_CC(CC_S, true); - - SetJumpTarget(skipToCoreStateCheck); - if (RipAccessible((const void *)&coreState)) { - CMP(32, M(&coreState), Imm32(0)); // rip accessible - } else { - MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState)); - CMP(32, MatR(RAX), Imm32(0)); - } - FixupBranch badCoreState = J_CC(CC_NZ, true); - FixupBranch skipToRealDispatch2 = J(); //skip the sync and compare first time - - dispatcher = GetCodePtr(); - - // The result of slice decrementation should be in flags if somebody jumped here - // IMPORTANT - We jump on negative, not carry!!! - FixupBranch bail = J_CC(CC_S, true); - - SetJumpTarget(skipToRealDispatch2); - - dispatcherNoCheck = GetCodePtr(); - - MOV(32, R(EAX), MIPSSTATE_VAR(pc)); - dispatcherInEAXNoCheck = GetCodePtr(); - - // Look the block up in the shadow. The address stays in EAX for MemFault if this faults. - AND(32, R(EAX), Imm32(Memory::BLOCK_SHADOW_ENTRY_MASK)); -#if PPSSPP_ARCH(X86) - _assert_msg_(Memory::blockShadow != 0, "Block shadow bogus"); - dispatcherFetch = GetCodePtr(); - MOV(32, R(EAX), MDisp(EAX, (u32)Memory::blockShadow)); -#elif PPSSPP_ARCH(AMD64) - MOV(64, R(RDX), ImmPtr(Memory::blockShadow)); - dispatcherFetch = GetCodePtr(); - MOV(32, R(EAX), MComplex(RDX, RAX, SCALE_1, 0)); -#endif - TEST(32, R(EAX), R(EAX)); - FixupBranch notfound = J_CC(CC_Z); - if (enableDebug) { - ADD(32, MIPSSTATE_VAR(debugCount), Imm8(1)); - } - // The entry is the block's offset into the code space. -#if PPSSPP_ARCH(X86) - ADD(32, R(EAX), ImmPtr(GetBasePtr())); -#elif PPSSPP_ARCH(AMD64) - if (jo.reserveR15ForAsm) { - ADD(64, R(RAX), R(JITBASEREG)); - } else { - // See above, reserveR15ForAsm is used when above 0x7FFFFFFF. - ADD(64, R(EAX), Imm32((u32)jitbase)); - } -#endif - JMPptr(R(EAX)); - SetJumpTarget(notfound); - - //Ok, no block, let's jit - // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt - // replaces it with the new block's entry, and destroying that block puts the hook value back. - blockShadowHook = GetCodePtr(); - RestoreRoundingMode(true); - LEA(PTRBITS, ECX, MDisp(CTXREG, -(s32)offsetof(MIPSState, f[0]))); // Adjust to get the real pointer. - ABI_CallFunctionR(reinterpret_cast(&MIPSComp::JitAt), ECX); - ApplyRoundingMode(true); - JMP(dispatcherNoCheck, true); // Let's just dispatch again, we'll enter the block since we know it's there. - - SetJumpTarget(bail); - SetJumpTarget(bailCoreState); - - if (RipAccessible((const void *)&coreState)) { - CMP(32, M(&coreState), Imm32(0)); // rip accessible - } else { - MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState)); - CMP(32, MatR(RAX), Imm32(0)); - } - J_CC(CC_Z, outerLoop, true); - - const uint8_t *quitLoop = GetCodePtr(); - SetJumpTarget(badCoreState); - RestoreRoundingMode(true); - ABI_PopAllCalleeSavedRegsAndAdjustStack(); - RET(); - - crashHandler = GetCodePtr(); - if (RipAccessible((const void *)&coreState)) { - MOV(32, M(&coreState), Imm32(CORE_RUNTIME_ERROR)); - } else { - MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState)); - MOV(32, MatR(RAX), Imm32(CORE_RUNTIME_ERROR)); - } - JMP(quitLoop, true); - - // Let's spare the pre-generated code from unprotect-reprotect. - endOfPregeneratedCode = AlignCodePage(); - EndWrite(); -} - -} // namespace - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompALU.cpp b/Core/MIPS/x86/CompALU.cpp deleted file mode 100644 index 3f8ecbbe95..0000000000 --- a/Core/MIPS/x86/CompALU.cpp +++ /dev/null @@ -1,1156 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include - -#include "Common/BitSet.h" -#include "Common/CommonTypes.h" -#include "Common/CPUDetect.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" - -using namespace MIPSAnalyst; - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE(ignore) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } - -namespace MIPSComp -{ - using namespace Gen; - using namespace X64JitConstants; - - static bool HasLowSubregister(OpArg arg) { -#if !PPSSPP_ARCH(AMD64) - // Can't use ESI or EDI (which we use), no 8-bit versions. Only these. - if (!arg.IsSimpleReg(EAX) && !arg.IsSimpleReg(EBX) && !arg.IsSimpleReg(ECX) && !arg.IsSimpleReg(EDX)) { - return false; - } -#endif - return arg.IsSimpleReg(); - } - - void Jit::CompImmLogic(MIPSOpcode op, void (XEmitter::*arith)(int, const OpArg &, const OpArg &)) - { - u32 uimm = (u16)(op & 0xFFFF); - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - gpr.Lock(rt, rs); - gpr.MapReg(rt, rt == rs, true); - if (rt != rs) - MOV(32, gpr.R(rt), gpr.R(rs)); - (this->*arith)(32, gpr.R(rt), Imm32(uimm)); - gpr.UnlockAll(); - } - - void Jit::Comp_IType(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU_IMM); - u32 uimm = op & 0xFFFF; - s32 simm = SignExtend16ToS32(op); - u32 suimm = SignExtend16ToU32(op); - - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - // noop, won't write to ZERO. - if (rt == MIPS_REG_ZERO) - return; - - switch (op >> 26) - { - case 8: // same as addiu? - case 9: // R(rt) = R(rs) + simm; break; //addiu - { - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, gpr.GetImm(rs) + simm); - break; - } - - gpr.Lock(rt, rs); - gpr.MapReg(rt, rt == rs, true); - if (rt == rs) { - if (simm > 0) { - ADD(32, gpr.R(rt), UImmAuto(simm)); - } else if (simm < 0) { - SUB(32, gpr.R(rt), UImmAuto(-simm)); - } - } else if (gpr.R(rs).IsSimpleReg()) { - LEA(32, gpr.RX(rt), MDisp(gpr.RX(rs), simm)); - } else { - MOV(32, gpr.R(rt), gpr.R(rs)); - if (simm > 0) - ADD(32, gpr.R(rt), UImmAuto(simm)); - else if (simm < 0) { - SUB(32, gpr.R(rt), UImmAuto(-simm)); - } - } - gpr.UnlockAll(); - } - break; - - case 10: // R(rt) = (s32)R(rs) < simm; break; //slti - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, (s32)gpr.GetImm(rs) < simm); - } else { - gpr.Lock(rt, rs); - // This is often used before a branch. If rs is not already mapped, let's leave it. - gpr.MapReg(rt, rt == rs, true); - - bool needsTemp = !HasLowSubregister(gpr.R(rt)) || rt == rs; - if (needsTemp) { - CMP(32, gpr.R(rs), Imm32(suimm)); - SETcc(CC_L, R(TEMPREG)); - MOVZX(32, 8, gpr.RX(rt), R(TEMPREG)); - } else { - XOR(32, gpr.R(rt), gpr.R(rt)); - CMP(32, gpr.R(rs), Imm32(suimm)); - SETcc(CC_L, gpr.R(rt)); - } - gpr.UnlockAll(); - } - break; - - case 11: // R(rt) = R(rs) < uimm; break; //sltiu - if (gpr.IsImm(rs)) { - gpr.SetImm(rt, gpr.GetImm(rs) < suimm); - } else { - gpr.Lock(rt, rs); - // This is often used before a branch. If rs is not already mapped, let's leave it. - gpr.MapReg(rt, rt == rs, true); - - bool needsTemp = !HasLowSubregister(gpr.R(rt)) || rt == rs; - if (needsTemp) { - CMP(32, gpr.R(rs), Imm32(suimm)); - SETcc(CC_B, R(TEMPREG)); - MOVZX(32, 8, gpr.RX(rt), R(TEMPREG)); - } else { - XOR(32, gpr.R(rt), gpr.R(rt)); - CMP(32, gpr.R(rs), Imm32(suimm)); - SETcc(CC_B, gpr.R(rt)); - } - gpr.UnlockAll(); - } - break; - - case 12: // R(rt) = R(rs) & uimm; break; //andi - if (uimm == 0) - gpr.SetImm(rt, 0); - else if (gpr.IsImm(rs)) - gpr.SetImm(rt, gpr.GetImm(rs) & uimm); - else - CompImmLogic(op, &XEmitter::AND); - break; - - case 13: // R(rt) = R(rs) | uimm; break; //ori - if (gpr.IsImm(rs)) - gpr.SetImm(rt, gpr.GetImm(rs) | uimm); - else - CompImmLogic(op, &XEmitter::OR); - break; - - case 14: // R(rt) = R(rs) ^ uimm; break; //xori - if (gpr.IsImm(rs)) - gpr.SetImm(rt, gpr.GetImm(rs) ^ uimm); - else - CompImmLogic(op, &XEmitter::XOR); - break; - - case 15: //R(rt) = uimm << 16; break; //lui - gpr.SetImm(rt, uimm << 16); - break; - - default: - Comp_Generic(op); - break; - } - } - - void Jit::Comp_RType2(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // Don't change $zr. - if (rd == MIPS_REG_ZERO) - return; - - switch (op & 63) - { - case 22: //clz - if (gpr.IsImm(rs)) - { - u32 value = gpr.GetImm(rs); - int x = 31; - int count = 0; - while (x >= 0 && !(value & (1 << x))) - { - count++; - x--; - } - gpr.SetImm(rd, count); - } - else - { - gpr.Lock(rd, rs); - gpr.MapReg(rd, rd == rs, true); - BSR(32, TEMPREG, gpr.R(rs)); - FixupBranch notFound = J_CC(CC_Z); - - MOV(32, gpr.R(rd), Imm32(31)); - SUB(32, gpr.R(rd), R(TEMPREG)); - FixupBranch skip = J(); - - SetJumpTarget(notFound); - MOV(32, gpr.R(rd), Imm32(32)); - - SetJumpTarget(skip); - gpr.UnlockAll(); - } - break; - case 23: //clo - if (gpr.IsImm(rs)) - { - u32 value = gpr.GetImm(rs); - int x = 31; - int count = 0; - while (x >= 0 && (value & (1 << x))) - { - count++; - x--; - } - gpr.SetImm(rd, count); - } - else - { - gpr.Lock(rd, rs); - gpr.MapReg(rd, rd == rs, true); - MOV(32, R(TEMPREG), gpr.R(rs)); - NOT(32, R(TEMPREG)); - BSR(32, TEMPREG, R(TEMPREG)); - FixupBranch notFound = J_CC(CC_Z); - - MOV(32, gpr.R(rd), Imm32(31)); - SUB(32, gpr.R(rd), R(TEMPREG)); - FixupBranch skip = J(); - - SetJumpTarget(notFound); - MOV(32, gpr.R(rd), Imm32(32)); - - SetJumpTarget(skip); - gpr.UnlockAll(); - } - break; - default: - DISABLE; - } - } - - static u32 RType3_ImmAdd(const u32 a, const u32 b) - { - return a + b; - } - - static u32 RType3_ImmSub(const u32 a, const u32 b) - { - return a - b; - } - - static u32 RType3_ImmAnd(const u32 a, const u32 b) - { - return a & b; - } - - static u32 RType3_ImmOr(const u32 a, const u32 b) - { - return a | b; - } - - static u32 RType3_ImmXor(const u32 a, const u32 b) - { - return a ^ b; - } - - //rd = rs X rt - void Jit::CompTriArith(MIPSOpcode op, void (XEmitter::*arith)(int, const OpArg &, const OpArg &), u32 (*doImm)(const u32, const u32), bool invertResult) - { - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // Both sides known, we can just evaporate the instruction. - if (doImm && gpr.IsImm(rs) && gpr.IsImm(rt)) { - u32 value = doImm(gpr.GetImm(rs), gpr.GetImm(rt)); - gpr.SetImm(rd, invertResult ? (~value) : value); - return; - } - - // Act like zero was used if the operand is equivalent. This happens. - if (gpr.IsImm(rs) && gpr.GetImm(rs) == 0) - rs = MIPS_REG_ZERO; - if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0) - rt = MIPS_REG_ZERO; - - // Special cases that translate nicely - if (doImm == &RType3_ImmSub && rs == MIPS_REG_ZERO && rt == rd) { - gpr.MapReg(rd, true, true); - NEG(32, gpr.R(rd)); - if (invertResult) { - NOT(32, gpr.R(rd)); - } - return; - } - - gpr.Lock(rt, rs, rd); - // Optimize out operations against 0... and is the only one that isn't a MOV. - if (rt == MIPS_REG_ZERO || (rs == MIPS_REG_ZERO && doImm != &RType3_ImmSub)) { - if (doImm == &RType3_ImmAnd) { - gpr.SetImm(rd, invertResult ? 0xFFFFFFFF : 0); - } else { - MIPSGPReg rsource = (rt == MIPS_REG_ZERO) ? rs : rt; - if (rsource != rd) { - gpr.MapReg(rd, false, true); - MOV(32, gpr.R(rd), gpr.R(rsource)); - if (invertResult) { - NOT(32, gpr.R(rd)); - } - } else if (invertResult) { - // rsource == rd, but still need to invert. - gpr.MapReg(rd, true, true); - NOT(32, gpr.R(rd)); - } - } - } else if (gpr.IsImm(rt)) { - // No temporary needed. - u32 rtval = gpr.GetImm(rt); - gpr.MapReg(rd, rs == rd, true); - if (rs != rd) { - MOV(32, gpr.R(rd), gpr.R(rs)); - } - (this->*arith)(32, gpr.R(rd), Imm32(rtval)); - if (invertResult) { - NOT(32, gpr.R(rd)); - } - } else { - // Use TEMPREG as a temporary if we'd overwrite it. - if (rd == rt) - MOV(32, R(TEMPREG), gpr.R(rt)); - gpr.MapReg(rd, rs == rd, true); - if (rs != rd) - MOV(32, gpr.R(rd), gpr.R(rs)); - (this->*arith)(32, gpr.R(rd), rd == rt ? R(TEMPREG) : gpr.R(rt)); - if (invertResult) { - NOT(32, gpr.R(rd)); - } - } - gpr.UnlockAll(); - } - - void Jit::Comp_RType3(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU); - - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - // noop, won't write to ZERO. - if (rd == MIPS_REG_ZERO) - return; - - switch (op & 63) - { - case 10: //if (R(rt) == 0) R(rd) = R(rs); break; //movz - if (rd == rs) - break; - gpr.Lock(rt, rs, rd); - if (!gpr.IsImm(rt)) - { - gpr.KillImmediate(rs, true, false); - // Need to load rd in case the condition fails. - gpr.MapReg(rd, true, true); - CMP(32, gpr.R(rt), Imm32(0)); - CMOVcc(32, gpr.RX(rd), gpr.R(rs), CC_E); - } - else if (gpr.GetImm(rt) == 0) - { - if (gpr.IsImm(rs)) - gpr.SetImm(rd, gpr.GetImm(rs)); - else if (rd != rs) - { - gpr.MapReg(rd, false, true); - MOV(32, gpr.R(rd), gpr.R(rs)); - } - } - gpr.UnlockAll(); - break; - - case 11: //if (R(rt) != 0) R(rd) = R(rs); break; //movn - if (rd == rs) - break; - gpr.Lock(rt, rs, rd); - if (!gpr.IsImm(rt)) - { - gpr.KillImmediate(rs, true, false); - // Need to load rd in case the condition fails. - gpr.MapReg(rd, true, true); - CMP(32, gpr.R(rt), Imm32(0)); - CMOVcc(32, gpr.RX(rd), gpr.R(rs), CC_NE); - } - else if (gpr.GetImm(rt) != 0) - { - if (gpr.IsImm(rs)) - gpr.SetImm(rd, gpr.GetImm(rs)); - else if (rd != rs) - { - gpr.MapReg(rd, false, true); - MOV(32, gpr.R(rd), gpr.R(rs)); - } - } - gpr.UnlockAll(); - break; - - case 32: //R(rd) = R(rs) + R(rt); break; //add - case 33: //R(rd) = R(rs) + R(rt); break; //addu - if (rd != rs && rd != rt && gpr.R(rs).IsSimpleReg() && gpr.R(rt).IsSimpleReg()) { - gpr.Lock(rt, rs, rd); - gpr.MapReg(rd, false, true); - LEA(32, gpr.RX(rd), MRegSum(gpr.RX(rs), gpr.RX(rt))); - gpr.UnlockAll(); - } else { - CompTriArith(op, &XEmitter::ADD, &RType3_ImmAdd); - } - break; - case 34: //R(rd) = R(rs) - R(rt); break; //sub - case 35: //R(rd) = R(rs) - R(rt); break; //subu - CompTriArith(op, &XEmitter::SUB, &RType3_ImmSub); - break; - case 36: //R(rd) = R(rs) & R(rt); break; //and - CompTriArith(op, &XEmitter::AND, &RType3_ImmAnd); - break; - case 37: //R(rd) = R(rs) | R(rt); break; //or - CompTriArith(op, &XEmitter::OR, &RType3_ImmOr); - break; - case 38: //R(rd) = R(rs) ^ R(rt); break; //xor - CompTriArith(op, &XEmitter::XOR, &RType3_ImmXor); - break; - - case 39: // R(rd) = ~(R(rs) | R(rt)); //nor - CompTriArith(op, &XEmitter::OR, &RType3_ImmOr, true); - break; - - case 42: //R(rd) = (int)R(rs) < (int)R(rt); break; //slt - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, (s32)gpr.GetImm(rs) < (s32)gpr.GetImm(rt)); - } else if (rs == rt) { - gpr.SetImm(rd, 0); - } else { - gpr.Lock(rd, rs, rt); - gpr.MapReg(rd, rd == rt || rd == rs, true); - - // Let's try to avoid loading rs or if it's an imm, flushing it. - MIPSGPReg lhs = rs; - MIPSGPReg rhs = rt; - CCFlags cc = CC_L; - if (gpr.IsImm(lhs)) { - // rhs is guaranteed not to be an imm (handled above.) - std::swap(lhs, rhs); - cc = SwapCCFlag(cc); - } else if (!gpr.R(lhs).CanDoOpWith(gpr.R(rhs))) { - // Let's try to pick which makes more sense to load. - if (MIPSAnalyst::IsRegisterUsed(rhs, GetCompilerPC() + 4, 3)) { - std::swap(lhs, rhs); - cc = SwapCCFlag(cc); - } - gpr.MapReg(lhs, true, false); - } - - bool needsTemp = !HasLowSubregister(gpr.R(rd)) || rd == rt || rd == rs; - if (needsTemp) { - CMP(32, gpr.R(lhs), gpr.R(rhs)); - SETcc(cc, R(TEMPREG)); - MOVZX(32, 8, gpr.RX(rd), R(TEMPREG)); - } else { - XOR(32, gpr.R(rd), gpr.R(rd)); - CMP(32, gpr.R(lhs), gpr.R(rhs)); - SETcc(cc, gpr.R(rd)); - } - gpr.UnlockAll(); - } - break; - - case 43: //R(rd) = R(rs) < R(rt); break; //sltu - if (gpr.IsImm(rs) && gpr.IsImm(rt)) { - gpr.SetImm(rd, gpr.GetImm(rs) < gpr.GetImm(rt)); - } else if (rs == rt) { - gpr.SetImm(rd, 0); - } else { - gpr.Lock(rd, rs, rt); - gpr.MapReg(rd, rd == rt || rd == rs, true); - - // Let's try to avoid loading rs or if it's an imm, flushing it. - MIPSGPReg lhs = rs; - MIPSGPReg rhs = rt; - CCFlags cc = CC_B; - if (gpr.IsImm(lhs)) { - // rhs is guaranteed not to be an imm (handled above.) - std::swap(lhs, rhs); - cc = SwapCCFlag(cc); - } else if (!gpr.R(lhs).CanDoOpWith(gpr.R(rhs))) { - // Let's try to pick which makes more sense to load. - if (MIPSAnalyst::IsRegisterUsed(rhs, GetCompilerPC() + 4, 3)) { - std::swap(lhs, rhs); - cc = SwapCCFlag(cc); - } - gpr.MapReg(lhs, true, false); - } - - bool needsTemp = !HasLowSubregister(gpr.R(rd)) || rd == rt || rd == rs; - if (needsTemp) { - CMP(32, gpr.R(lhs), gpr.R(rhs)); - SETcc(cc, R(TEMPREG)); - MOVZX(32, 8, gpr.RX(rd), R(TEMPREG)); - } else { - XOR(32, gpr.R(rd), gpr.R(rd)); - CMP(32, gpr.R(lhs), gpr.R(rhs)); - SETcc(cc, gpr.R(rd)); - } - gpr.UnlockAll(); - } - break; - - case 44: //R(rd) = (R(rs) > R(rt)) ? R(rs) : R(rt); break; //max - if (gpr.IsImm(rs) && gpr.IsImm(rt)) - gpr.SetImm(rd, std::max((s32)gpr.GetImm(rs), (s32)gpr.GetImm(rt))); - else - { - MIPSGPReg rsrc = rd == rt ? rs : rt; - gpr.Lock(rd, rs, rt); - gpr.KillImmediate(rsrc, true, false); - gpr.MapReg(rd, rd == rs || rd == rt, true); - if (rd != rt && rd != rs) - MOV(32, gpr.R(rd), gpr.R(rs)); - CMP(32, gpr.R(rd), gpr.R(rsrc)); - CMOVcc(32, gpr.RX(rd), gpr.R(rsrc), CC_L); - gpr.UnlockAll(); - } - break; - - case 45: //R(rd) = (R(rs) < R(rt)) ? R(rs) : R(rt); break; //min - if (gpr.IsImm(rs) && gpr.IsImm(rt)) - gpr.SetImm(rd, std::min((s32)gpr.GetImm(rs), (s32)gpr.GetImm(rt))); - else - { - MIPSGPReg rsrc = rd == rt ? rs : rt; - gpr.Lock(rd, rs, rt); - gpr.KillImmediate(rsrc, true, false); - gpr.MapReg(rd, rd == rs || rd == rt, true); - if (rd != rt && rd != rs) - MOV(32, gpr.R(rd), gpr.R(rs)); - CMP(32, gpr.R(rd), gpr.R(rsrc)); - CMOVcc(32, gpr.RX(rd), gpr.R(rsrc), CC_G); - gpr.UnlockAll(); - } - break; - - default: - Comp_Generic(op); - break; - } - } - - static u32 ShiftType_ImmLogicalLeft(const u32 a, const u32 b) - { - return a << (b & 0x1f); - } - - static u32 ShiftType_ImmLogicalRight(const u32 a, const u32 b) - { - return a >> (b & 0x1f); - } - - static u32 ShiftType_ImmArithRight(const u32 a, const u32 b) - { - return ((s32) a) >> (b & 0x1f); - } - - static u32 ShiftType_ImmRotateRight(const u32 a, const u32 b) - { - const s8 sa = b & 0x1f; - return (a >> sa) | (a << (32 - sa)); - } - - void Jit::CompShiftImm(MIPSOpcode op, void (XEmitter::*shift)(int, OpArg, OpArg), u32 (*doImm)(const u32, const u32)) - { - MIPSGPReg rd = _RD; - MIPSGPReg rt = _RT; - int sa = _SA; - - if (doImm && gpr.IsImm(rt)) - { - gpr.SetImm(rd, doImm(gpr.GetImm(rt), sa)); - return; - } - - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - (this->*shift)(32, gpr.R(rd), Imm8(sa)); - gpr.UnlockAll(); - } - - // "over-shifts" work the same as on x86 - only bottom 5 bits are used to get the shift value - void Jit::CompShiftVar(MIPSOpcode op, void (XEmitter::*shift)(int, OpArg, OpArg), u32 (*doImm)(const u32, const u32)) - { - MIPSGPReg rd = _RD; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - if (doImm && gpr.IsImm(rs) && gpr.IsImm(rt)) - { - gpr.SetImm(rd, doImm(gpr.GetImm(rt), gpr.GetImm(rs))); - return; - } - - gpr.Lock(rd, rt, rs); - if (gpr.IsImm(rs)) { - int sa = gpr.GetImm(rs); - gpr.MapReg(rd, rd == rt, true); - if (cpu_info.bBMI2 && shift == &XEmitter::ROR) { - _assert_(!gpr.IsImm(rt)); - RORX(32, gpr.RX(rd), gpr.R(rt), sa & 0x1F); - } else { - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - (this->*shift)(32, gpr.R(rd), Imm8(sa & 0x1F)); - } - } else if (cpu_info.bBMI2 && shift != &XEmitter::ROR) { - gpr.MapReg(rd, rd == rt || rd == rs, true); - gpr.MapReg(rs, true, false); - MIPSGPReg src = rt; - if (gpr.IsImm(rt) && rd == rs) { - gpr.MapReg(rt, true, false); - } else if (gpr.IsImm(rt)) { - MOV(32, gpr.R(rd), gpr.R(rt)); - src = rd; - } - if (shift == &XEmitter::SHL) - SHLX(32, gpr.RX(rd), gpr.R(src), gpr.RX(rs)); - else if (shift == &XEmitter::SHR) - SHRX(32, gpr.RX(rd), gpr.R(src), gpr.RX(rs)); - else if (shift == &XEmitter::SAR) - SARX(32, gpr.RX(rd), gpr.R(src), gpr.RX(rs)); - else - _assert_msg_(false, "Unexpected shift type"); - } else { - gpr.FlushLockX(ECX); - gpr.MapReg(rd, rd == rt || rd == rs, true); - MOV(32, R(ECX), gpr.R(rs)); // Only ECX can be used for variable shifts. - AND(32, R(ECX), Imm32(0x1f)); - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - (this->*shift)(32, gpr.R(rd), R(ECX)); - gpr.UnlockAllX(); - } - gpr.UnlockAll(); - } - - void Jit::Comp_ShiftType(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU); - int rs = (op>>21) & 0x1F; - MIPSGPReg rd = _RD; - int fd = (op>>6) & 0x1F; - - // noop, won't write to ZERO. - if (rd == MIPS_REG_ZERO) - return; - - // WARNING : ROTR - switch (op & 0x3f) - { - case 0: CompShiftImm(op, &XEmitter::SHL, &ShiftType_ImmLogicalLeft); break; - case 2: CompShiftImm(op, (rs & 1) ? &XEmitter::ROR : &XEmitter::SHR, (rs & 1) ? &ShiftType_ImmRotateRight : &ShiftType_ImmLogicalRight); break; // srl, rotr - case 3: CompShiftImm(op, &XEmitter::SAR, &ShiftType_ImmArithRight); break; // sra - - case 4: CompShiftVar(op, &XEmitter::SHL, &ShiftType_ImmLogicalLeft); break; //sllv - case 6: CompShiftVar(op, (fd & 1) ? &XEmitter::ROR : &XEmitter::SHR, (fd & 1) ? &ShiftType_ImmRotateRight : &ShiftType_ImmLogicalRight); break; //srlv - case 7: CompShiftVar(op, &XEmitter::SAR, &ShiftType_ImmArithRight); break; //srav - - default: - Comp_Generic(op); - break; - } - } - - void Jit::Comp_Special3(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rs = _RS; - MIPSGPReg rt = _RT; - - int pos = _POS; - int size = _SIZE + 1; - u32 mask = 0xFFFFFFFFUL >> (32 - size); - - // Don't change $zr. - if (rt == MIPS_REG_ZERO) - return; - - switch (op & 0x3f) - { - case 0x0: //ext - if (gpr.IsImm(rs)) - { - gpr.SetImm(rt, (gpr.GetImm(rs) >> pos) & mask); - return; - } - - gpr.Lock(rs, rt); - gpr.MapReg(rt, rs == rt, true); - if (rs != rt) - MOV(32, gpr.R(rt), gpr.R(rs)); - if (pos != 0) { - SHR(32, gpr.R(rt), Imm8(pos)); - } - // Might not need to AND if we used a wall anyway. - if ((0xFFFFFFFF >> pos) != mask) { - AND(32, gpr.R(rt), Imm32(mask)); - } - gpr.UnlockAll(); - break; - - case 0x4: //ins - { - u32 sourcemask = mask >> pos; - u32 destmask = ~(sourcemask << pos); - if (gpr.IsImm(rs)) - { - u32 inserted = (gpr.GetImm(rs) & sourcemask) << pos; - if (gpr.IsImm(rt)) - { - gpr.SetImm(rt, (gpr.GetImm(rt) & destmask) | inserted); - return; - } - - gpr.Lock(rs, rt); - gpr.MapReg(rt, true, true); - AND(32, gpr.R(rt), Imm32(destmask)); - if (inserted != 0) - OR(32, gpr.R(rt), Imm32(inserted)); - gpr.UnlockAll(); - } - else if (gpr.IsImm(rt)) - { - // This happens. We can skip the AND and a load. - gpr.Lock(rs, rt); - u32 rtImm = gpr.GetImm(rt) & destmask; - gpr.MapReg(rt, false, true); - MOV(32, gpr.R(rt), gpr.R(rs)); - AND(32, gpr.R(rt), Imm32(sourcemask)); - if (pos != 0) { - SHL(32, gpr.R(rt), Imm8(pos)); - } - OR(32, gpr.R(rt), Imm32(rtImm)); - gpr.UnlockAll(); - } - else - { - gpr.Lock(rs, rt); - gpr.MapReg(rt, true, true); - MOV(32, R(TEMPREG), gpr.R(rs)); - AND(32, R(TEMPREG), Imm32(sourcemask)); - if (pos != 0) { - SHL(32, R(TEMPREG), Imm8(pos)); - } - AND(32, gpr.R(rt), Imm32(destmask)); - OR(32, gpr.R(rt), R(TEMPREG)); - gpr.UnlockAll(); - } - } - break; - } - } - - - void Jit::Comp_Allegrex(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rt = _RT; - MIPSGPReg rd = _RD; - // Don't change $zr. - if (rd == MIPS_REG_ZERO) - return; - - switch (AllegrexBitOp(op)) - { - case 16: // seb // R(rd) = SignExtend8ToU32(R(rt)); - if (gpr.IsImm(rt)) - { - gpr.SetImm(rd, SignExtend8ToU32(gpr.GetImm(rt))); - break; - } - - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - // Work around the byte-register addressing problem. - if (gpr.R(rt).IsSimpleReg() && !HasLowSubregister(gpr.R(rt))) - { - MOV(32, R(TEMPREG), gpr.R(rt)); - MOVSX(32, 8, gpr.RX(rd), R(TEMPREG)); - } - else - { - gpr.KillImmediate(rt, true, false); - MOVSX(32, 8, gpr.RX(rd), gpr.R(rt)); - } - gpr.UnlockAll(); - break; - - case 20: //bitrev - if (gpr.IsImm(rt)) - { - // http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel - u32 v = gpr.GetImm(rt); - // swap odd and even bits - v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1); - // swap consecutive pairs - v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2); - // swap nibbles ... - v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4); - // swap bytes - v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8); - // swap 2-byte long pairs - v = ( v >> 16 ) | ( v << 16); - gpr.SetImm(rd, v); - break; - } - - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - - LEA(32, TEMPREG, MScaled(gpr.RX(rd), 2, 0)); - SHR(32, gpr.R(rd), Imm8(1)); - XOR(32, gpr.R(rd), R(TEMPREG)); - AND(32, gpr.R(rd), Imm32(0x55555555)); - XOR(32, gpr.R(rd), R(TEMPREG)); - - LEA(32, TEMPREG, MScaled(gpr.RX(rd), 4, 0)); - SHR(32, gpr.R(rd), Imm8(2)); - XOR(32, gpr.R(rd), R(TEMPREG)); - AND(32, gpr.R(rd), Imm32(0x33333333)); - XOR(32, gpr.R(rd), R(TEMPREG)); - - MOV(32, R(TEMPREG), gpr.R(rd)); - SHL(32, R(TEMPREG), Imm8(4)); - SHR(32, gpr.R(rd), Imm8(4)); - XOR(32, gpr.R(rd), R(TEMPREG)); - AND(32, gpr.R(rd), Imm32(0x0F0F0F0F)); - XOR(32, gpr.R(rd), R(TEMPREG)); - - MOV(32, R(TEMPREG), gpr.R(rd)); - SHL(32, R(TEMPREG), Imm8(8)); - SHR(32, gpr.R(rd), Imm8(8)); - XOR(32, gpr.R(rd), R(TEMPREG)); - AND(32, gpr.R(rd), Imm32(0x00FF00FF)); - XOR(32, gpr.R(rd), R(TEMPREG)); - - ROL(32, gpr.R(rd), Imm8(16)); - - gpr.UnlockAll(); - break; - - case 24: // seh // R(rd) = SignExtend16ToU32(R(rt)); - if (gpr.IsImm(rt)) - { - gpr.SetImm(rd, SignExtend16ToU32(gpr.GetImm(rt))); - break; - } - - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - MOVSX(32, 16, gpr.RX(rd), gpr.R(rt)); - gpr.UnlockAll(); - break; - - default: - Comp_Generic(op); - return; - } - } - - void Jit::Comp_Allegrex2(MIPSOpcode op) - { - CONDITIONAL_DISABLE(ALU_BIT); - MIPSGPReg rt = _RT; - MIPSGPReg rd = _RD; - // Don't change $zr. - if (rd == MIPS_REG_ZERO) - return; - - switch (AllegrexBitOp(op)) - { - case 2: //wsbh - if (gpr.IsImm(rt)) { - u32 rtImm = gpr.GetImm(rt); - gpr.SetImm(rd, ((rtImm & 0xFF00FF00) >> 8) | ((rtImm & 0x00FF00FF) << 8)); - break; - } - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - // Swap both 16-bit halfwords by rotating afterward. - BSWAP(32, gpr.RX(rd)); - ROR(32, gpr.R(rd), Imm8(16)); - gpr.UnlockAll(); - break; - case 3: //wsbw - if (gpr.IsImm(rt)) { - gpr.SetImm(rd, swap32(gpr.GetImm(rt))); - break; - } - gpr.Lock(rd, rt); - gpr.MapReg(rd, rd == rt, true); - if (rd != rt) - MOV(32, gpr.R(rd), gpr.R(rt)); - BSWAP(32, gpr.RX(rd)); - gpr.UnlockAll(); - break; - default: - Comp_Generic(op); - break; - } - } - - void Jit::Comp_MulDivType(MIPSOpcode op) - { - CONDITIONAL_DISABLE(MULDIV); - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - - switch (op & 63) - { - case 16: // R(rd) = HI; //mfhi - if (rd != MIPS_REG_ZERO) { - gpr.MapReg(rd, false, true); - MOV(32, gpr.R(rd), gpr.R(MIPS_REG_HI)); - } - break; - - case 17: // HI = R(rs); //mthi - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.MapReg(rs, true, false); - MOV(32, gpr.R(MIPS_REG_HI), gpr.R(rs)); - break; - - case 18: // R(rd) = LO; break; //mflo - if (rd != MIPS_REG_ZERO) { - gpr.MapReg(rd, false, true); - MOV(32, gpr.R(rd), gpr.R(MIPS_REG_LO)); - } - break; - - case 19: // LO = R(rs); break; //mtlo - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.MapReg(rs, true, false); - MOV(32, gpr.R(MIPS_REG_LO), gpr.R(rs)); - break; - - case 24: //mult (the most popular one). lo,hi = signed mul (rs * rt) - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - // Mul, this must be EAX! - MOV(32, R(EAX), gpr.R(rs)); - IMUL(32, gpr.R(rt)); - MOV(32, gpr.R(MIPS_REG_HI), R(EDX)); - MOV(32, gpr.R(MIPS_REG_LO), R(EAX)); - gpr.UnlockAllX(); - break; - - - case 25: //multu (2nd) lo,hi = unsigned mul (rs * rt) - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - MOV(32, R(EAX), gpr.R(rs)); - MUL(32, gpr.R(rt)); - MOV(32, gpr.R(MIPS_REG_HI), R(EDX)); - MOV(32, gpr.R(MIPS_REG_LO), R(EAX)); - gpr.UnlockAllX(); - break; - - case 26: //div - { - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - // For CMP. - gpr.KillImmediate(rs, true, false); - gpr.KillImmediate(rt, true, false); - - MOV(32, R(EAX), gpr.R(rs)); - - CMP(32, gpr.R(rt), Imm32(0)); - FixupBranch divZero = J_CC(CC_E); - - // INT_MAX / -1 would overflow. - CMP(32, gpr.R(rs), Imm32(0x80000000)); - FixupBranch notOverflow = J_CC(CC_NE); - CMP(32, gpr.R(rt), Imm32((u32) -1)); - FixupBranch notOverflow2 = J_CC(CC_NE); - MOV(32, gpr.R(MIPS_REG_LO), Imm32(0x80000000)); - MOV(32, gpr.R(MIPS_REG_HI), Imm32(0)); // Hardware leaves the remainder at zero. - FixupBranch skip2 = J(); - - SetJumpTarget(notOverflow); - SetJumpTarget(notOverflow2); - - CDQ(); - IDIV(32, gpr.R(rt)); - MOV(32, gpr.R(MIPS_REG_HI), R(EDX)); - MOV(32, gpr.R(MIPS_REG_LO), R(EAX)); - FixupBranch skip = J(); - - SetJumpTarget(divZero); - MOV(32, gpr.R(MIPS_REG_HI), R(EAX)); - MOV(32, gpr.R(MIPS_REG_LO), Imm32(-1)); - CMP(32, R(EAX), Imm32(0)); - FixupBranch positiveDivZero = J_CC(CC_GE); - MOV(32, gpr.R(MIPS_REG_LO), Imm32(1)); - - SetJumpTarget(positiveDivZero); - SetJumpTarget(skip); - SetJumpTarget(skip2); - gpr.UnlockAllX(); - } - break; - - case 27: //divu - { - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - - MOV(32, R(EAX), gpr.R(rs)); - MOV(32, R(EDX), Imm32(0)); - - CMP(32, gpr.R(rt), Imm32(0)); - FixupBranch divZero = J_CC(CC_E); - - DIV(32, gpr.R(rt)); - MOV(32, gpr.R(MIPS_REG_HI), R(EDX)); - MOV(32, gpr.R(MIPS_REG_LO), R(EAX)); - FixupBranch skip = J(); - - SetJumpTarget(divZero); - MOV(32, gpr.R(MIPS_REG_HI), R(EAX)); - MOV(32, gpr.R(MIPS_REG_LO), Imm32(-1)); - CMP(32, R(EAX), Imm32(0xFFFF)); - FixupBranch moreThan16Bit = J_CC(CC_A); - MOV(32, gpr.R(MIPS_REG_LO), Imm32(0xFFFF)); - - SetJumpTarget(moreThan16Bit); - SetJumpTarget(skip); - gpr.UnlockAllX(); - } - break; - - case 28: // madd - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - MOV(32, R(EAX), gpr.R(rs)); - IMUL(32, gpr.R(rt)); - ADD(32, gpr.R(MIPS_REG_LO), R(EAX)); - ADC(32, gpr.R(MIPS_REG_HI), R(EDX)); - gpr.UnlockAllX(); - break; - - case 29: // maddu - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - MOV(32, R(EAX), gpr.R(rs)); - MUL(32, gpr.R(rt)); - ADD(32, gpr.R(MIPS_REG_LO), R(EAX)); - ADC(32, gpr.R(MIPS_REG_HI), R(EDX)); - gpr.UnlockAllX(); - break; - - case 46: // msub - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - MOV(32, R(EAX), gpr.R(rs)); - IMUL(32, gpr.R(rt)); - SUB(32, gpr.R(MIPS_REG_LO), R(EAX)); - SBB(32, gpr.R(MIPS_REG_HI), R(EDX)); - gpr.UnlockAllX(); - break; - - case 47: // msubu - gpr.FlushLockX(EDX); - gpr.KillImmediate(MIPS_REG_HI, false, true); - gpr.KillImmediate(MIPS_REG_LO, false, true); - gpr.KillImmediate(rt, true, false); - MOV(32, R(EAX), gpr.R(rs)); - MUL(32, gpr.R(rt)); - SUB(32, gpr.R(MIPS_REG_LO), R(EAX)); - SBB(32, gpr.R(MIPS_REG_HI), R(EDX)); - gpr.UnlockAllX(); - break; - - default: - DISABLE; - } - } -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompBranch.cpp b/Core/MIPS/x86/CompBranch.cpp deleted file mode 100644 index dff306a95c..0000000000 --- a/Core/MIPS/x86/CompBranch.cpp +++ /dev/null @@ -1,711 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Common/Data/Convert/SmallDataConvert.h" -#include "Common/Profiler/Profiler.h" - -#include "Core/Config.h" -#include "Core/Core.h" -#include "Core/Reporting.h" -#include "Core/HLE/HLE.h" -#include "Core/HLE/HLETables.h" -#include "Core/MemMap.h" - -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSTables.h" - -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" -#include "Core/MIPS/JitCommon/JitBlockCache.h" - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM26 (op & 0x03FFFFFF) -#define TARGET16 ((int)(SignExtend16ToU32(op) << 2)) -#define TARGET26 (_IMM26 << 2) - -#define LOOPOPTIMIZATION 0 - -using namespace MIPSAnalyst; - -// NOTE: Can't use CONDITIONAL_DISABLE in this file, branches are so special -// that they cannot be interpreted in the context of the Jit. - -// But we can at least log and compare. -// #define DO_CONDITIONAL_LOG 1 -#define DO_CONDITIONAL_LOG 0 - -// We can also disable nice delay slots. -// #define CONDITIONAL_NICE_DELAYSLOT branchInfo.delaySlotIsNice = false; -#define CONDITIONAL_NICE_DELAYSLOT ; - -#if DO_CONDITIONAL_LOG -#define CONDITIONAL_LOG BranchLog(op); -#define CONDITIONAL_LOG_EXIT(addr) BranchLogExit(op, addr, false); -#define CONDITIONAL_LOG_EXIT_EAX() BranchLogExit(op, 0, true); -#else -#define CONDITIONAL_LOG ; -#define CONDITIONAL_LOG_EXIT(addr) ; -#define CONDITIONAL_LOG_EXIT_EAX() ; -#endif - -namespace MIPSComp -{ -using namespace Gen; - -static void JitBranchLog(MIPSOpcode op, u32 pc) { - currentMIPS->pc = pc; - currentMIPS->inDelaySlot = false; - - MIPSInterpretFunc func = MIPSGetInterpretFunc(op); - MIPSInfo info = MIPSGetInfo(op); - func(currentMIPS, op); - - // Branch taken, use nextPC. - if (currentMIPS->inDelaySlot) - currentMIPS->intBranchExit = currentMIPS->nextPC; - else - { - // Branch not taken, likely delay slot skipped. - if (info & LIKELY) - currentMIPS->intBranchExit = currentMIPS->pc; - // Branch not taken, so increment over delay slot. - else - currentMIPS->intBranchExit = currentMIPS->pc + 4; - } - - currentMIPS->pc = pc; - currentMIPS->inDelaySlot = false; -} - -static void JitBranchLogMismatch(MIPSOpcode op, u32 pc) -{ - char temp[256]; - MIPSDisAsm(op, pc, temp, sizeof(temp), true); - ERROR_LOG(Log::JIT, "Bad jump: %s - int:%08x jit:%08x", temp, currentMIPS->intBranchExit, currentMIPS->jitBranchExit); - Core_Break(BreakReason::JitBranchDebug, pc); -} - -void Jit::BranchLog(MIPSOpcode op) -{ - FlushAll(); - ABI_CallFunctionCC(thunks.ProtectFunction(&JitBranchLog), op.encoding, GetCompilerPC()); -} - -void Jit::BranchLogExit(MIPSOpcode op, u32 dest, bool useEAX) -{ - OpArg destArg = useEAX ? R(EAX) : Imm32(dest); - - CMP(32, MIPSSTATE_VAR(intBranchExit), destArg); - FixupBranch skip = J_CC(CC_E); - - MOV(32, MIPSSTATE_VAR(jitBranchExit), destArg); - ABI_CallFunctionCC(thunks.ProtectFunction(&JitBranchLogMismatch), op.encoding, GetCompilerPC()); - // Restore EAX, we probably ruined it. - if (useEAX) - MOV(32, R(EAX), MIPSSTATE_VAR(jitBranchExit)); - - SetJumpTarget(skip); -} - -CCFlags Jit::FlipCCFlag(CCFlags flag) -{ - switch (flag) - { - case CC_O: return CC_NO; - case CC_NO: return CC_O; - case CC_B: return CC_NB; - case CC_NB: return CC_B; - case CC_Z: return CC_NZ; - case CC_NZ: return CC_Z; - case CC_BE: return CC_NBE; - case CC_NBE: return CC_BE; - case CC_S: return CC_NS; - case CC_NS: return CC_S; - case CC_P: return CC_NP; - case CC_NP: return CC_P; - case CC_L: return CC_NL; - case CC_NL: return CC_L; - case CC_LE: return CC_NLE; - case CC_NLE: return CC_LE; - } - ERROR_LOG_REPORT(Log::JIT, "FlipCCFlag: Unexpected CC flag: %d", flag); - return CC_O; -} - -CCFlags Jit::SwapCCFlag(CCFlags flag) -{ - // This swaps the comparison for an lhs/rhs swap, but doesn't flip/invert the logic. - switch (flag) - { - case CC_O: return CC_O; - case CC_NO: return CC_NO; - case CC_B: return CC_A; - case CC_NB: return CC_NA; - case CC_Z: return CC_Z; - case CC_NZ: return CC_NZ; - case CC_BE: return CC_AE; - case CC_NBE: return CC_NAE; - case CC_S: return CC_S; - case CC_NS: return CC_NS; - case CC_P: return CC_P; - case CC_NP: return CC_NP; - case CC_L: return CC_G; - case CC_NL: return CC_NG; - case CC_LE: return CC_GE; - case CC_NLE: return CC_NGE; - } - ERROR_LOG_REPORT(Log::JIT, "SwapCCFlag: Unexpected CC flag: %d", flag); - return CC_O; -} - -bool Jit::PredictTakeBranch(u32 targetAddr, bool likely) { - // If it's likely, it's... probably likely, right? - if (likely) - return true; - - // TODO: Normal branch prediction would be to take branches going upward to lower addresses. - // However, this results in worse performance as of this comment's writing. - // The reverse check generally gives better or same performance. - return targetAddr > GetCompilerPC(); -} - -void Jit::CompBranchExits(CCFlags cc, u32 targetAddr, u32 notTakenAddr, const BranchInfo &branchInfo) { - if (branchInfo.andLink) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - - // We may want to try to continue along this branch a little while, to reduce reg flushing. - bool predictTakeBranch = PredictTakeBranch(targetAddr, branchInfo.likely); - - Gen::FixupBranch ptr; - if (!branchInfo.likely) { - if (!branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_SAFE_FLUSH); - else - FlushAll(); - ptr = J_CC(cc, true); - } else { - FlushAll(); - ptr = J_CC(cc, true); - if (!branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_FLUSH); - } - - // Handle the linkage of a delay slot, even when we're taking the branch. - if (branchInfo.delaySlotIsBranch) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - FlushAll(); - } - - // Take the branch - CONDITIONAL_LOG_EXIT(targetAddr); - WriteExit(targetAddr, js.nextExit++); - - // Not taken - SetJumpTarget(ptr); - CONDITIONAL_LOG_EXIT(notTakenAddr); - WriteExit(notTakenAddr, js.nextExit++); - js.compiling = false; -} - -void Jit::CompBranchExit(bool taken, u32 targetAddr, u32 notTakenAddr, const BranchInfo &branchInfo) { - // Continuing is handled in the imm branch case... TODO: move it here? - if (branchInfo.andLink) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - if (branchInfo.delaySlotIsBranch) { - if (taken) { - // We still link when the branch is taken (targetAddr case.) - // Remember, it's from the perspective of the delay slot, so +12. - if ((branchInfo.delaySlotInfo & OUT_RA) != 0) - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 12); - if ((branchInfo.delaySlotInfo & OUT_RD) != 0) - gpr.SetImm(MIPS_GET_RD(branchInfo.delaySlotOp), GetCompilerPC() + 12); - } - FlushAll(); - } else if (taken || !branchInfo.likely) { - CompileDelaySlot(DELAYSLOT_FLUSH); - } else { - FlushAll(); - } - - const u32 destAddr = taken ? targetAddr : notTakenAddr; - CONDITIONAL_LOG_EXIT(destAddr); - WriteExit(destAddr, js.nextExit++); - js.compiling = false; -} - -void Jit::BranchRSRTComp(MIPSOpcode op, Gen::CCFlags cc, bool likely) -{ - CONDITIONAL_LOG; - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in RSRTComp delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceReg(op, branchInfo.delaySlotOp, rt, rs); - CONDITIONAL_NICE_DELAYSLOT; - - bool immBranch = false; - bool immBranchTaken = false; - if (gpr.IsImm(rs) && gpr.IsImm(rt) && !branchInfo.delaySlotIsBranch) { - // The cc flags are opposites: when NOT to take the branch. - bool immBranchNotTaken; - s32 rsImm = (s32)gpr.GetImm(rs); - s32 rtImm = (s32)gpr.GetImm(rt); - - switch (cc) - { - case CC_E: immBranchNotTaken = rsImm == rtImm; break; - case CC_NE: immBranchNotTaken = rsImm != rtImm; break; - default: immBranchNotTaken = false; _dbg_assert_msg_(false, "Bad cc flag in BranchRSRTComp()."); - } - immBranch = true; - immBranchTaken = !immBranchNotTaken; - } - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - if (immBranch) - CompBranchExit(immBranchTaken, targetAddr, notTakenTarget, branchInfo); - else - { - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0) - { - gpr.KillImmediate(rs, true, false); - CMP(32, gpr.R(rs), Imm32(0)); - } - else - { - gpr.MapReg(rs, true, false); - CMP(32, gpr.R(rs), gpr.R(rt)); - } - - CompBranchExits(cc, targetAddr, notTakenTarget, branchInfo); - } -} - -void Jit::BranchRSZeroComp(MIPSOpcode op, Gen::CCFlags cc, bool andLink, bool likely) -{ - CONDITIONAL_LOG; - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in RSZeroComp delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - MIPSGPReg rs = _RS; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), andLink, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceReg(op, branchInfo.delaySlotOp, rs); - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - CONDITIONAL_NICE_DELAYSLOT; - - bool immBranch = false; - bool immBranchTaken = false; - if (gpr.IsImm(rs) && !branchInfo.delaySlotIsBranch) { - // The cc flags are opposites: when NOT to take the branch. - bool immBranchNotTaken; - s32 imm = (s32)gpr.GetImm(rs); - - switch (cc) - { - case CC_G: immBranchNotTaken = imm > 0; break; - case CC_GE: immBranchNotTaken = imm >= 0; break; - case CC_L: immBranchNotTaken = imm < 0; break; - case CC_LE: immBranchNotTaken = imm <= 0; break; - default: immBranchNotTaken = false; _dbg_assert_msg_(false, "Bad cc flag in BranchRSZeroComp()."); - } - immBranch = true; - immBranchTaken = !immBranchNotTaken; - } - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - if (immBranch) - CompBranchExit(immBranchTaken, targetAddr, notTakenTarget, branchInfo); - else - { - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - gpr.MapReg(rs, true, false); - CMP(32, gpr.R(rs), Imm32(0)); - - CompBranchExits(cc, targetAddr, notTakenTarget, branchInfo); - } -} - - -void Jit::Comp_RelBranch(MIPSOpcode op) -{ - switch (op>>26) - { - case 4: BranchRSRTComp(op, CC_NZ, false); break;//beq - case 5: BranchRSRTComp(op, CC_Z, false); break;//bne - - case 6: BranchRSZeroComp(op, CC_G, false, false); break;//blez - case 7: BranchRSZeroComp(op, CC_LE, false, false); break;//bgtz - - case 20: BranchRSRTComp(op, CC_NZ, true); break;//beql - case 21: BranchRSRTComp(op, CC_Z, true); break;//bnel - - case 22: BranchRSZeroComp(op, CC_G, false, true); break;//blezl - case 23: BranchRSZeroComp(op, CC_LE, false, true); break;//bgtzl - - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } -} - -void Jit::Comp_RelBranchRI(MIPSOpcode op) -{ - switch ((op >> 16) & 0x1F) - { - case 0: BranchRSZeroComp(op, CC_GE, false, false); break; //if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 4; break;//bltz - case 1: BranchRSZeroComp(op, CC_L, false, false); break; //if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 4; break;//bgez - case 2: BranchRSZeroComp(op, CC_GE, false, true); break; //if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 8; break;//bltzl - case 3: BranchRSZeroComp(op, CC_L, false, true); break; //if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 8; break;//bgezl - case 16: BranchRSZeroComp(op, CC_GE, true, false); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) < 0) DelayBranchTo(addr); else PC += 4; break;//bltzal - case 17: BranchRSZeroComp(op, CC_L, true, false); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) >= 0) DelayBranchTo(addr); else PC += 4; break;//bgezal - case 18: BranchRSZeroComp(op, CC_GE, true, true); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) < 0) DelayBranchTo(addr); else SkipLikely(); break;//bltzall - case 19: BranchRSZeroComp(op, CC_L, true, true); break; //R(MIPS_REG_RA) = PC + 8; if ((s32)R(rs) >= 0) DelayBranchTo(addr); else SkipLikely(); break;//bgezall - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } -} - - -// If likely is set, discard the branch slot if NOT taken. -void Jit::BranchFPFlag(MIPSOpcode op, Gen::CCFlags cc, bool likely) -{ - CONDITIONAL_LOG; - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in FPFlag delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - branchInfo.delaySlotIsNice = IsDelaySlotNiceFPU(op, branchInfo.delaySlotOp); - CONDITIONAL_NICE_DELAYSLOT; - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - gpr.KillImmediate(MIPS_REG_FPCOND, true, false); - TEST(32, gpr.R(MIPS_REG_FPCOND), Imm32(1)); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - CompBranchExits(cc, targetAddr, notTakenTarget, branchInfo); -} - - -void Jit::Comp_FPUBranch(MIPSOpcode op) -{ - switch((op >> 16) & 0x1f) - { - case 0: BranchFPFlag(op, CC_NZ, false); break; //bc1f - case 1: BranchFPFlag(op, CC_Z, false); break; //bc1t - case 2: BranchFPFlag(op, CC_NZ, true); break; //bc1fl - case 3: BranchFPFlag(op, CC_Z, true); break; //bc1tl - default: - _dbg_assert_msg_(false,"Trying to interpret instruction that can't be interpreted"); - break; - } -} - -// If likely is set, discard the branch slot if NOT taken. -void Jit::BranchVFPUFlag(MIPSOpcode op, Gen::CCFlags cc, bool likely) -{ - CONDITIONAL_LOG; - if (js.inDelaySlot) { - // I think we can safely just warn-log this without reporting, it's pretty clear that this type - // of branch is ignored. - WARN_LOG(Log::JIT, "Branch in VFPU delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - int offset = TARGET16; - u32 targetAddr = GetCompilerPC() + offset + 4; - - BranchInfo branchInfo(GetCompilerPC(), op, GetOffsetInstruction(1), false, likely); - // Sometimes there's a VFPU branch in a delay slot (Disgaea 2: Dark Hero Days, Zettai Hero Project, La Pucelle) - // The behavior is undefined - the CPU may take the second branch even if the first one passes. - // However, it does consistently try each branch, which these games seem to expect. - branchInfo.delaySlotIsNice = IsDelaySlotNiceVFPU(op, branchInfo.delaySlotOp); - CONDITIONAL_NICE_DELAYSLOT; - - js.downcountAmount += MIPSGetInstructionCycleEstimate(branchInfo.delaySlotOp); - if (!likely && branchInfo.delaySlotIsNice && !branchInfo.delaySlotIsBranch) - CompileDelaySlot(DELAYSLOT_NICE); - - // THE CONDITION - int imm3 = (op >> 18) & 7; - - gpr.KillImmediate(MIPS_REG_VFPUCC, true, false); - TEST(32, gpr.R(MIPS_REG_VFPUCC), Imm32(1 << imm3)); - - u32 notTakenTarget = ResolveNotTakenTarget(branchInfo); - CompBranchExits(cc, targetAddr, notTakenTarget, branchInfo); -} - - -void Jit::Comp_VBranch(MIPSOpcode op) -{ - switch ((op >> 16) & 3) - { - case 0: BranchVFPUFlag(op, CC_NZ, false); break; //bvf - case 1: BranchVFPUFlag(op, CC_Z, false); break; //bvt - case 2: BranchVFPUFlag(op, CC_NZ, true); break; //bvfl - case 3: BranchVFPUFlag(op, CC_Z, true); break; //bvtl - default: - _dbg_assert_msg_(false,"Comp_VBranch: Invalid instruction"); - break; - } -} - -static void HitInvalidJump(uint32_t dest) { - Core_ExecException(dest, currentMIPS->pc - 8, ExecExceptionType::JUMP); -} - -void Jit::Comp_Jump(MIPSOpcode op) { - CONDITIONAL_LOG; - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in Jump delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - u32 off = TARGET26; - u32 targetAddr = (GetCompilerPC() & 0xF0000000) | off; - - // Might be a stubbed address or something? - if (!Memory::IsValidAddress(targetAddr) || (targetAddr & 3) != 0) { - if (js.nextExit == 0) { - ERROR_LOG(Log::JIT, "Jump to invalid address: %08x PC %08x RA %08x", targetAddr, GetCompilerPC(), currentMIPS->r[MIPS_REG_RA]); - } else { - js.compiling = false; - } - // TODO: Mark this block dirty or something? May be indication it will be changed by imports. - - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC() + 8)); - ABI_CallFunctionC(&HitInvalidJump, targetAddr); - WriteSyscallExit(); - return; - } - - switch (op >> 26) { - case 2: //j - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - CONDITIONAL_LOG_EXIT(targetAddr); - WriteExit(targetAddr, js.nextExit++); - break; - - case 3: //jal - // Save return address - might be overwritten by delay slot. - gpr.SetImm(MIPS_REG_RA, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - FlushAll(); - CONDITIONAL_LOG_EXIT(targetAddr); - WriteExit(targetAddr, js.nextExit++); - break; - - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } - js.compiling = false; -} - -void Jit::Comp_JumpReg(MIPSOpcode op) { - CONDITIONAL_LOG; - if (js.inDelaySlot) { - ERROR_LOG_REPORT(Log::JIT, "Branch in JumpReg delay slot at %08x in block starting at %08x", GetCompilerPC(), js.blockStart); - return; - } - MIPSGPReg rs = _RS; - MIPSGPReg rd = _RD; - bool andLink = (op & 0x3f) == 9 && rd != MIPS_REG_ZERO; - - MIPSOpcode delaySlotOp = GetOffsetInstruction(1); - js.downcountAmount += MIPSGetInstructionCycleEstimate(delaySlotOp); - bool delaySlotIsNice = IsDelaySlotNiceReg(op, delaySlotOp, rs); - if (andLink && rs == rd) - delaySlotIsNice = false; - CONDITIONAL_NICE_DELAYSLOT; - - X64Reg destReg = EAX; - if (IsSyscall(delaySlotOp)) - { - // If this is a syscall, write the pc (for thread switching and other good reasons.) - gpr.MapReg(rs, true, false); - MOV(32, MIPSSTATE_VAR(pc), gpr.R(rs)); - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_FLUSH); - - // Syscalls write the exit code for us. - _dbg_assert_msg_(!js.compiling, "Expected syscall to write an exit code."); - return; - } - else if (delaySlotIsNice) - { - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - - if (!andLink && rs == MIPS_REG_RA && g_Config.bDiscardRegsOnJRRA) { - // According to the MIPS ABI, there are some regs we don't need to preserve. - // Let's discard them so we don't need to write them back. - // NOTE: Not all games follow the MIPS ABI! Tekken 6, for example, will crash - // with this enabled. - gpr.DiscardRegContentsIfCached(MIPS_REG_COMPILER_SCRATCH); - for (int i = MIPS_REG_A0; i <= MIPS_REG_T7; i++) - gpr.DiscardRegContentsIfCached((MIPSGPReg)i); - gpr.DiscardRegContentsIfCached(MIPS_REG_T8); - gpr.DiscardRegContentsIfCached(MIPS_REG_T9); - } - - if (gpr.R(rs).IsSimpleReg()) { - destReg = gpr.R(rs).GetSimpleReg(); - } else { - MOV(32, R(EAX), gpr.R(rs)); - } - FlushAll(); - } else { - // Latch destination now - save it in memory. - gpr.MapReg(rs, true, false); - MOV(32, MIPSSTATE_VAR(savedPC), gpr.R(rs)); - if (andLink) - gpr.SetImm(rd, GetCompilerPC() + 8); - CompileDelaySlot(DELAYSLOT_NICE); - MOV(32, R(EAX), MIPSSTATE_VAR(savedPC)); - FlushAll(); - } - - switch (op & 0x3f) { - case 8: //jr - break; - case 9: //jalr - break; - default: - _dbg_assert_msg_(false,"Trying to compile instruction that can't be compiled"); - break; - } - - CONDITIONAL_LOG_EXIT_EAX(); - WriteExitDestInReg(destReg); - js.compiling = false; -} - -void Jit::Comp_Syscall(MIPSOpcode op) -{ - if (op.encoding == 0x03FFFFcc) { - WARN_LOG(Log::JIT, "Encountered bad syscall instruction at %08x (%08x)", js.compilerPC, op.encoding); - } - - if (!g_Config.bSkipDeadbeefFilling) - { - // All of these will be overwritten with DEADBEEF anyway. - gpr.DiscardR(MIPS_REG_COMPILER_SCRATCH); - // We need to keep A0 - T3, which are used for args. - gpr.DiscardR(MIPS_REG_T4); - gpr.DiscardR(MIPS_REG_T5); - gpr.DiscardR(MIPS_REG_T6); - gpr.DiscardR(MIPS_REG_T7); - gpr.DiscardR(MIPS_REG_T8); - gpr.DiscardR(MIPS_REG_T9); - - gpr.DiscardR(MIPS_REG_HI); - gpr.DiscardR(MIPS_REG_LO); - } - FlushAll(); - - // If we're in a delay slot, this is off by one. - const int offset = js.inDelaySlot ? -1 : 0; - WriteDowncount(offset); - RestoreRoundingMode(); - js.downcountAmount = -offset; - - if (!js.inDelaySlot) { - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC() + 4)); - } - -#ifdef USE_PROFILER - // When profiling, we can't skip CallSyscall, since it times syscalls. - ABI_CallFunctionC(&CallSyscall, op.encoding); -#else - // Skip the CallSyscall where possible. - const HLEFunction *func = GetSyscallFunctionData(op, js.compilerPC); - if (func) { - void *quickFunc = GetQuickSyscallFunc(func, op); - if (quickFunc) { - ABI_CallFunctionP(quickFunc, (void *)func); - } else { - ABI_CallFunctionC(&CallSyscall, op.encoding); - } - } else { - ABI_CallFunctionC(&CallSyscallUnresolvedAtPC, js.compilerPC); - } -#endif - - ApplyRoundingMode(); - WriteSyscallExit(); - js.compiling = false; -} - -void Jit::Comp_Break(MIPSOpcode op) -{ - Comp_Generic(op); - WriteSyscallExit(); - js.compiling = false; -} - -} // namespace Mipscomp - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompFPU.cpp b/Core/MIPS/x86/CompFPU.cpp deleted file mode 100644 index ecdb21225b..0000000000 --- a/Core/MIPS/x86/CompFPU.cpp +++ /dev/null @@ -1,501 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Core/Config.h" -#include "Core/MemMap.h" -#include "Common/CommonTypes.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE(flag) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } - -namespace MIPSComp { - -using namespace Gen; -using namespace X64JitConstants; - -alignas(16) const u32 reverseQNAN[4] = { 0x803FFFFF, 0x803FFFFF, 0x803FFFFF, 0x803FFFFF }; - -void Jit::CopyFPReg(X64Reg dst, OpArg src) { - if (src.IsSimpleReg()) { - MOVAPS(dst, src); - } else { - MOVSS(dst, src); - } -} - -void Jit::CompFPTriArith(MIPSOpcode op, void (XEmitter::*arith)(X64Reg reg, OpArg), bool orderMatters) { - int ft = _FT; - int fs = _FS; - int fd = _FD; - fpr.SpillLock(fd, fs, ft); - - if (fs == fd) { - fpr.MapReg(fd, true, true); - (this->*arith)(fpr.RX(fd), fpr.R(ft)); - } else if (ft == fd && !orderMatters) { - fpr.MapReg(fd, true, true); - (this->*arith)(fpr.RX(fd), fpr.R(fs)); - } else if (ft != fd) { - // fs can't be fd (handled above.) - fpr.MapReg(fd, false, true); - CopyFPReg(fpr.RX(fd), fpr.R(fs)); - (this->*arith)(fpr.RX(fd), fpr.R(ft)); - } else { - // fd must be ft, and order must matter. - fpr.MapReg(fd, true, true); - CopyFPReg(XMM0, fpr.R(fs)); - (this->*arith)(XMM0, fpr.R(ft)); - MOVAPS(fpr.RX(fd), R(XMM0)); - } - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_FPU3op(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU); - switch (op & 0x3f) { - case 0: CompFPTriArith(op, &XEmitter::ADDSS, false); break; //F(fd) = F(fs) + F(ft); //add - case 1: CompFPTriArith(op, &XEmitter::SUBSS, true); break; //F(fd) = F(fs) - F(ft); //sub - case 2: //F(fd) = F(fs) * F(ft); //mul - // XMM1 = !my_isnan(fs) && !my_isnan(ft) - MOVSS(XMM1, fpr.R(_FS)); - CMPORDSS(XMM1, fpr.R(_FT)); - CompFPTriArith(op, &XEmitter::MULSS, false); - - // fd must still be in a reg, save it in XMM0 for now. - MOVAPS(XMM0, fpr.R(_FD)); - // fd = my_isnan(fd) && !my_isnan(fs) && !my_isnan(ft) - CMPUNORDSS(fpr.RX(_FD), fpr.R(_FD)); - ANDPS(fpr.RX(_FD), R(XMM1)); - // At this point fd = FFFFFFFF if non-NAN inputs produced a NAN output. - // We'll AND it with the inverse QNAN bits to clear (00000000 means no change.) - if (RipAccessible(&reverseQNAN)) { - ANDPS(fpr.RX(_FD), M(&reverseQNAN)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&reverseQNAN)); - ANDPS(fpr.RX(_FD), MatR(TEMPREG)); - } - // ANDN is backwards, which is why we saved XMM0 to start. Now put it back. - ANDNPS(fpr.RX(_FD), R(XMM0)); - break; - case 3: CompFPTriArith(op, &XEmitter::DIVSS, true); break; //F(fd) = F(fs) / F(ft); //div - default: - _dbg_assert_msg_(false,"Trying to compile FPU3Op instruction that can't be interpreted"); - break; - } -} - -void Jit::Comp_FPULS(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU_FPU); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - s32 offset = _IMM16; - int ft = _FT; - MIPSGPReg rs = _RS; - - CheckMemoryBreakpoint(0, rs, offset); - - switch (op >> 26) { - case 49: // lwc1 - { - gpr.Lock(rs); - fpr.SpillLock(ft); - fpr.MapReg(ft, false, true); - - JitSafeMem safe(this, rs, offset); - OpArg src; - if (safe.PrepareRead(src, 4)) - MOVSS(fpr.RX(ft), src); - if (safe.PrepareSlowRead(safeMemFuncs.readU32)) - MOVD_xmm(fpr.RX(ft), R(EAX)); - safe.Finish(); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - case 57: // swc1 - { - gpr.Lock(rs); - fpr.SpillLock(ft); - fpr.MapReg(ft, true, false); - - JitSafeMem safe(this, rs, offset); - OpArg dest; - if (safe.PrepareWrite(dest, 4)) - MOVSS(dest, fpr.RX(ft)); - if (safe.PrepareSlowWrite()) - { - MOVSS(MIPSSTATE_VAR(temp), fpr.RX(ft)); - safe.DoSlowWrite(safeMemFuncs.writeU32, MIPSSTATE_VAR(temp)); - } - safe.Finish(); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - - default: - _dbg_assert_msg_(false,"Trying to interpret FPULS instruction that can't be interpreted"); - break; - } -} - -alignas(16) static const u64 ssSignBits2[2] = {0x8000000080000000ULL, 0x8000000080000000ULL}; -alignas(16) static const u64 ssNoSignMask[2] = {0x7FFFFFFF7FFFFFFFULL, 0x7FFFFFFF7FFFFFFFULL}; - -void Jit::CompFPComp(int lhs, int rhs, u8 compare, bool allowNaN) { - gpr.MapReg(MIPS_REG_FPCOND, false, true); - - // This means that NaN also means true, e.g. !<> or !>, etc. - if (allowNaN) { - CopyFPReg(XMM0, fpr.R(lhs)); - CopyFPReg(XMM1, fpr.R(lhs)); - CMPSS(XMM0, fpr.R(rhs), compare); - CMPUNORDSS(XMM1, fpr.R(rhs)); - - POR(XMM0, R(XMM1)); - } else { - CopyFPReg(XMM0, fpr.R(lhs)); - CMPSS(XMM0, fpr.R(rhs), compare); - } - - MOVD_xmm(gpr.R(MIPS_REG_FPCOND), XMM0); - // 1 rather than the compare's all-ones mask, as on the other cores. - AND(32, gpr.R(MIPS_REG_FPCOND), Imm32(1)); -} - -void Jit::Comp_FPUComp(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU_COMP); - - int fs = _FS; - int ft = _FT; - - switch (op & 0xf) { - case 0: //f - case 8: //sf - gpr.SetImm(MIPS_REG_FPCOND, 0); - break; - - case 1: //un - case 9: //ngle - CompFPComp(fs, ft, CMP_UNORD); - break; - - case 2: //eq - case 10: //seq - CompFPComp(fs, ft, CMP_EQ); - break; - - case 3: //ueq - case 11: //ngl - CompFPComp(fs, ft, CMP_EQ, true); - break; - - case 4: //olt - case 12: //lt - CompFPComp(fs, ft, CMP_LT); - break; - - case 5: //ult - case 13: //nge - CompFPComp(ft, fs, CMP_NLE); - break; - - case 6: //ole - case 14: //le - CompFPComp(fs, ft, CMP_LE); - break; - - case 7: //ule - case 15: //ngt - CompFPComp(ft, fs, CMP_NLT); - break; - - default: - DISABLE; - } -} - -void Jit::Comp_FPU2op(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU); - - int fs = _FS; - int fd = _FD; - - auto execRounding = [&](void (XEmitter::*conv)(X64Reg, OpArg), int setMXCSR) { - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fs == fd, true); - - // Small optimization: 0 is our default mode anyway. - if (setMXCSR == 0 && !js.hasSetRounding) { - setMXCSR = -1; - } - if (setMXCSR != -1) { - STMXCSR(MIPSSTATE_VAR(mxcsrTemp)); - MOV(32, R(TEMPREG), MIPSSTATE_VAR(mxcsrTemp)); - AND(32, R(TEMPREG), Imm32(~(3 << 13))); - OR(32, R(TEMPREG), Imm32(setMXCSR << 13)); - MOV(32, MIPSSTATE_VAR(temp), R(TEMPREG)); - LDMXCSR(MIPSSTATE_VAR(temp)); - } - - (this->*conv)(TEMPREG, fpr.R(fs)); - - // Did we get an indefinite integer value? - CMP(32, R(TEMPREG), Imm32(0x80000000)); - FixupBranch skip = J_CC(CC_NE); - if (fd != fs) { - CopyFPReg(fpr.RX(fd), fpr.R(fs)); - } - XORPS(XMM1, R(XMM1)); - CMPSS(fpr.RX(fd), R(XMM1), CMP_LT); - - // At this point, -inf = 0xffffffff, inf/nan = 0x00000000. - // We want -inf to be 0x80000000 inf/nan to be 0x7fffffff, so we flip those bits. - MOVD_xmm(R(TEMPREG), fpr.RX(fd)); - XOR(32, R(TEMPREG), Imm32(0x7fffffff)); - - SetJumpTarget(skip); - MOVD_xmm(fpr.RX(fd), R(TEMPREG)); - - if (setMXCSR != -1) { - LDMXCSR(MIPSSTATE_VAR(mxcsrTemp)); - } - }; - - switch (op & 0x3f) { - case 5: //F(fd) = fabsf(F(fs)); break; //abs - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fd == fs, true); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&ssNoSignMask[0])); - if (fd != fs && fpr.IsMapped(fs)) { - MOVAPS(fpr.RX(fd), MatR(TEMPREG)); - ANDPS(fpr.RX(fd), fpr.R(fs)); - } else { - if (fd != fs) { - MOVSS(fpr.RX(fd), fpr.R(fs)); - } - ANDPS(fpr.RX(fd), MatR(TEMPREG)); - } - break; - - case 6: //F(fd) = F(fs); break; //mov - if (fd != fs) { - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fd == fs, true); - CopyFPReg(fpr.RX(fd), fpr.R(fs)); - } - break; - - case 7: //F(fd) = -F(fs); break; //neg - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fd == fs, true); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&ssSignBits2[0])); - if (fd != fs && fpr.IsMapped(fs)) { - MOVAPS(fpr.RX(fd), MatR(TEMPREG)); - XORPS(fpr.RX(fd), fpr.R(fs)); - } else { - if (fd != fs) { - MOVSS(fpr.RX(fd), fpr.R(fs)); - } - XORPS(fpr.RX(fd), MatR(TEMPREG)); - } - break; - - case 4: //F(fd) = sqrtf(F(fs)); break; //sqrt - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fd == fs, true); - // x86 gives a negative NaN for a negative input, the PSP a positive one: clear the sign - // where the input was negative. -0 and NaN inputs come through as they are. - XORPS(XMM1, R(XMM1)); - CopyFPReg(XMM0, fpr.R(fs)); - CMPSS(XMM0, R(XMM1), CMP_LT); - if (RipAccessible(&ssSignBits2[0])) { - ANDPS(XMM0, M(&ssSignBits2[0])); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&ssSignBits2[0])); - ANDPS(XMM0, MatR(TEMPREG)); - } - SQRTSS(fpr.RX(fd), fpr.R(fs)); - XORPS(fpr.RX(fd), R(XMM0)); - break; - - case 13: //FsI(fd) = F(fs)>=0 ? (int)floorf(F(fs)) : (int)ceilf(F(fs)); break; //trunc.w.s - execRounding(&XEmitter::CVTTSS2SI, -1); - break; - - case 32: //F(fd) = (float)FsI(fs); break; //cvt.s.w - fpr.SpillLock(fd, fs); - fpr.MapReg(fd, fs == fd, true); - if (fpr.IsMapped(fs)) { - CVTDQ2PS(fpr.RX(fd), fpr.R(fs)); - } else { - // If fs was fd, we'd be in the case above since we mapped fd. - MOVSS(fpr.RX(fd), fpr.R(fs)); - CVTDQ2PS(fpr.RX(fd), fpr.R(fd)); - } - break; - - case 36: //FsI(fd) = (int) F(fs); break; //cvt.w.s - // Uses the current rounding mode. - execRounding(&XEmitter::CVTSS2SI, -1); - break; - - case 12: //FsI(fd) = (int)floorf(F(fs)+0.5f); break; //round.w.s - execRounding(&XEmitter::CVTSS2SI, 0); - break; - case 14: //FsI(fd) = (int)ceilf (F(fs)); break; //ceil.w.s - execRounding(&XEmitter::CVTSS2SI, 2); - break; - case 15: //FsI(fd) = (int)floorf(F(fs)); break; //floor.w.s - execRounding(&XEmitter::CVTSS2SI, 1); - break; - default: - DISABLE; - return; - } - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_mxc1(MIPSOpcode op) { - CONDITIONAL_DISABLE(FPU_XFER); - - int fs = _FS; - MIPSGPReg rt = _RT; - - switch ((op >> 21) & 0x1f) { - case 0: // R(rt) = FI(fs); break; //mfc1 - if (rt == MIPS_REG_ZERO) - return; - gpr.MapReg(rt, false, true); - // If fs is not mapped, most likely it's being abandoned. - // Just load from memory in that case. - if (fpr.R(fs).IsSimpleReg()) { - MOVD_xmm(gpr.R(rt), fpr.RX(fs)); - } else { - MOV(32, gpr.R(rt), fpr.R(fs)); - } - break; - - case 2: // R(rt) = currentMIPS->ReadFCR(fs); break; //cfc1 - if (rt == MIPS_REG_ZERO) - return; - if (fs == 31) { - bool wasImm = gpr.IsImm(MIPS_REG_FPCOND); - if (!wasImm) { - gpr.Lock(rt, MIPS_REG_FPCOND); - gpr.MapReg(MIPS_REG_FPCOND, true, false); - } - gpr.MapReg(rt, false, true); - MOV(32, gpr.R(rt), MIPSSTATE_VAR(fcr31)); - if (wasImm) { - if (gpr.GetImm(MIPS_REG_FPCOND) & 1) { - OR(32, gpr.R(rt), Imm32(1 << 23)); - } else { - AND(32, gpr.R(rt), Imm32(~(1 << 23))); - } - } else { - AND(32, gpr.R(rt), Imm32(~(1 << 23))); - MOV(32, R(TEMPREG), gpr.R(MIPS_REG_FPCOND)); - AND(32, R(TEMPREG), Imm32(1)); - SHL(32, R(TEMPREG), Imm8(23)); - OR(32, gpr.R(rt), R(TEMPREG)); - } - gpr.UnlockAll(); - } else if (fs == 0) { - gpr.SetImm(rt, MIPSState::FCR0_VALUE); - } else { - Comp_Generic(op); - } - return; - - case 4: //FI(fs) = R(rt); break; //mtc1 - fpr.MapReg(fs, false, true); - if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0) { - XORPS(fpr.RX(fs), fpr.R(fs)); - } else { - gpr.KillImmediate(rt, true, false); - MOVD_xmm(fpr.RX(fs), gpr.R(rt)); - } - return; - - case 6: //currentMIPS->WriteFCR(fs, R(rt)); break; //ctc1 - if (fs == 31) { - // Must clear before setting, since ApplyRoundingMode() assumes it was cleared. - RestoreRoundingMode(); - if (gpr.IsImm(rt)) { - gpr.SetImm(MIPS_REG_FPCOND, (gpr.GetImm(rt) >> 23) & 1); - MOV(32, MIPSSTATE_VAR(fcr31), Imm32(gpr.GetImm(rt) & 0x0181FFFF)); - if ((gpr.GetImm(rt) & 0x1000003) == 0) { - // Default nearest / no-flush mode, just leave it cleared. - } else { - UpdateRoundingMode(gpr.GetImm(rt)); - ApplyRoundingMode(); - } - } else { - gpr.Lock(rt, MIPS_REG_FPCOND); - gpr.MapReg(rt, true, false); - gpr.MapReg(MIPS_REG_FPCOND, false, true); - MOV(32, gpr.R(MIPS_REG_FPCOND), gpr.R(rt)); - SHR(32, gpr.R(MIPS_REG_FPCOND), Imm8(23)); - AND(32, gpr.R(MIPS_REG_FPCOND), Imm32(1)); - MOV(32, MIPSSTATE_VAR(fcr31), gpr.R(rt)); - AND(32, MIPSSTATE_VAR(fcr31), Imm32(0x0181FFFF)); - gpr.UnlockAll(); - UpdateRoundingMode(); - ApplyRoundingMode(); - } - } else { - Comp_Generic(op); - } - return; - } -} - -} // namespace MIPSComp - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompLoadStore.cpp b/Core/MIPS/x86/CompLoadStore.cpp deleted file mode 100644 index 70d77337e1..0000000000 --- a/Core/MIPS/x86/CompLoadStore.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Core/MemMap.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/Config.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" - - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE(flag) { Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { Comp_Generic(op); return; } -#define INVALIDOP { Comp_Generic(op); return; } - -namespace MIPSComp { - using namespace Gen; - - void Jit::CompITypeMemRead(MIPSOpcode op, u32 bits, void (XEmitter::*mov)(int, int, X64Reg, OpArg), const void *safeFunc) - { - CONDITIONAL_DISABLE(LSU); - int offset = _IMM16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - gpr.Lock(rt, rs); - gpr.MapReg(rt, rt == rs, true); - - JitSafeMem safe(this, rs, offset); - OpArg src; - if (safe.PrepareRead(src, bits / 8)) - (this->*mov)(32, bits, gpr.RX(rt), src); - if (safe.PrepareSlowRead(safeFunc)) - (this->*mov)(32, bits, gpr.RX(rt), R(EAX)); - safe.Finish(); - - gpr.UnlockAll(); - } - - static OpArg DowncastImm(OpArg in, int bits) { - if (!in.IsImm()) - return in; - if (in.GetImmBits() > bits) { - in.SetImmBits(bits); - return in; - } - return in; - } - - void Jit::CompITypeMemWrite(MIPSOpcode op, u32 bits, const void *safeFunc, bool makeRTWritable) - { - CONDITIONAL_DISABLE(LSU); - int offset = _IMM16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - gpr.Lock(rt, rs); - - if (rt == MIPS_REG_ZERO || gpr.R(rt).IsImm()) { - if (makeRTWritable) { - gpr.MapReg(rt, true, true); - } - // NOTICE_LOG(Log::JIT, "%d-bit Imm at %08x : %08x", bits, js.blockStart, (u32)gpr.R(rt).GetImmValue()); - } else { - gpr.MapReg(rt, true, false); - } - -#if PPSSPP_ARCH(X86) - // We use EDX so we can have DL for 8-bit ops. - const bool needSwap = bits == 8 && !gpr.R(rt).IsSimpleReg(EDX) && !gpr.R(rt).IsSimpleReg(ECX); - if (needSwap) - gpr.FlushLockX(EDX); -#else - const bool needSwap = false; -#endif - - JitSafeMem safe(this, rs, offset); - OpArg dest; - if (safe.PrepareWrite(dest, bits / 8)) - { - if (needSwap) - { - MOV(32, R(EDX), gpr.R(rt)); - MOV(bits, dest, R(EDX)); - } - else { - if (rt == MIPS_REG_ZERO) { - switch (bits) { - case 8: MOV(8, dest, Imm8(0)); break; - case 16: MOV(16, dest, Imm16(0)); break; - case 32: MOV(32, dest, Imm32(0)); break; - } - } else { - // The downcast is needed so we don't try to generate a 8-bit write with a 32-bit imm - // (that might have been generated from an li instruction) which is illegal. - MOV(bits, dest, DowncastImm(gpr.R(rt), bits)); - } - } - } - if (safe.PrepareSlowWrite()) - safe.DoSlowWrite(safeFunc, gpr.R(rt)); - safe.Finish(); - - if (needSwap) - gpr.UnlockAllX(); - gpr.UnlockAll(); - } - - void Jit::CompITypeMemUnpairedLR(MIPSOpcode op, bool isStore) - { - CONDITIONAL_DISABLE(LSU); - int offset = _IMM16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - - X64Reg shiftReg = ECX; - gpr.FlushLockX(ECX, EDX); -#if PPSSPP_ARCH(AMD64) - // On x64, we need ECX for CL, but it's also the first arg and gets lost. Annoying. - gpr.FlushLockX(R9); - shiftReg = R9; -#endif - - gpr.Lock(rt, rs); - gpr.MapReg(rt, true, !isStore); - - // Grab the offset from alignment for shifting (<< 3 for bytes -> bits.) - MOV(32, R(shiftReg), gpr.R(rs)); - ADD(32, R(shiftReg), Imm32(offset)); - AND(32, R(shiftReg), Imm32(3)); - SHL(32, R(shiftReg), Imm8(3)); - - { - JitSafeMem safe(this, rs, offset, ~3); - OpArg src; - if (safe.PrepareRead(src, 4)) - { - if (!src.IsSimpleReg(EAX)) - MOV(32, R(EAX), src); - - CompITypeMemUnpairedLRInner(op, shiftReg); - } - if (safe.PrepareSlowRead(safeMemFuncs.readU32)) - CompITypeMemUnpairedLRInner(op, shiftReg); - safe.Finish(); - } - - // For store ops, write EDX back to memory. - if (isStore) - { - JitSafeMem safe(this, rs, offset, ~3); - OpArg dest; - if (safe.PrepareWrite(dest, 4)) - MOV(32, dest, R(EDX)); - if (safe.PrepareSlowWrite()) - safe.DoSlowWrite(safeMemFuncs.writeU32, R(EDX)); - safe.Finish(); - } - - gpr.UnlockAll(); - gpr.UnlockAllX(); - } - - void Jit::CompITypeMemUnpairedLRInner(MIPSOpcode op, X64Reg shiftReg) - { - CONDITIONAL_DISABLE(LSU); - int o = op>>26; - MIPSGPReg rt = _RT; - - // Make sure we have the shift for the target in ECX. - if (shiftReg != ECX) - MOV(32, R(ECX), R(shiftReg)); - - // Now use that shift (left on target, right on source.) - switch (o) - { - case 34: //lwl - MOV(32, R(EDX), Imm32(0x00ffffff)); - SHR(32, R(EDX), R(CL)); - AND(32, gpr.R(rt), R(EDX)); - break; - - case 38: //lwr - SHR(32, R(EAX), R(CL)); - break; - - case 42: //swl - MOV(32, R(EDX), Imm32(0xffffff00)); - SHL(32, R(EDX), R(CL)); - AND(32, R(EAX), R(EDX)); - break; - - case 46: //swr - MOV(32, R(EDX), gpr.R(rt)); - SHL(32, R(EDX), R(CL)); - // EDX is already the target value to write, but may be overwritten below. Save it. - PUSH(EDX); - break; - - default: - _dbg_assert_msg_(false, "Unsupported left/right load/store instruction."); - } - - // Flip ECX around from 3 bytes / 24 bits. - if (shiftReg == ECX) - { - MOV(32, R(EDX), Imm32(24)); - SUB(32, R(EDX), R(ECX)); - MOV(32, R(ECX), R(EDX)); - } - else - { - MOV(32, R(ECX), Imm32(24)); - SUB(32, R(ECX), R(shiftReg)); - } - - // Use the flipped shift (left on source, right on target) and write target. - switch (o) - { - case 34: //lwl - SHL(32, R(EAX), R(CL)); - - OR(32, gpr.R(rt), R(EAX)); - break; - - case 38: //lwr - MOV(32, R(EDX), Imm32(0xffffff00)); - SHL(32, R(EDX), R(CL)); - AND(32, gpr.R(rt), R(EDX)); - - OR(32, gpr.R(rt), R(EAX)); - break; - - case 42: //swl - MOV(32, R(EDX), gpr.R(rt)); - SHR(32, R(EDX), R(CL)); - - OR(32, R(EDX), R(EAX)); - break; - - case 46: //swr - MOV(32, R(EDX), Imm32(0x00ffffff)); - SHR(32, R(EDX), R(CL)); - AND(32, R(EAX), R(EDX)); - - // This is the target value we saved earlier. - POP(EDX); - OR(32, R(EDX), R(EAX)); - break; - - default: - _dbg_assert_msg_(false, "Unsupported left/right load/store instruction."); - } - } - - void Jit::Comp_ITypeMem(MIPSOpcode op) - { - CONDITIONAL_DISABLE(LSU); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - // NOTE: This should be done before the $zr check in case there are reads with side effects. - DISABLE; - return; - } - - int offset = _IMM16; - MIPSGPReg rs = _RS; - MIPSGPReg rt = _RT; - int o = op>>26; - // 16-bit accesses can go through the depth layout (Memory::DepthMirrored16). - if (Memory::DepthMirrorsActive() && (o == 33 || o == 37 || o == 41)) { - DISABLE; - } - if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) { - // Don't load anything into $zr - return; - } - - CheckMemoryBreakpoint(0, rs, offset); - - switch (o) - { - case 37: //R(rt) = ReadMem16(addr); break; //lhu - CompITypeMemRead(op, 16, &XEmitter::MOVZX, safeMemFuncs.readU16); - break; - - case 36: //R(rt) = ReadMem8 (addr); break; //lbu - CompITypeMemRead(op, 8, &XEmitter::MOVZX, safeMemFuncs.readU8); - break; - - case 35: //R(rt) = ReadMem32(addr); break; //lw - CompITypeMemRead(op, 32, &XEmitter::MOVZX, safeMemFuncs.readU32); - break; - - case 32: //R(rt) = SignExtend8ToU32 (ReadMem8 (addr)); break; //lb - CompITypeMemRead(op, 8, &XEmitter::MOVSX, safeMemFuncs.readU8); - break; - - case 33: //R(rt) = SignExtend16ToU32(ReadMem16(addr)); break; //lh - CompITypeMemRead(op, 16, &XEmitter::MOVSX, safeMemFuncs.readU16); - break; - - case 40: //WriteMem8 (addr, R(rt)); break; //sb - CompITypeMemWrite(op, 8, safeMemFuncs.writeU8); - break; - - case 41: //WriteMem16(addr, R(rt)); break; //sh - CompITypeMemWrite(op, 16, safeMemFuncs.writeU16); - break; - - case 43: //WriteMem32(addr, R(rt)); break; //sw - CompITypeMemWrite(op, 32, safeMemFuncs.writeU32); - break; - - case 34: //lwl - { - MIPSOpcode nextOp = GetOffsetInstruction(1); - // Looking for lwr rd, offset-3(rs) which makes a pair. - u32 desiredOp = ((op & 0xFFFF0000) + (4 << 26)) + (offset - 3); - if (!js.inDelaySlot && nextOp == desiredOp && !jo.Disabled(JitDisable::LSU_UNALIGNED)) - { - CheckMemoryBreakpoint(1, rs, offset - 3); - EatInstruction(nextOp); - // nextOp has the correct address. - CompITypeMemRead(nextOp, 32, &XEmitter::MOVZX, safeMemFuncs.readU32); - } - else - CompITypeMemUnpairedLR(op, false); - } - break; - - case 38: //lwr - { - MIPSOpcode nextOp = GetOffsetInstruction(1); - // Looking for lwl rd, offset+3(rs) which makes a pair. - u32 desiredOp = ((op & 0xFFFF0000) - (4 << 26)) + (offset + 3); - if (!js.inDelaySlot && nextOp == desiredOp && !jo.Disabled(JitDisable::LSU_UNALIGNED)) - { - CheckMemoryBreakpoint(1, rs, offset + 3); - EatInstruction(nextOp); - // op has the correct address. - CompITypeMemRead(op, 32, &XEmitter::MOVZX, safeMemFuncs.readU32); - } - else - CompITypeMemUnpairedLR(op, false); - } - break; - - case 42: //swl - { - MIPSOpcode nextOp = GetOffsetInstruction(1); - // Looking for swr rd, offset-3(rs) which makes a pair. - u32 desiredOp = ((op & 0xFFFF0000) + (4 << 26)) + (offset - 3); - if (!js.inDelaySlot && nextOp == desiredOp && !jo.Disabled(JitDisable::LSU_UNALIGNED)) - { - CheckMemoryBreakpoint(1, rs, offset - 3); - EatInstruction(nextOp); - // nextOp has the correct address. - CompITypeMemWrite(nextOp, 32, safeMemFuncs.writeU32); - } - else - CompITypeMemUnpairedLR(op, true); - } - break; - - case 46: //swr - { - MIPSOpcode nextOp = GetOffsetInstruction(1); - // Looking for swl rd, offset+3(rs) which makes a pair. - u32 desiredOp = ((op & 0xFFFF0000) - (4 << 26)) + (offset + 3); - if (!js.inDelaySlot && nextOp == desiredOp && !jo.Disabled(JitDisable::LSU_UNALIGNED)) - { - CheckMemoryBreakpoint(1, rs, offset + 3); - EatInstruction(nextOp); - // op has the correct address. - CompITypeMemWrite(op, 32, safeMemFuncs.writeU32); - } - else - CompITypeMemUnpairedLR(op, true); - } - break; - - default: - Comp_Generic(op); - return; - } - - } - - void Jit::Comp_StoreSync(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - int offset = _IMM16; - MIPSGPReg rt = _RT; - MIPSGPReg rs = _RS; - // Note: still does something even if loading to zero. - - CheckMemoryBreakpoint(0, rs, offset); - - FixupBranch skipStore; - FixupBranch finish; - switch (op >> 26) { - case 48: // ll - CompITypeMemRead(op, 32, &XEmitter::MOVZX, safeMemFuncs.readU32); - MOV(8, MDisp(X64JitConstants::CTXREG, -128 + offsetof(MIPSState, llBit)), Imm8(1)); - break; - - case 56: // sc - // Map before the jump in case any regs spill. Unlock happens inside CompITypeMemWrite(). - // This is not a very common op, but it's in jit so memory breakpoints can trip. - gpr.Lock(rt, rs); - gpr.MapReg(rt, true, true); - gpr.MapReg(rs, true, false); - - CMP(8, MDisp(X64JitConstants::CTXREG, -128 + offsetof(MIPSState, llBit)), Imm8(1)); - skipStore = J_CC(CC_NE); - - CompITypeMemWrite(op, 32, safeMemFuncs.writeU32, true); - // $zero is an immediate, and has no result to write. - if (rt != MIPS_REG_ZERO) { - MOV(32, gpr.R(rt), Imm32(1)); - finish = J(); - SetJumpTarget(skipStore); - MOV(32, gpr.R(rt), Imm32(0)); - SetJumpTarget(finish); - } else { - SetJumpTarget(skipStore); - } - break; - - default: - INVALIDOP; - } - } - - void Jit::Comp_Cache(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU); - - int func = (op >> 16) & 0x1F; - - // See Int_Cache for the definitions. - switch (func) { - case 24: break; - case 25: break; - case 27: break; - case 30: break; - default: - // Fall back to the interpreter. - DISABLE; - } - } -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompReplace.cpp b/Core/MIPS/x86/CompReplace.cpp deleted file mode 100644 index c7b98aa1b1..0000000000 --- a/Core/MIPS/x86/CompReplace.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2013- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Core/MemMap.h" -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/x86/RegCache.h" -#include "Core/MIPS/x86/Jit.h" - -alignas(16) static const u64 ssNoSignMask[2] = {0x7FFFFFFF7FFFFFFFULL, 0x7FFFFFFF7FFFFFFFULL}; - -namespace MIPSComp { -using namespace Gen; - -int Jit::Replace_fabsf() { - fpr.SpillLock(0, 12); - fpr.MapReg(0, false, true); - MOVSS(fpr.RX(0), fpr.R(12)); - MOV(PTRBITS, R(RAX), ImmPtr(&ssNoSignMask)); - ANDPS(fpr.RX(0), MatR(RAX)); - fpr.ReleaseSpillLocks(); - return 4; // Number of instructions in the MIPS function -} - -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/CompVFPU.cpp b/Core/MIPS/x86/CompVFPU.cpp deleted file mode 100644 index 9cdf0587ec..0000000000 --- a/Core/MIPS/x86/CompVFPU.cpp +++ /dev/null @@ -1,3860 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -// Table 13.10 in http://agner.org/optimize/optimizing_assembly.pdf is cool - generate constants with -// short instruction sequences. Surprisingly many are possible. - -#include "ppsspp_config.h" - -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) -#include -#include -#include "Common/Math/math_util.h" - -#include "Common/CPUDetect.h" -#include "Common/Math/SIMDHeaders.h" -#include "Common/Log.h" -#include "Core/Compatibility.h" -#include "Core/Config.h" -#include "Core/MemMap.h" -#include "Core/Reporting.h" -#include "Core/System.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/MIPSVFPUUtils.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" - -// All functions should have CONDITIONAL_DISABLE, so we can narrow things down to a file quickly. -// Currently known non working ones should have DISABLE. - -// #define CONDITIONAL_DISABLE { fpr.ReleaseSpillLocks(); Comp_Generic(op); return; } -#define CONDITIONAL_DISABLE(flag) if (jo.Disabled(JitDisable::flag)) { Comp_Generic(op); return; } -#define DISABLE { fpr.ReleaseSpillLocks(); Comp_Generic(op); return; } - -#define _RS MIPS_GET_RS(op) -#define _RT MIPS_GET_RT(op) -#define _RD MIPS_GET_RD(op) -#define _FS MIPS_GET_FS(op) -#define _FT MIPS_GET_FT(op) -#define _FD MIPS_GET_FD(op) -#define _SA MIPS_GET_SA(op) -#define _POS ((op>> 6) & 0x1F) -#define _SIZE ((op>>11) & 0x1F) -#define _IMM16 (signed short)(op & 0xFFFF) -#define _IMM26 (op & 0x03FFFFFF) - -namespace MIPSComp -{ -using namespace Gen; -using namespace X64JitConstants; - -static const float one = 1.0f; -static const float minus_one = -1.0f; - -alignas(16) const u32 noSignMask[4] = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF}; -alignas(16) const u32 signBitAll[4] = {0x80000000, 0x80000000, 0x80000000, 0x80000000}; -alignas(16) const u32 signBitLower[4] = {0x80000000, 0, 0, 0}; -alignas(16) const float oneOneOneOne[4] = {1.0f, 1.0f, 1.0f, 1.0f}; -alignas(16) const u32 fourinfnan[4] = {0x7F800000, 0x7F800000, 0x7F800000, 0x7F800000}; -alignas(16) const float identityMatrix[4][4] = { { 1.0f, 0, 0, 0 }, { 0, 1.0f, 0, 0 }, { 0, 0, 1.0f, 0 }, { 0, 0, 0, 1.0f} }; - -void Jit::Comp_VPFX(MIPSOpcode op) -{ - CONDITIONAL_DISABLE(VFPU_XFER); - int data = op & 0xFFFFF; - int regnum = (op >> 24) & 3; - switch (regnum) { - case 0: // S - js.prefixS = data; - js.prefixSFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - case 1: // T - js.prefixT = data; - js.prefixTFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - case 2: // D - js.prefixD = data & 0x00000FFF; - js.prefixDFlag = JitState::PREFIX_KNOWN_DIRTY; - break; - } -} - -void Jit::ApplyPrefixST(u8 *vregs, u32 prefix, VectorSize sz) { - if (prefix == 0xE4) return; - - int n = GetNumVectorElements(sz); - u8 origV[4]; - static const float constantArray[8] = {0.f, 1.f, 2.f, 0.5f, 3.f, 1.f/3.f, 0.25f, 1.f/6.f}; - - for (int i = 0; i < n; i++) - origV[i] = vregs[i]; - - for (int i = 0; i < n; i++) { - int regnum = (prefix >> (i*2)) & 3; - int abs = (prefix >> (8+i)) & 1; - int negate = (prefix >> (16+i)) & 1; - int constants = (prefix >> (12+i)) & 1; - - // Unchanged, hurray. - if (!constants && regnum == i && !abs && !negate) - continue; - - // This puts the value into a temp reg, so we won't write the modified value back. - vregs[i] = fpr.GetTempV(); - fpr.MapRegV(vregs[i], MAP_NOINIT | MAP_DIRTY); - - if (!constants) { - // Prefix may say "z, z, z, z" but if this is a pair, we force to x. - // TODO: But some ops seem to use const 0 instead? - if (regnum >= n) { - ERROR_LOG_REPORT(Log::CPU, "Invalid VFPU swizzle: %08x / %d", prefix, sz); - regnum = 0; - } - fpr.SimpleRegV(origV[regnum], 0); - MOVSS(fpr.VX(vregs[i]), fpr.V(origV[regnum])); - if (abs) { - if (RipAccessible(&noSignMask)) { - ANDPS(fpr.VX(vregs[i]), M(&noSignMask)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&noSignMask)); - ANDPS(fpr.VX(vregs[i]), MatR(TEMPREG)); - } - } - } else { - if (RipAccessible(constantArray)) { - MOVSS(fpr.VX(vregs[i]), M(&constantArray[regnum + (abs << 2)])); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&constantArray[regnum + (abs << 2)])); - MOVSS(fpr.VX(vregs[i]), MatR(TEMPREG)); - } - } - - if (negate) { - if (RipAccessible(&signBitLower)) { - XORPS(fpr.VX(vregs[i]), M(&signBitLower)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&signBitLower)); - XORPS(fpr.VX(vregs[i]), MatR(TEMPREG)); - } - } - // TODO: This probably means it will swap out soon, inefficiently... - fpr.ReleaseSpillLockV(vregs[i]); - } -} - -void Jit::GetVectorRegsPrefixD(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixDFlag & JitState::PREFIX_KNOWN); - - GetVectorRegs(regs, sz, vectorReg); - if (js.prefixD == 0) - return; - - int n = GetNumVectorElements(sz); - for (int i = 0; i < n; i++) { - // Hopefully this is rare, we'll just write it into a reg we drop. - if (js.VfpuWriteMask(i)) - regs[i] = fpr.GetTempV(); - } -} - -void Jit::ApplyPrefixD(const u8 *vregs, VectorSize sz) { - _assert_(js.prefixDFlag & JitState::PREFIX_KNOWN); - if (!js.prefixD) return; - - int n = GetNumVectorElements(sz); - for (int i = 0; i < n; i++) { - if (js.VfpuWriteMask(i)) - continue; - - int sat = (js.prefixD >> (i * 2)) & 3; - if (sat == 1) { - fpr.MapRegV(vregs[i], MAP_DIRTY); - - // Zero out XMM0 if it was <= +0.0f (but skip NAN.) - MOVSS(R(XMM0), fpr.VX(vregs[i])); - XORPS(XMM1, R(XMM1)); - CMPLESS(XMM0, R(XMM1)); - ANDNPS(XMM0, fpr.V(vregs[i])); - - // Retain a NAN in XMM0 (must be second operand.) - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(fpr.VX(vregs[i]), MatR(TEMPREG)); - MINSS(fpr.VX(vregs[i]), R(XMM0)); - } else if (sat == 3) { - fpr.MapRegV(vregs[i], MAP_DIRTY); - - // Check for < -1.0f, but careful of NANs. - MOV(PTRBITS, R(TEMPREG), ImmPtr(&minus_one)); - MOVSS(XMM1, MatR(TEMPREG)); - MOVSS(R(XMM0), fpr.VX(vregs[i])); - CMPLESS(XMM0, R(XMM1)); - // If it was NOT less, the three ops below do nothing. - // Otherwise, they replace the value with -1.0f. - ANDPS(XMM1, R(XMM0)); - ANDNPS(XMM0, fpr.V(vregs[i])); - ORPS(XMM0, R(XMM1)); - - // Retain a NAN in XMM0 (must be second operand.) - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(fpr.VX(vregs[i]), MatR(TEMPREG)); - MINSS(fpr.VX(vregs[i]), R(XMM0)); - } - } -} - -// Vector regs can overlap in all sorts of swizzled ways. -// This does allow a single overlap in sregs[i]. -// True if the prefix only touches lanes the op has. A position past the size may only be the -// identity, and a position within it may not name a lane past it (which zeroes the result lane -// on hardware, see cpu/vfpu/prefix_ctrl - the interpreter handles that). -static bool IsPrefixWithinSize(u32 prefix, VectorSize sz) { - int n = GetNumVectorElements(sz); - for (int i = 0; i < 4; i++) { - int regnum = (prefix >> (i * 2)) & 3; - int abs = (prefix >> (8 + i)) & 1; - int negate = (prefix >> (16 + i)) & 1; - int constants = (prefix >> (12 + i)) & 1; - if (constants) { - continue; - } - if (i >= n) { - if (abs || negate || regnum != i) - return false; - } else if (regnum >= n) { - return false; - } - } - return true; -} - -static bool IsPrefixWithinSize(u32 prefix, MIPSOpcode op) { - return IsPrefixWithinSize(prefix, GetVecSize(op)); -} - -bool IsOverlapSafeAllowS(int dreg, int di, int sn, const u8 sregs[], int tn = 0, const u8 tregs[] = NULL) { - for (int i = 0; i < sn; ++i) { - if (sregs[i] == dreg && i != di) - return false; - } - for (int i = 0; i < tn; ++i) { - if (tregs[i] == dreg) - return false; - } - - // Hurray, no overlap, we can write directly. - return true; -} - -bool IsOverlapSafe(int dreg, int di, int sn, const u8 sregs[], int tn = 0, const u8 tregs[] = NULL) { - return IsOverlapSafeAllowS(dreg, di, sn, sregs, tn, tregs) && sregs[di] != dreg; -} - -void Jit::Comp_SV(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU_VFPU); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - s32 imm = (signed short)(op&0xFFFC); - int vt = ((op >> 16) & 0x1f) | ((op & 3) << 5); - MIPSGPReg rs = _RS; - - CheckMemoryBreakpoint(0, rs, imm); - - switch (op >> 26) { - case 50: // lv.s - { - gpr.Lock(rs); - fpr.MapRegV(vt, MAP_DIRTY | MAP_NOINIT); - - JitSafeMem safe(this, rs, imm); - OpArg src; - if (safe.PrepareRead(src, 4)) { - MOVSS(fpr.VX(vt), safe.NextFastAddress(0)); - } - if (safe.PrepareSlowRead(safeMemFuncs.readU32)) { - MOVD_xmm(fpr.VX(vt), R(EAX)); - } - safe.Finish(); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - - case 58: // sv.s - { - gpr.Lock(rs); - - fpr.MapRegV(vt, 0); - - JitSafeMem safe(this, rs, imm); - OpArg dest; - if (safe.PrepareWrite(dest, 4)) { - MOVSS(safe.NextFastAddress(0), fpr.VX(vt)); - } - if (safe.PrepareSlowWrite()) { - MOVSS(MIPSSTATE_VAR(temp), fpr.VX(vt)); - safe.DoSlowWrite(safeMemFuncs.writeU32, MIPSSTATE_VAR(temp), 0); - } - safe.Finish(); - - fpr.ReleaseSpillLocks(); - gpr.UnlockAll(); - } - break; - - default: - DISABLE; - } -} - -void Jit::Comp_SVQ(MIPSOpcode op) { - CONDITIONAL_DISABLE(LSU_VFPU); - - if (js.kernelMode) { - // Send all memory accesses to the interpreter in kernel mode. - // TODO: Do something faster - but it hardly matters, currently this is VSH-only. - DISABLE; - return; - } - - int imm = (signed short)(op&0xFFFC); - int vt = (((op >> 16) & 0x1f)) | ((op&1) << 5); - MIPSGPReg rs = _RS; - - CheckMemoryBreakpoint(0, rs, imm); - - switch (op >> 26) { - case 53: //lvl.q/lvr.q - { - if (!g_Config.bFastMemory) { - DISABLE; - } - DISABLE; // The code below isn't quite working, so we fall back to interpreter for now. - - gpr.MapReg(rs, true, false); - gpr.FlushLockX(ECX); - u8 vregs[4]; - GetVectorRegs(vregs, V_Quad, vt); - MOV(32, R(EAX), gpr.R(rs)); - ADD(32, R(EAX), Imm32(imm)); -#ifdef MASKED_PSP_MEMORY - AND(32, R(EAX), Imm32(Memory::MEMVIEW32_MASK)); -#endif - MOV(32, R(ECX), R(EAX)); - SHR(32, R(EAX), Imm8(2)); - AND(32, R(EAX), Imm32(0x3)); - CMP(32, R(EAX), Imm32(0)); - FixupBranch next = J_CC(CC_NE); - - auto PSPMemAddr = [](X64Reg scaled, int offset) { -#if PPSSPP_ARCH(X86) - return MDisp(scaled, (u32)Memory::base + offset); -#else - return MComplex(MEMBASEREG, scaled, 1, offset); -#endif - }; - - fpr.MapRegsV(vregs, V_Quad, MAP_DIRTY); - - // Offset = 0 - MOVSS(fpr.RX(vregs[3]), PSPMemAddr(EAX, 0)); - - FixupBranch skip0 = J(); - SetJumpTarget(next); - CMP(32, R(EAX), Imm32(1)); - next = J_CC(CC_NE); - - // Offset = 1 - MOVSS(fpr.RX(vregs[3]), PSPMemAddr(EAX, 4)); - MOVSS(fpr.RX(vregs[2]), PSPMemAddr(EAX, 0)); - - FixupBranch skip1 = J(); - SetJumpTarget(next); - CMP(32, R(EAX), Imm32(2)); - next = J_CC(CC_NE); - - // Offset = 2 - MOVSS(fpr.RX(vregs[3]), PSPMemAddr(EAX, 8)); - MOVSS(fpr.RX(vregs[2]), PSPMemAddr(EAX, 4)); - MOVSS(fpr.RX(vregs[1]), PSPMemAddr(EAX, 0)); - - FixupBranch skip2 = J(); - SetJumpTarget(next); - CMP(32, R(EAX), Imm32(3)); - next = J_CC(CC_NE); - - // Offset = 3 - MOVSS(fpr.RX(vregs[3]), PSPMemAddr(EAX, 12)); - MOVSS(fpr.RX(vregs[2]), PSPMemAddr(EAX, 8)); - MOVSS(fpr.RX(vregs[1]), PSPMemAddr(EAX, 4)); - MOVSS(fpr.RX(vregs[0]), PSPMemAddr(EAX, 0)); - - SetJumpTarget(next); - SetJumpTarget(skip0); - SetJumpTarget(skip1); - SetJumpTarget(skip2); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - - case 54: //lv.q - { - gpr.Lock(rs); - // This must be in a reg or an immediate. - // Otherwise, it'll get put in EAX and we'll clobber that during NextSlowRead(). - if (!gpr.IsImm(rs)) - gpr.MapReg(rs, true, false); - - u8 vregs[4]; - GetVectorRegs(vregs, V_Quad, vt); - - if (fpr.TryMapRegsVS(vregs, V_Quad, MAP_NOINIT | MAP_DIRTY)) { - JitSafeMem safe(this, rs, imm); - OpArg src; - if (safe.PrepareRead(src, 16)) { - // Should be safe, since lv.q must be aligned, but let's try to avoid crashing in safe mode. - if (g_Config.bFastMemory) { - MOVAPS(fpr.VSX(vregs), safe.NextFastAddress(0)); - } else { - MOVUPS(fpr.VSX(vregs), safe.NextFastAddress(0)); - } - } - if (safe.PrepareSlowRead(safeMemFuncs.readU32)) { - for (int i = 0; i < 4; i++) { - safe.NextSlowRead(safeMemFuncs.readU32, i * 4); - // We use XMM0 as a temporary since MOVSS and MOVD would clear the higher bits. - MOVD_xmm(XMM0, R(EAX)); - MOVSS(fpr.VSX(vregs), R(XMM0)); - // Rotate things so we can read in the next higher float. - // By the end (4 rotates), they'll all be back into place. - SHUFPS(fpr.VSX(vregs), fpr.VS(vregs), _MM_SHUFFLE(0, 3, 2, 1)); - } - } - safe.Finish(); - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - return; - } - - fpr.MapRegsV(vregs, V_Quad, MAP_DIRTY | MAP_NOINIT); - - JitSafeMem safe(this, rs, imm); - OpArg src; - if (safe.PrepareRead(src, 16)) { - // Just copy 4 words the easiest way while not wasting registers. - for (int i = 0; i < 4; i++) - MOVSS(fpr.VX(vregs[i]), safe.NextFastAddress(i * 4)); - } - if (safe.PrepareSlowRead(safeMemFuncs.readU32)) { - for (int i = 0; i < 4; i++) { - safe.NextSlowRead(safeMemFuncs.readU32, i * 4); - MOVD_xmm(fpr.VX(vregs[i]), R(EAX)); - } - } - safe.Finish(); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - - case 62: //sv.q - { - gpr.Lock(rs); - // This must be in a reg or an immediate. - // Otherwise, it'll get put in EAX and we'll clobber that during NextSlowRead(). - if (!gpr.IsImm(rs)) - gpr.MapReg(rs, true, false); - - u8 vregs[4]; - GetVectorRegs(vregs, V_Quad, vt); - - if (fpr.TryMapRegsVS(vregs, V_Quad, 0)) { - JitSafeMem safe(this, rs, imm); - OpArg dest; - if (safe.PrepareWrite(dest, 16)) { - // Should be safe, since sv.q must be aligned, but let's try to avoid crashing in safe mode. - if (g_Config.bFastMemory) { - MOVAPS(safe.NextFastAddress(0), fpr.VSX(vregs)); - } else { - MOVUPS(safe.NextFastAddress(0), fpr.VSX(vregs)); - } - } - if (safe.PrepareSlowWrite()) { - MOVAPS(XMM0, fpr.VS(vregs)); - for (int i = 0; i < 4; i++) { - MOVSS(MIPSSTATE_VAR(temp), XMM0); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 3, 2, 1)); - safe.DoSlowWrite(safeMemFuncs.writeU32, MIPSSTATE_VAR(temp), i * 4); - } - } - safe.Finish(); - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - return; - } - - // Even if we don't use real SIMD there's still 8 or 16 scalar float registers. - fpr.MapRegsV(vregs, V_Quad, 0); - - JitSafeMem safe(this, rs, imm); - OpArg dest; - if (safe.PrepareWrite(dest, 16)) { - for (int i = 0; i < 4; i++) - MOVSS(safe.NextFastAddress(i * 4), fpr.VX(vregs[i])); - } - if (safe.PrepareSlowWrite()) { - for (int i = 0; i < 4; i++) { - MOVSS(MIPSSTATE_VAR(temp), fpr.VX(vregs[i])); - safe.DoSlowWrite(safeMemFuncs.writeU32, MIPSSTATE_VAR(temp), i * 4); - } - } - safe.Finish(); - - gpr.UnlockAll(); - fpr.ReleaseSpillLocks(); - } - break; - - default: - DISABLE; - break; - } -} - -void Jit::Comp_VVectorInit(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - // vzero/vone are vmov with a constant forced into the S prefix, so a pending one changes them. - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int type = (op >> 16) & 0xF; - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, _VD); - - if (fpr.TryMapRegsVS(dregs, sz, MAP_NOINIT | MAP_DIRTY)) { - if (type == 6) { - XORPS(fpr.VSX(dregs), fpr.VS(dregs)); - } else if (type == 7) { - if (RipAccessible(&oneOneOneOne)) { - MOVAPS(fpr.VSX(dregs), M(&oneOneOneOne)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - MOVAPS(fpr.VSX(dregs), MatR(TEMPREG)); - } - } else { - DISABLE; - } - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); - return; - } - - switch (type) { - case 6: // v=zeros; break; //vzero - XORPS(XMM0, R(XMM0)); - break; - case 7: // v=ones; break; //vone - if (RipAccessible(&one)) { - MOVSS(XMM0, M(&one)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(XMM0, MatR(TEMPREG)); - } - break; - default: - DISABLE; - break; - } - - int n = GetNumVectorElements(sz); - fpr.MapRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - for (int i = 0; i < n; ++i) - MOVSS(fpr.VX(dregs[i]), R(XMM0)); - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VIdt(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - // Like vone, a pending S prefix changes the result. - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - int vd = _VD; - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - // Only pairs and quads are handled here. - if (sz != V_Pair && sz != V_Quad) - DISABLE; - - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, _VD); - if (fpr.TryMapRegsVS(dregs, sz, MAP_NOINIT | MAP_DIRTY)) { - int row = vd & (n - 1); - if (RipAccessible(identityMatrix)) { - MOVAPS(fpr.VSX(dregs), M(identityMatrix[row])); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&identityMatrix[row])); - MOVAPS(fpr.VSX(dregs), MatR(TEMPREG)); - } - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); - return; - } - - XORPS(XMM0, R(XMM0)); - if (RipAccessible(&one)) { - MOVSS(XMM1, M(&one)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(XMM1, MatR(TEMPREG)); - } - fpr.MapRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - switch (sz) { - case V_Pair: - MOVSS(fpr.VX(dregs[0]), R((vd&1)==0 ? XMM1 : XMM0)); - MOVSS(fpr.VX(dregs[1]), R((vd&1)==1 ? XMM1 : XMM0)); - break; - case V_Quad: - MOVSS(fpr.VX(dregs[0]), R((vd&3)==0 ? XMM1 : XMM0)); - MOVSS(fpr.VX(dregs[1]), R((vd&3)==1 ? XMM1 : XMM0)); - MOVSS(fpr.VX(dregs[2]), R((vd&3)==2 ? XMM1 : XMM0)); - MOVSS(fpr.VX(dregs[3]), R((vd&3)==3 ? XMM1 : XMM0)); - break; - default: - _dbg_assert_msg_(false,"Trying to interpret instruction that can't be interpreted"); - break; - } - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VDot(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - // TODO: Force read one of them into regs? probably not. - u8 sregs[4], tregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, V_Single, _VD); - - // With SSE2, these won't really give any performance benefit on their own, but may reduce - // conversion costs from/to SIMD form. However, the SSE4.1 DPPS may be worth it. - // Benchmarking will have to decide whether to enable this on < SSE4.1. Also a HADDPS version - // for SSE3 could be written. - if (fpr.TryMapDirtyInInVS(dregs, V_Single, sregs, sz, tregs, sz)) { - switch (sz) { - case V_Pair: - if (cpu_info.bSSE4_1) { - if (fpr.VSX(dregs) != fpr.VSX(sregs) && fpr.VSX(dregs) != fpr.VSX(tregs)) { - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - DPPS(fpr.VSX(dregs), fpr.VS(tregs), 0x31); - } else { - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, fpr.VS(tregs), 0x31); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } - } else { - MOVAPS(XMM0, fpr.VS(sregs)); - MULPS(XMM0, fpr.VS(tregs)); - MOVAPS(R(XMM1), XMM0); - SHUFPS(XMM1, R(XMM0), _MM_SHUFFLE(1, 1, 1, 1)); - ADDPS(XMM1, R(XMM0)); - MOVAPS(fpr.VS(dregs), XMM1); - } - break; - case V_Triple: - if (cpu_info.bSSE4_1) { - if (fpr.VSX(dregs) != fpr.VSX(sregs) && fpr.VSX(dregs) != fpr.VSX(tregs)) { - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - DPPS(fpr.VSX(dregs), fpr.VS(tregs), 0x71); - } else { - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, fpr.VS(tregs), 0x71); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } - } else { - MOVAPS(XMM0, fpr.VS(sregs)); - MULPS(XMM0, fpr.VS(tregs)); - MOVAPS(R(XMM1), XMM0); - SHUFPS(XMM1, R(XMM0), _MM_SHUFFLE(3, 2, 1, 1)); - ADDSS(XMM1, R(XMM0)); - SHUFPS(XMM0, R(XMM1), _MM_SHUFFLE(3, 2, 2, 2)); - ADDSS(XMM1, R(XMM0)); - MOVAPS(fpr.VS(dregs), XMM1); - } - break; - case V_Quad: - if (cpu_info.bSSE4_1) { - if (fpr.VSX(dregs) != fpr.VSX(sregs) && fpr.VSX(dregs) != fpr.VSX(tregs)) { - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - DPPS(fpr.VSX(dregs), fpr.VS(tregs), 0xF1); - } else { - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, fpr.VS(tregs), 0xF1); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } - } /* else if (cpu_info.bSSE3) { // This is slower than the SSE2 solution on my Ivy! - MOVAPS(XMM0, fpr.VS(sregs)); - MOVAPS(XMM1, fpr.VS(tregs)); - HADDPS(XMM0, R(XMM1)); - HADDPS(XMM0, R(XMM0)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } */ else { - MOVAPS(XMM0, fpr.VS(sregs)); - MOVAPS(XMM1, fpr.VS(tregs)); - MULPS(XMM0, R(XMM1)); - MOVAPS(XMM1, R(XMM0)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(2, 3, 0, 1)); - ADDPS(XMM0, R(XMM1)); - MOVAPS(XMM1, R(XMM0)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(0, 1, 2, 3)); - ADDSS(XMM0, R(XMM1)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } - break; - default: - DISABLE; - } - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - - X64Reg tempxreg = XMM0; - if (IsOverlapSafe(dregs[0], 0, n, sregs, n, tregs)) { - fpr.MapRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - tempxreg = fpr.VX(dregs[0]); - } - - // Need to start with +0.0f so it doesn't result in -0.0f. - MOVSS(tempxreg, fpr.V(sregs[0])); - MULSS(tempxreg, fpr.V(tregs[0])); - for (int i = 1; i < n; i++) - { - // sum += s[i]*t[i]; - MOVSS(XMM1, fpr.V(sregs[i])); - MULSS(XMM1, fpr.V(tregs[i])); - ADDSS(tempxreg, R(XMM1)); - } - - if (!fpr.V(dregs[0]).IsSimpleReg(tempxreg)) { - fpr.MapRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - MOVSS(fpr.V(dregs[0]), tempxreg); - } - - ApplyPrefixD(dregs, V_Single); - - fpr.ReleaseSpillLocks(); -} - - -void Jit::Comp_VHdp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || js.HasSPrefix() || !IsPrefixWithinSize(js.prefixT, op)) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], tregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, V_Single, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - - X64Reg tempxreg = XMM0; - if (IsOverlapSafe(dregs[0], 0, n, sregs, n, tregs)) { - fpr.MapRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - tempxreg = fpr.VX(dregs[0]); - } - - // Need to start with +0.0f so it doesn't result in -0.0f. - MOVSS(tempxreg, fpr.V(sregs[0])); - MULSS(tempxreg, fpr.V(tregs[0])); - for (int i = 1; i < n; i++) { - // sum += (i == n-1) ? t[i] : s[i]*t[i]; - if (i == n - 1) { - ADDSS(tempxreg, fpr.V(tregs[i])); - } else { - MOVSS(XMM1, fpr.V(sregs[i])); - MULSS(XMM1, fpr.V(tregs[i])); - ADDSS(tempxreg, R(XMM1)); - } - } - - if (!fpr.V(dregs[0]).IsSimpleReg(tempxreg)) { - fpr.MapRegsV(dregs, V_Single, MAP_DIRTY | MAP_NOINIT); - MOVSS(fpr.V(dregs[0]), tempxreg); - } - - ApplyPrefixD(dregs, V_Single); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VCrossQuat(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - // The prefixes apply in their own way to these (the IR frontend says "weird prefixes"). - if (!js.HasNoPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegs(sregs, sz, _VS); - GetVectorRegs(tregs, sz, _VT); - GetVectorRegs(dregs, sz, _VD); - - if (sz == V_Triple) { - // Cross product vcrsp.t - if (fpr.TryMapDirtyInInVS(dregs, sz, sregs, sz, tregs, sz)) { - MOVAPS(XMM0, fpr.VS(tregs)); - MOVAPS(XMM1, fpr.VS(sregs)); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 0, 2, 1)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 0, 2, 1)); - MULPS(XMM0, fpr.VS(sregs)); - MULPS(XMM1, fpr.VS(tregs)); - SUBPS(XMM0, R(XMM1)); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 0, 2, 1)); - MOVAPS(fpr.VS(dregs), XMM0); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - fpr.MapRegsV(sregs, sz, 0); - - // Compute X - MOVSS(XMM0, fpr.V(sregs[1])); - MULSS(XMM0, fpr.V(tregs[2])); - MOVSS(XMM1, fpr.V(sregs[2])); - MULSS(XMM1, fpr.V(tregs[1])); - SUBSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[0]), XMM0); - - // Compute Y - MOVSS(XMM0, fpr.V(sregs[2])); - MULSS(XMM0, fpr.V(tregs[0])); - MOVSS(XMM1, fpr.V(sregs[0])); - MULSS(XMM1, fpr.V(tregs[2])); - SUBSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[1]), XMM0); - - // Compute Z - MOVSS(XMM0, fpr.V(sregs[0])); - MULSS(XMM0, fpr.V(tregs[1])); - MOVSS(XMM1, fpr.V(sregs[1])); - MULSS(XMM1, fpr.V(tregs[0])); - SUBSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[2]), XMM0); - } else if (sz == V_Quad) { - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // Quaternion product vqmul.q - fpr.MapRegsV(sregs, sz, 0); - - // Compute X - // d[0] = s[0] * t[3] + s[1] * t[2] - s[2] * t[1] + s[3] * t[0]; - MOVSS(XMM0, fpr.V(sregs[0])); - MULSS(XMM0, fpr.V(tregs[3])); - MOVSS(XMM1, fpr.V(sregs[1])); - MULSS(XMM1, fpr.V(tregs[2])); - ADDSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[2])); - MULSS(XMM1, fpr.V(tregs[1])); - SUBSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[3])); - MULSS(XMM1, fpr.V(tregs[0])); - ADDSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[0]), XMM0); - - // Compute Y, summed in the interpreter's order - //d[1] = -s[0] * t[2] + s[1] * t[3] + s[2] * t[0] + s[3] * t[1]; - MOVSS(XMM0, fpr.V(sregs[1])); - MULSS(XMM0, fpr.V(tregs[3])); - MOVSS(XMM1, fpr.V(sregs[0])); - MULSS(XMM1, fpr.V(tregs[2])); - SUBSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[2])); - MULSS(XMM1, fpr.V(tregs[0])); - ADDSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[3])); - MULSS(XMM1, fpr.V(tregs[1])); - ADDSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[1]), XMM0); - - // Compute Z - //d[2] = s[0] * t[1] - s[1] * t[0] + s[2] * t[3] + s[3] * t[2]; - MOVSS(XMM0, fpr.V(sregs[0])); - MULSS(XMM0, fpr.V(tregs[1])); - MOVSS(XMM1, fpr.V(sregs[1])); - MULSS(XMM1, fpr.V(tregs[0])); - SUBSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[2])); - MULSS(XMM1, fpr.V(tregs[3])); - ADDSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[3])); - MULSS(XMM1, fpr.V(tregs[2])); - ADDSS(XMM0, R(XMM1)); - MOVSS(fpr.V(dregs[2]), XMM0); - - // Compute W, summed in the interpreter's order: s[3] * t[3] - (s[0] * t[0] + s[1] * t[1] + s[2] * t[2]) - //d[3] = -s[0] * t[0] - s[1] * t[1] - s[2] * t[2] + s[3] * t[3]; - MOVSS(XMM0, fpr.V(sregs[0])); - MULSS(XMM0, fpr.V(tregs[0])); - MOVSS(XMM1, fpr.V(sregs[1])); - MULSS(XMM1, fpr.V(tregs[1])); - ADDSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[2])); - MULSS(XMM1, fpr.V(tregs[2])); - ADDSS(XMM0, R(XMM1)); - MOVSS(XMM1, fpr.V(sregs[3])); - MULSS(XMM1, fpr.V(tregs[3])); - SUBSS(XMM1, R(XMM0)); - MOVSS(fpr.V(dregs[3]), XMM1); - } - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vcmov(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_COMP); - - // D is read as T, prefix included, which only the interpreter does. - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - int tf = (op >> 19) & 1; - int imm3 = (op >> 16) & 7; - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - - for (int i = 0; i < n; ++i) { - // Simplification: Disable if overlap unsafe - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs)) { - DISABLE; - } - } - - // 7 tests a CC bit that is always clear (cpu/vfpu/undefined). - if (imm3 != 6) { - gpr.MapReg(MIPS_REG_VFPUCC, true, false); - fpr.MapRegsV(dregs, sz, MAP_DIRTY); - // Test one bit of CC. This bit decides whether none or all subregisters are copied. - TEST(32, gpr.R(MIPS_REG_VFPUCC), Imm32(1 << imm3)); - FixupBranch skip = J_CC(tf ? CC_NZ : CC_Z, true); - for (int i = 0; i < n; i++) { - MOVSS(fpr.VX(dregs[i]), fpr.V(sregs[i])); - } - SetJumpTarget(skip); - } else { - gpr.MapReg(MIPS_REG_VFPUCC, true, false); - fpr.MapRegsV(dregs, sz, MAP_DIRTY); - // Look at the bottom four bits of CC to individually decide if the subregisters should be copied. - for (int i = 0; i < n; i++) { - TEST(32, gpr.R(MIPS_REG_VFPUCC), Imm32(1 << i)); - FixupBranch skip = J_CC(tf ? CC_NZ : CC_Z, true); - MOVSS(fpr.VX(dregs[i]), fpr.V(sregs[i])); - SetJumpTarget(skip); - } - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VecDo3(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - // The prefix rules here follow the IR frontend, which follows the interpreter and the prefix - // tests in pspautotests. What's not handled goes to the interpreter. - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) - DISABLE; - // vdiv applies the prefixes to its last lane only, from position 0. - if ((op >> 26) == 24 && ((op >> 23) & 7) == 7 && GetVecSize(op) != V_Single && !js.HasNoPrefix()) - DISABLE; - - // Check that we can support the ops, and prepare temporary values for ops that need it. - bool allowSIMD = true; - switch (op >> 26) { - case 24: //VFPU0 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] + t[i]; break; //vadd - case 1: // d[i] = s[i] - t[i]; break; //vsub - case 7: // d[i] = s[i] / t[i]; break; //vdiv - break; - default: - DISABLE; - } - break; - case 25: //VFPU1 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] * t[i]; break; //vmul - break; - default: - DISABLE; - } - break; - case 27: //VFPU3 - switch ((op >> 23) & 7) { - case 2: // vmin - case 3: // vmax - allowSIMD = false; - break; - case 6: // vsge - case 7: // vslt - break; - default: - DISABLE; - } - break; - default: - DISABLE; - break; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], tregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - GetVectorRegsPrefixD(dregs, sz, _VD); - - if (allowSIMD && fpr.TryMapDirtyInInVS(dregs, sz, sregs, sz, tregs, sz)) { - void (XEmitter::*opFunc)(X64Reg, OpArg) = nullptr; - bool symmetric = false; - switch (op >> 26) { - case 24: //VFPU0 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] + t[i]; break; //vadd - opFunc = &XEmitter::ADDPS; - symmetric = true; - break; - case 1: // d[i] = s[i] - t[i]; break; //vsub - opFunc = &XEmitter::SUBPS; - break; - case 7: // d[i] = s[i] / t[i]; break; //vdiv - opFunc = &XEmitter::DIVPS; - break; - } - break; - case 25: //VFPU1 - switch ((op >> 23) & 7) - { - case 0: // d[i] = s[i] * t[i]; break; //vmul - opFunc = &XEmitter::MULPS; - symmetric = true; - break; - } - break; - case 27: //VFPU3 - switch ((op >> 23) & 7) - { - case 2: // vmin - // TODO: Mishandles NaN. Disabled for now. - MOVAPS(XMM1, fpr.VS(sregs)); - MINPS(XMM1, fpr.VS(tregs)); - MOVAPS(fpr.VSX(dregs), R(XMM1)); - break; - case 3: // vmax - // TODO: Mishandles NaN. Disabled for now. - MOVAPS(XMM1, fpr.VS(sregs)); - MAXPS(XMM1, fpr.VS(tregs)); - MOVAPS(fpr.VSX(dregs), R(XMM1)); - break; - case 6: // vsge - MOVAPS(XMM0, fpr.VS(tregs)); - MOVAPS(XMM1, fpr.VS(sregs)); - CMPPS(XMM0, R(XMM1), CMP_ORD); - CMPPS(XMM1, fpr.VS(tregs), CMP_NLT); - - ANDPS(XMM1, R(XMM0)); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - ANDPS(XMM1, MatR(TEMPREG)); - MOVAPS(fpr.VSX(dregs), R(XMM1)); - break; - case 7: // vslt - MOVAPS(XMM1, fpr.VS(sregs)); - CMPPS(XMM1, fpr.VS(tregs), CMP_LT); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - ANDPS(XMM1, MatR(TEMPREG)); - MOVAPS(fpr.VSX(dregs), R(XMM1)); - break; - } - break; - } - - if (opFunc != nullptr) { - if (fpr.VSX(dregs) != fpr.VSX(tregs)) { - if (fpr.VSX(dregs) != fpr.VSX(sregs)) { - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - } - (this->*opFunc)(fpr.VSX(dregs), fpr.VS(tregs)); - } else if (symmetric) { - // We already know d = t. - (this->*opFunc)(fpr.VSX(dregs), fpr.VS(sregs)); - } else { - MOVAPS(XMM1, fpr.VS(sregs)); - (this->*opFunc)(XMM1, fpr.VS(tregs)); - MOVAPS(fpr.VSX(dregs), R(XMM1)); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) - { - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs, n, tregs)) - { - // On 32-bit we only have 6 xregs for mips regs, use XMM0/XMM1 if possible. - // But for vmin/vmax/vsge, we need XMM0/XMM1, so avoid. - if (i < 2 && (op >> 26) != 27) - tempxregs[i] = (X64Reg) (XMM0 + i); - else - { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } - } - else - { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - - for (int i = 0; i < n; ++i) - { - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - } - - const bool minmax = (op >> 26) == 27 && (((op >> 23) & 7) == 2 || ((op >> 23) & 7) == 3); - if (minmax) - gpr.FlushLockX(EDX, ECX); - - for (int i = 0; i < n; ++i) { - switch (op >> 26) { - case 24: //VFPU0 - switch ((op >> 23) & 7) { - case 0: // d[i] = s[i] + t[i]; break; //vadd - ADDSS(tempxregs[i], fpr.V(tregs[i])); - break; - case 1: // d[i] = s[i] - t[i]; break; //vsub - SUBSS(tempxregs[i], fpr.V(tregs[i])); - break; - case 7: // d[i] = s[i] / t[i]; break; //vdiv - DIVSS(tempxregs[i], fpr.V(tregs[i])); - break; - } - break; - case 25: //VFPU1 - switch ((op >> 23) & 7) - { - case 0: // d[i] = s[i] * t[i]; break; //vmul - MULSS(tempxregs[i], fpr.V(tregs[i])); - break; - } - break; - case 27: //VFPU3 - switch ((op >> 23) & 7) - { - case 2: // vmin - case 3: // vmax - { - // Compare by vfpu_minmax_key, select the operand with its bits. A tie takes t. - auto key = [&](X64Reg k, X64Reg x) { - MOVD_xmm(R(TEMPREG), x); - MOV(32, R(k), R(TEMPREG)); - SAR(32, R(k), Imm8(31)); - SHR(32, R(k), Imm8(1)); - XOR(32, R(k), R(TEMPREG)); - // All ones unless the exponent is zero. - AND(32, R(TEMPREG), Imm32(0x7F800000)); - NEG(32, R(TEMPREG)); - SBB(32, R(TEMPREG), R(TEMPREG)); - AND(32, R(k), R(TEMPREG)); - }; - // tempxregs[i] holds s. - MOVSS(XMM0, fpr.V(tregs[i])); - key(EDX, tempxregs[i]); - key(ECX, XMM0); - CMP(32, R(EDX), R(ECX)); - FixupBranch keepS = J_CC(((op >> 23) & 7) == 2 ? CC_L : CC_G); - MOVSS(tempxregs[i], R(XMM0)); - SetJumpTarget(keepS); - } - break; - case 6: // vsge - // We can't just reverse, because of 0/-0. - MOVSS(XMM0, fpr.V(tregs[i])); - MOVSS(XMM1, R(tempxregs[i])); - CMPORDSS(XMM1, R(XMM0)); - CMPNLTSS(tempxregs[i], R(XMM0)); - ANDPS(tempxregs[i], R(XMM1)); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - ANDPS(tempxregs[i], MatR(TEMPREG)); - break; - case 7: // vslt - CMPLTSS(tempxregs[i], fpr.V(tregs[i])); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - ANDPS(tempxregs[i], MatR(TEMPREG)); - break; - } - break; - } - } - - if (minmax) - gpr.UnlockAllX(); - - for (int i = 0; i < n; ++i) - { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -alignas(16) static const u32 vcmpMask[4][4] = { - {0x00000031, 0x00000000, 0x00000000, 0x00000000}, - {0x00000011, 0x00000012, 0x00000000, 0x00000000}, - {0x00000011, 0x00000012, 0x00000014, 0x00000000}, - {0x00000011, 0x00000012, 0x00000014, 0x00000018}, -}; - -void Jit::Comp_Vcmp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_COMP); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || !IsPrefixWithinSize(js.prefixT, op)) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - VCondition cond = (VCondition)(op & 0xF); - - u8 sregs[4], tregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(tregs, sz, _VT); - - // Some, we just fall back to the interpreter. - switch (cond) { - case VC_EI: // c = my_isinf(s[i]); break; - case VC_NI: // c = !my_isinf(s[i]); break; - DISABLE; - break; - case VC_ES: // c = my_isnan(s[i]) || my_isinf(s[i]); break; // Tekken Dark Resurrection - case VC_NS: // c = !my_isnan(s[i]) && !my_isinf(s[i]); break; - case VC_EN: // c = my_isnan(s[i]); break; - case VC_NN: // c = !my_isnan(s[i]); break; - if (_VS != _VT) - DISABLE; - break; - default: - break; - } - - // First, let's get the trivial ones. - - static const int true_bits[4] = {0x31, 0x33, 0x37, 0x3f}; - - if (cond == VC_TR) { - gpr.MapReg(MIPS_REG_VFPUCC, true, true); - OR(32, gpr.R(MIPS_REG_VFPUCC), Imm32(true_bits[n-1])); - return; - } else if (cond == VC_FL) { - gpr.MapReg(MIPS_REG_VFPUCC, true, true); - AND(32, gpr.R(MIPS_REG_VFPUCC), Imm32(~true_bits[n-1])); - return; - } - - if (n > 1) - gpr.FlushLockX(ECX); - - // Start with zero in each lane for the compare to zero. - if (cond == VC_EZ || cond == VC_NZ) { - XORPS(XMM0, R(XMM0)); - if (n > 1) { - XORPS(XMM1, R(XMM1)); - } - } - - bool inverse = false; - - if (cond == VC_GE || cond == VC_GT) { - // We flip, and we need them in regs so we don't clear the high lanes. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.MapRegsV(tregs, sz, 0); - } else { - fpr.SimpleRegsV(tregs, sz, 0); - fpr.MapRegsV(sregs, sz, 0); - } - - // We go backwards because it's more convenient to put things in the right lanes. - int affected_bits = (1 << 4) | (1 << 5); // 4 and 5 - for (int i = n - 1; i >= 0; --i) { - // Alternate between XMM0 and XMM1 - X64Reg reg = i == 1 || i == 3 ? XMM1 : XMM0; - if ((i == 0 || i == 1) && n > 2) { - // We need to swap lanes... this also puts them in the right place. - SHUFPS(reg, R(reg), _MM_SHUFFLE(3, 2, 0, 1)); - } - - // Let's only handle the easy ones, and fall back on the interpreter for the rest. - bool compareTwo = false; - bool compareToZero = false; - int comparison = -1; - bool flip = false; - - switch (cond) { - case VC_ES: - comparison = -1; // We will do the compare at the end. XMM1 will have the bits. - MOVSS(reg, fpr.V(sregs[i])); - break; - - case VC_NS: - comparison = -1; // We will do the compare at the end. XMM1 will have the bits. - MOVSS(reg, fpr.V(sregs[i])); - // Note that we do this all at once at the end. - inverse = true; - break; - - case VC_EN: - comparison = CMP_UNORD; - compareTwo = true; - break; - - case VC_NN: - comparison = CMP_UNORD; - compareTwo = true; - // Note that we do this all at once at the end. - inverse = true; - break; - - case VC_EQ: // c = s[i] == t[i]; break; - comparison = CMP_EQ; - compareTwo = true; - break; - - case VC_LT: // c = s[i] < t[i]; break; - comparison = CMP_LT; - compareTwo = true; - break; - - case VC_LE: // c = s[i] <= t[i]; break; - comparison = CMP_LE; - compareTwo = true; - break; - - case VC_NE: // c = s[i] != t[i]; break; - comparison = CMP_NEQ; - compareTwo = true; - break; - - case VC_GE: // c = s[i] >= t[i]; break; - comparison = CMP_LE; - flip = true; - compareTwo = true; - break; - - case VC_GT: // c = s[i] > t[i]; break; - comparison = CMP_LT; - flip = true; - compareTwo = true; - break; - - case VC_EZ: // c = s[i] == 0.0f || s[i] == -0.0f; break; - comparison = CMP_EQ; - compareToZero = true; - break; - - case VC_NZ: // c = s[i] != 0; break; - comparison = CMP_NEQ; - compareToZero = true; - break; - - default: - DISABLE; - } - - if (comparison != -1) { - if (compareTwo) { - if (!flip) { - MOVSS(reg, fpr.V(sregs[i])); - CMPSS(reg, fpr.V(tregs[i]), comparison); - } else { - MOVSS(reg, fpr.V(tregs[i])); - CMPSS(reg, fpr.V(sregs[i]), comparison); - } - } else if (compareToZero) { - CMPSS(reg, fpr.V(sregs[i]), comparison); - } - } - - affected_bits |= 1 << i; - } - - if (n > 1) { - XOR(32, R(ECX), R(ECX)); - - // This combines them together. - UNPCKLPS(XMM0, R(XMM1)); - - // Finalize the comparison for ES/NS. - if (cond == VC_ES || cond == VC_NS) { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&fourinfnan)); - ANDPS(XMM0, MatR(TEMPREG)); - PCMPEQD(XMM0, MatR(TEMPREG)); // Integer comparison - // It's inversed below for NS. - } - - if (inverse) { - // The canonical way to generate a bunch of ones, see https://stackoverflow.com/questions/35085059/what-are-the-best-instruction-sequences-to-generate-vector-constants-on-the-fly - PCMPEQW(XMM1, R(XMM1)); - XORPS(XMM0, R(XMM1)); - } - MOV(PTRBITS, R(TEMPREG), ImmPtr(&vcmpMask[n - 1])); - ANDPS(XMM0, MatR(TEMPREG)); - MOVAPS(MIPSSTATE_VAR(vcmpResult), XMM0); - - MOV(32, R(TEMPREG), MIPSSTATE_VAR(vcmpResult[0])); - for (int i = 1; i < n; ++i) { - OR(32, R(TEMPREG), MIPSSTATE_VAR_ELEM32(vcmpResult[0], i)); - } - - // Aggregate the bits. Urgh, expensive. Can optimize for the case of one comparison, - // which is the most common after all. - CMP(32, R(TEMPREG), Imm8(affected_bits & 0x1F)); - SETcc(CC_E, R(ECX)); - SHL(32, R(ECX), Imm8(5)); - OR(32, R(TEMPREG), R(ECX)); - } else { - // Finalize the comparison for ES/NS. - if (cond == VC_ES || cond == VC_NS) { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&fourinfnan)); - ANDPS(XMM0, MatR(TEMPREG)); - PCMPEQD(XMM0, MatR(TEMPREG)); // Integer comparison - // It's inversed below for NS. - } - - MOVD_xmm(R(TEMPREG), XMM0); - if (inverse) { - XOR(32, R(TEMPREG), Imm32(0xFFFFFFFF)); - } - AND(32, R(TEMPREG), Imm32(0x31)); - } - - gpr.UnlockAllX(); - gpr.MapReg(MIPS_REG_VFPUCC, true, true); - AND(32, gpr.R(MIPS_REG_VFPUCC), Imm32(~affected_bits)); - OR(32, gpr.R(MIPS_REG_VFPUCC), R(TEMPREG)); - - fpr.ReleaseSpillLocks(); -} - -// There are no immediates for floating point, so we need to load these -// from RAM. Might as well have a table ready. -extern const float mulTableVi2f[32] = { - 1.0f/(1UL<<0),1.0f/(1UL<<1),1.0f/(1UL<<2),1.0f/(1UL<<3), - 1.0f/(1UL<<4),1.0f/(1UL<<5),1.0f/(1UL<<6),1.0f/(1UL<<7), - 1.0f/(1UL<<8),1.0f/(1UL<<9),1.0f/(1UL<<10),1.0f/(1UL<<11), - 1.0f/(1UL<<12),1.0f/(1UL<<13),1.0f/(1UL<<14),1.0f/(1UL<<15), - 1.0f/(1UL<<16),1.0f/(1UL<<17),1.0f/(1UL<<18),1.0f/(1UL<<19), - 1.0f/(1UL<<20),1.0f/(1UL<<21),1.0f/(1UL<<22),1.0f/(1UL<<23), - 1.0f/(1UL<<24),1.0f/(1UL<<25),1.0f/(1UL<<26),1.0f/(1UL<<27), - 1.0f/(1UL<<28),1.0f/(1UL<<29),1.0f/(1UL<<30),1.0f/(1UL<<31), -}; - -void Jit::Comp_Vi2f(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - int imm = (op >> 16) & 0x1f; - const float *mult = &mulTableVi2f[imm]; - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - int tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - if (*mult != 1.0f) { - if (RipAccessible(mult)) { - MOVSS(XMM1, M(mult)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(mult)); - MOVSS(XMM1, MatR(TEMPREG)); - } - } - for (int i = 0; i < n; i++) { - fpr.MapRegV(tempregs[i], sregs[i] == dregs[i] ? MAP_DIRTY : MAP_NOINIT); - if (fpr.V(sregs[i]).IsSimpleReg()) { - CVTDQ2PS(fpr.VX(tempregs[i]), fpr.V(sregs[i])); - } else { - MOVSS(fpr.VX(tempregs[i]), fpr.V(sregs[i])); - CVTDQ2PS(fpr.VX(tempregs[i]), R(fpr.VX(tempregs[i]))); - } - if (*mult != 1.0f) - MULSS(fpr.VX(tempregs[i]), R(XMM1)); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - MOVSS(fpr.VX(dregs[i]), fpr.V(tempregs[i])); - } - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); -} - -// Planning for true SIMD - -// Sequence for gathering sparse registers into one SIMD: -// MOVSS(XMM0, fpr.R(sregs[0])); -// MOVSS(XMM1, fpr.R(sregs[1])); -// MOVSS(XMM2, fpr.R(sregs[2])); -// MOVSS(XMM3, fpr.R(sregs[3])); -// SHUFPS(XMM0, R(XMM1), _MM_SHUFFLE(0, 0, 0, 0)); // XMM0 = S1 S1 S0 S0 -// SHUFPS(XMM2, R(XMM3), _MM_SHUFFLE(0, 0, 0, 0)); // XMM2 = S3 S3 S2 S2 -// SHUFPS(XMM0, R(XMM2), _MM_SHUFFLE(2, 0, 2, 0)); // XMM0 = S3 S2 S1 S0 -// Some punpckwd etc would also work. -// Alternatively, MOVSS and three PINSRD (SSE4) with mem source. -// Why PINSRD instead of INSERTPS? -// http://software.intel.com/en-us/blogs/2009/01/07/using-sse41-for-mp3-encoding-quantization - -// Sequence for scattering a SIMD register to sparse registers: -// (Very serial though, better methods may be possible) -// MOVSS(fpr.R(sregs[0]), XMM0); -// SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 3, 2, 1)); -// MOVSS(fpr.R(sregs[1]), XMM0); -// SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 3, 2, 1)); -// MOVSS(fpr.R(sregs[2]), XMM0); -// SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(3, 3, 2, 1)); -// MOVSS(fpr.R(sregs[3]), XMM0); -// On SSE4 we should use EXTRACTPS. - -// Translation of ryg's half_to_float5_SSE2 -void Jit::Comp_Vh2f(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - -#define SSE_CONST4(name, val) alignas(16) static const u32 name[4] = { (val), (val), (val), (val) } - - SSE_CONST4(mask_nosign, 0x7fff); - SSE_CONST4(nan_mantissa, 0x800003ff); - SSE_CONST4(magic, (254 - 15) << 23); - SSE_CONST4(was_infnan, 0x7bff); - SSE_CONST4(exp_infnan, 255 << 23); - SSE_CONST4(max_subnormal, 0x03ff); - SSE_CONST4(sign_only, 0x80000000); - - OpArg mask_nosign_arg, nan_mantissa_arg, magic_arg, was_infnan_arg, exp_infnan_arg, max_subnormal_arg, sign_only_arg; - if (RipAccessible(mask_nosign)) { - mask_nosign_arg = M(&mask_nosign[0]); - nan_mantissa_arg = M(&nan_mantissa[0]); - magic_arg = M(&magic[0]); - was_infnan_arg = M(&was_infnan[0]); - exp_infnan_arg = M(&exp_infnan[0]); - max_subnormal_arg = M(&max_subnormal[0]); - sign_only_arg = M(&sign_only[0]); - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&mask_nosign[0])); - mask_nosign_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &mask_nosign[0]); - nan_mantissa_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &nan_mantissa[0]); - magic_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &magic[0]); - was_infnan_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &was_infnan[0]); - exp_infnan_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &exp_infnan[0]); - max_subnormal_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &max_subnormal[0]); - sign_only_arg = MAccessibleDisp(TEMPREG, &mask_nosign[0], &sign_only[0]); - } - -#undef SSE_CONST4 - VectorSize sz = GetVecSize(op); - VectorSize outsize; - switch (sz) { - case V_Single: - outsize = V_Pair; - break; - case V_Pair: - outsize = V_Quad; - break; - default: - DISABLE; - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, outsize, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - - // Force ourselves two extra xregs as temp space. - X64Reg temps[2]; - fpr.GetFreeXRegs(temps, 2); - X64Reg tempR = temps[0]; - X64Reg keepMask = temps[1]; - - MOVSS(XMM0, fpr.V(sregs[0])); - if (sz != V_Single) { - MOVSS(XMM1, fpr.V(sregs[1])); - PUNPCKLDQ(XMM0, R(XMM1)); - } - XORPS(XMM1, R(XMM1)); - PUNPCKLWD(XMM0, R(XMM1)); - - // OK, 16 bits in each word. - // Let's go. Deep magic here. - MOVAPS(XMM1, R(XMM0)); - ANDPS(XMM0, mask_nosign_arg); // xmm0 = expmant - XORPS(XMM1, R(XMM0)); // xmm1 = justsign = expmant ^ xmm0 - MOVAPS(tempR, R(XMM0)); - // The hardware flushes subnormal halves to a signed zero: keep only the sign where the - // exponent is zero. - MOVAPS(keepMask, R(XMM0)); - PCMPGTD(keepMask, max_subnormal_arg); // keepMask = exponent != 0 - ORPS(keepMask, sign_only_arg); - PSLLD(XMM0, 13); - MULPS(XMM0, magic_arg); /// xmm0 = scaled - PSLLD(XMM1, 16); // xmm1 = sign - ORPS(XMM0, R(XMM1)); - - // Now create a NAN mask, adding in the sign. - ORPS(XMM1, R(tempR)); // xmm1 = sign + original mantissa. - ANDPS(XMM1, nan_mantissa_arg); // xmm1 = original mantissa - PCMPGTD(tempR, was_infnan_arg); // xmm2 = b_wasinfnan - ORPS(XMM1, exp_infnan_arg); // xmm1 = infnan result - ANDPS(XMM1, R(tempR)); // xmm1 = infnan result OR zero if not infnan - ANDNPS(tempR, R(XMM0)); // tempR = result OR zero if infnan - ORPS(XMM1, R(tempR)); - ANDPS(XMM1, R(keepMask)); - - fpr.MapRegsV(dregs, outsize, MAP_NOINIT | MAP_DIRTY); - - // TODO: Could apply D-prefix in parallel here... - - MOVSS(fpr.V(dregs[0]), XMM1); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 3, 2, 1)); - MOVSS(fpr.V(dregs[1]), XMM1); - - if (sz != V_Single) { - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 3, 2, 1)); - MOVSS(fpr.V(dregs[2]), XMM1); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 3, 2, 1)); - MOVSS(fpr.V(dregs[3]), XMM1); - } - - ApplyPrefixD(dregs, outsize); - gpr.UnlockAllX(); - fpr.ReleaseSpillLocks(); -} - -// The goal is to map (reversed byte order for clarity): -// AABBCCDD -> 000000AA 000000BB 000000CC 000000DD -alignas(16) static s8 vc2i_shuffle[16] = { -1, -1, -1, 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3 }; -// AABBCCDD -> AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD -alignas(16) static s8 vuc2i_shuffle[16] = { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 }; - -void Jit::Comp_Vx2i(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - int bits = ((op >> 16) & 2) == 0 ? 8 : 16; // vuc2i/vc2i (0/1), vus2i/vs2i (2/3) - bool unsignedOp = ((op >> 16) & 1) == 0; // vuc2i (0), vus2i (2) - - // vs2i or vus2i unpack pairs of 16-bit integers into 32-bit integers, with the values - // at the top. vus2i shifts it an extra bit right afterward. - // vc2i and vuc2i unpack quads of 8-bit integers into 32-bit integers, with the values - // at the top too. vuc2i is a bit special (see below.) - // Let's do this similarly as h2f - we do a solution that works for both singles and pairs - // then use it for both. - - VectorSize sz = GetVecSize(op); - VectorSize outsize; - if (bits == 8) { - outsize = V_Quad; - } else { - switch (sz) { - case V_Single: - outsize = V_Pair; - break; - case V_Pair: - outsize = V_Quad; - break; - default: - DISABLE; - } - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, outsize, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - - if (bits == 16) { - MOVSS(XMM1, fpr.V(sregs[0])); - if (sz != V_Single) { - MOVSS(XMM0, fpr.V(sregs[1])); - PUNPCKLDQ(XMM1, R(XMM0)); - } - - // Unpack 16-bit words into 32-bit words, upper position, and we're done! - PXOR(XMM0, R(XMM0)); - PUNPCKLWD(XMM0, R(XMM1)); - } else if (bits == 8) { - if (unsignedOp) { - // vuc2i is a bit special. It spreads out the bits like this: - // s[0] = 0xDDCCBBAA -> d[0] = (0xAAAAAAAA >> 1), d[1] = (0xBBBBBBBB >> 1), etc. - MOVSS(XMM0, fpr.V(sregs[0])); - if (cpu_info.bSSSE3 && RipAccessible(vuc2i_shuffle)) { - // Not really different speed. Generates a bit less code. - PSHUFB(XMM0, M(&vuc2i_shuffle[0])); // rip accessible - } else { - // First, we change 0xDDCCBBAA to 0xDDDDCCCCBBBBAAAA. - PUNPCKLBW(XMM0, R(XMM0)); - // Now, interleave each 16 bits so they're all 32 bits wide. - PUNPCKLWD(XMM0, R(XMM0)); - } - } else { - if (cpu_info.bSSSE3 && RipAccessible(vc2i_shuffle)) { - MOVSS(XMM0, fpr.V(sregs[0])); - PSHUFB(XMM0, M(&vc2i_shuffle[0])); - } else { - PXOR(XMM1, R(XMM1)); - MOVSS(XMM0, fpr.V(sregs[0])); - PUNPCKLBW(XMM1, R(XMM0)); - PXOR(XMM0, R(XMM0)); - PUNPCKLWD(XMM0, R(XMM1)); - } - } - } - - // At this point we have the regs in the 4 lanes. - // In the "u" mode, we need to shift it out of the sign bit. - if (unsignedOp) { - PSRLD(XMM0, 1); - } - - if (fpr.TryMapRegsVS(dregs, outsize, MAP_NOINIT | MAP_DIRTY)) { - MOVAPS(fpr.VSX(dregs), R(XMM0)); - } else { - // Done! TODO: The rest of this should be possible to extract into a function. - fpr.MapRegsV(dregs, outsize, MAP_NOINIT | MAP_DIRTY); - - // TODO: Could apply D-prefix in parallel here... - - MOVSS(fpr.V(dregs[0]), XMM0); - PSRLDQ(XMM0, 4); - MOVSS(fpr.V(dregs[1]), XMM0); - - if (outsize != V_Pair) { - PSRLDQ(XMM0, 4); - MOVSS(fpr.V(dregs[2]), XMM0); - PSRLDQ(XMM0, 4); - MOVSS(fpr.V(dregs[3]), XMM0); - } - } - - ApplyPrefixD(dregs, outsize); - gpr.UnlockAllX(); - fpr.ReleaseSpillLocks(); -} - -extern const double mulTableVf2i[32] = { - (1ULL<<0),(1ULL<<1),(1ULL<<2),(1ULL<<3), - (1ULL<<4),(1ULL<<5),(1ULL<<6),(1ULL<<7), - (1ULL<<8),(1ULL<<9),(1ULL<<10),(1ULL<<11), - (1ULL<<12),(1ULL<<13),(1ULL<<14),(1ULL<<15), - (1ULL<<16),(1ULL<<17),(1ULL<<18),(1ULL<<19), - (1ULL<<20),(1ULL<<21),(1ULL<<22),(1ULL<<23), - (1ULL<<24),(1ULL<<25),(1ULL<<26),(1ULL<<27), - (1ULL<<28),(1ULL<<29),(1ULL<<30),(1ULL<<31), -}; - -static const double maxMinIntAsDouble[2] = { (double)0x7fffffff, (double)(int)0x80000000 }; // that's not equal to 0x80000000 - -void Jit::Comp_Vf2i(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || (js.prefixD & 0xFF) != 0) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - int imm = (op >> 16) & 0x1f; - const double *mult = &mulTableVf2i[imm]; - - int setMXCSR = -1; - int rmode = (op >> 21) & 0x1f; - switch (rmode) { - case 17: - break; //z - truncate. Easy to support. - case 16: - setMXCSR = 0; - break; - case 18: - setMXCSR = 2; - break; - case 19: - setMXCSR = 1; - break; - } - - // Small optimization: 0 is our default mode anyway. - if (setMXCSR == 0 && !js.hasSetRounding) { - setMXCSR = -1; - } - // Except for truncate, we need to update MXCSR to our preferred rounding mode. - if (setMXCSR != -1) { - STMXCSR(MIPSSTATE_VAR(mxcsrTemp)); - MOV(32, R(TEMPREG), MIPSSTATE_VAR(mxcsrTemp)); - AND(32, R(TEMPREG), Imm32(~(3 << 13))); - if (setMXCSR != 0) { - OR(32, R(TEMPREG), Imm32(setMXCSR << 13)); - } - MOV(32, MIPSSTATE_VAR(temp), R(TEMPREG)); - LDMXCSR(MIPSSTATE_VAR(temp)); - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Really tricky to SIMD due to double precision requirement... - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_DIRTY | MAP_NOINIT); - - u8 tempregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - tempregs[i] = fpr.GetTempV(); - } else { - tempregs[i] = dregs[i]; - } - } - - if (*mult != 1.0f) { - if (RipAccessible(mult)) { - MOVSD(XMM1, M(mult)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(mult)); - MOVSD(XMM1, MatR(TEMPREG)); - } - } - - fpr.MapRegsV(tempregs, sz, MAP_DIRTY | MAP_NOINIT); - for (int i = 0; i < n; i++) { - // Need to do this in double precision to clamp correctly as float - // doesn't have enough precision to represent 0x7fffffff for example exactly. - MOVSS(XMM0, fpr.V(sregs[i])); - CVTSS2SD(XMM0, R(XMM0)); // convert to double precision - if (*mult != 1.0f) { - MULSD(XMM0, R(XMM1)); - } - MOV(PTRBITS, R(TEMPREG), ImmPtr(maxMinIntAsDouble)); - MINSD(XMM0, MDisp(TEMPREG, 0)); - MAXSD(XMM0, MDisp(TEMPREG, sizeof(double))); - // We've set the rounding mode above, so this part's easy. - switch ((op >> 21) & 0x1f) { - case 16: CVTSD2SI(TEMPREG, R(XMM0)); break; //n - case 17: CVTTSD2SI(TEMPREG, R(XMM0)); break; //z - truncate - case 18: CVTSD2SI(TEMPREG, R(XMM0)); break; //u - case 19: CVTSD2SI(TEMPREG, R(XMM0)); break; //d - } - MOVD_xmm(fpr.VX(tempregs[i]), R(TEMPREG)); - } - - for (int i = 0; i < n; ++i) { - if (dregs[i] != tempregs[i]) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - MOVSS(fpr.VX(dregs[i]), fpr.V(tempregs[i])); - fpr.DiscardV(tempregs[i]); - } - } - - if (setMXCSR != -1) { - LDMXCSR(MIPSSTATE_VAR(mxcsrTemp)); - } - - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vcst(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - if (js.HasUnknownPrefix()) - DISABLE; - - int conNum = (op >> 16) & 0x1f; - int vd = _VD; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 dregs[4]; - GetVectorRegsPrefixD(dregs, sz, vd); - - if (RipAccessible(cst_constants)) { - MOVSS(XMM0, M(&cst_constants[conNum])); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&cst_constants[conNum])); - MOVSS(XMM0, MatR(TEMPREG)); - } - - // The SIMD path below skips ApplyPrefixD, so only take it without a D prefix. - if (!js.HasDPrefix() && fpr.TryMapRegsVS(dregs, sz, MAP_NOINIT | MAP_DIRTY)) { - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(0,0,0,0)); - MOVAPS(fpr.VS(dregs), XMM0); - fpr.ReleaseSpillLocks(); - return; - } - - fpr.MapRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - for (int i = 0; i < n; i++) { - MOVSS(fpr.V(dregs[i]), XMM0); - } - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vsgn(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs)) { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } else { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - - // Would be nice with more temp regs here so we could put signBitLower and oneOneOneOne into regs... - for (int i = 0; i < n; ++i) { - // A denormal signs as zero like on the hardware, so test the exponent field, not the value. - MOVSS(XMM1, fpr.V(sregs[i])); - MOVAPS(tempxregs[i], R(XMM1)); - if (RipAccessible(fourinfnan)) { - ANDPS(tempxregs[i], M(&fourinfnan)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&fourinfnan)); - ANDPS(tempxregs[i], MatR(TEMPREG)); - } - XORPS(XMM0, R(XMM0)); - CMPEQSS(XMM0, R(tempxregs[i])); // XMM0 = exponent of s[i] == 0 - // Preserve sign bit, replace rest with ones - if (RipAccessible(signBitLower)) { - ANDPS(XMM1, M(&signBitLower)); // rip accessible - ORPS(XMM1, M(&oneOneOneOne)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&signBitLower)); - ANDPS(XMM1, MatR(TEMPREG)); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - ORPS(XMM1, MatR(TEMPREG)); - } - // If really was equal to zero, zap. Note that ANDN negates the destination. - ANDNPS(XMM0, R(XMM1)); - MOVAPS(tempxregs[i], R(XMM0)); - } - - for (int i = 0; i < n; ++i) { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vocp(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix() || (js.prefixS & VFPU_NEGATE(1, 1, 1, 1)) != 0) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - // This is a hack that modifies prefixes. We eat them later, so just overwrite. - // S prefix forces the negate flags. - js.prefixS |= 0x000F0000; - // T prefix forces constants on and regnum to 1. - // That means negate still works, and abs activates a different constant. - js.prefixT = (js.prefixT & ~0x000000FF) | 0x00000055 | 0x0000F000; - - u8 sregs[4], tregs[4], dregs[4]; - // Actually uses the T prefixes (despite being VS.) - GetVectorRegsPrefixS(sregs, sz, _VS); - if (js.prefixT != 0x0000F055) - GetVectorRegsPrefixT(tregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - if (js.prefixT != 0x0000F055) - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs)) { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } else { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - - if (js.prefixT == 0x0000F055) { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(XMM1, MatR(TEMPREG)); - } - for (int i = 0; i < n; ++i) { - if (js.prefixT == 0x0000F055) { - MOVSS(XMM0, R(XMM1)); - } else { - MOVSS(XMM0, fpr.V(tregs[i])); - } - ADDSS(XMM0, fpr.V(sregs[i])); - MOVSS(tempxregs[i], R(XMM0)); - } - - for (int i = 0; i < n; ++i) { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vbfy(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix() || (js.prefixS & VFPU_NEGATE(1, 1, 1, 1)) != 0) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - if (n != 2 && n != 4) { - DISABLE; - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) { - if (!IsOverlapSafe(dregs[i], i, n, sregs)) { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } else { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - - int subop = (op >> 16) & 0x1F; - if (subop == 3) { - // vbfy2 - MOVSS(tempxregs[0], fpr.V(sregs[0])); - MOVSS(tempxregs[1], fpr.V(sregs[1])); - MOVSS(tempxregs[2], fpr.V(sregs[0])); - MOVSS(tempxregs[3], fpr.V(sregs[1])); - ADDSS(tempxregs[0], fpr.V(sregs[2])); - ADDSS(tempxregs[1], fpr.V(sregs[3])); - SUBSS(tempxregs[2], fpr.V(sregs[2])); - SUBSS(tempxregs[3], fpr.V(sregs[3])); - } else if (subop == 2) { - // vbfy1 - MOVSS(tempxregs[0], fpr.V(sregs[0])); - MOVSS(tempxregs[1], fpr.V(sregs[0])); - ADDSS(tempxregs[0], fpr.V(sregs[1])); - SUBSS(tempxregs[1], fpr.V(sregs[1])); - if (n == 4) { - MOVSS(tempxregs[2], fpr.V(sregs[2])); - MOVSS(tempxregs[3], fpr.V(sregs[2])); - ADDSS(tempxregs[2], fpr.V(sregs[3])); - SUBSS(tempxregs[3], fpr.V(sregs[3])); - } - } else { - DISABLE; - } - - for (int i = 0; i < n; ++i) { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -union u32float { - u32 u; - float f; - - operator float() const { - return f; - } - - inline u32float &operator *=(const float &other) { - f *= other; - return *this; - } -}; - -#if PPSSPP_ARCH(AMD64) -typedef float SinCosArg; -#else -typedef u32float SinCosArg; -#endif - -void SinCos(SinCosArg angle, float *output) { - vfpu_sincos(angle, output[0], output[1]); -} - -void SinOnly(SinCosArg angle, float *output) { - output[0] = vfpu_sin(angle); -} - -void NegSinOnly(SinCosArg angle, float *output) { - output[0] = -vfpu_sin(angle); -} - -void CosOnly(SinCosArg angle, float *output) { - output[1] = vfpu_cos(angle); -} - -void ASinScaled(SinCosArg sine, float *output) { - output[0] = vfpu_asin(sine); -} - -void SinCosNegSin(SinCosArg angle, float *output) { - vfpu_sincos(angle, output[0], output[1]); - output[0] = -output[0]; -} - -void VSqrt(SinCosArg arg, float *output) { - output[0] = vfpu_sqrt(arg); -} - -void VRSqrt(SinCosArg arg, float *output) { - output[0] = vfpu_rsqrt(arg); -} - -void VRcp(SinCosArg arg, float *output) { - output[0] = vfpu_rcp(arg); -} - -void VNRcp(SinCosArg arg, float *output) { - output[0] = -vfpu_rcp(arg); -} - -void Exp2(SinCosArg arg, float *output) { - output[0] = vfpu_exp2(arg); -} - -void Log2(SinCosArg arg, float *output) { - output[0] = vfpu_log2(arg); -} - -void RExp2(SinCosArg arg, float *output) { - output[0] = vfpu_rexp2(arg); -} - -#if PPSSPP_ARCH(AMD64) -static float NegRcp(float x) { - return -vfpu_rcp(x); -} - -static float NegSin(float x) { - return -vfpu_sin(x); -} - -// The VV2Op math functions, called with CallProtectedLeaf. They take and return their float in XMM0. -static float (*VV2OpMathFunc(int optype))(float) { - switch (optype) { - case 16: return &vfpu_rcp; - case 17: return &vfpu_rsqrt; - case 18: return &vfpu_sin; - case 19: return &vfpu_cos; - case 20: return &vfpu_exp2; - case 21: return &vfpu_log2; - case 22: return &vfpu_sqrt; - case 23: return &vfpu_asin; - case 24: return &NegRcp; - case 26: return &NegSin; - case 28: return &vfpu_rexp2; - default: return nullptr; - } -} -#endif - -void Jit::Comp_VV2Op(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix()) - DISABLE; - { - int optype = (op >> 16) & 0x1f; - if (optype == 0) { - if (!IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - } else if (optype == 1 || optype == 2) { - // vabs and vneg are vmov with the abs/negate bit forced, so a negate in the S prefix - // doesn't negate twice. D prefix is fine for these, and used sometimes. - if (js.HasSPrefix()) - DISABLE; - } else if (optype == 5 && js.HasDPrefix()) { - // vsat1 doesn't apply the D saturation. - DISABLE; - } - if (optype >= 16 && !js.HasNoPrefix()) { - // These apply the S and D prefixes to their last lane only, from prefix position 0. - // That's the whole vector for a single, so only that case is handled here. - if (GetVecSize(op) != V_Single) - DISABLE; - if (!IsPrefixWithinSize(js.prefixS, op)) - DISABLE; - // The negative ones seem to use negate flags as a prefix hack. - if (optype >= 24 && (js.prefixS & 0x000F0000) != 0) - DISABLE; - } - } - - auto specialFuncCallHelper = [this](void (*specialFunc)(SinCosArg, float *output), u8 sreg) { -#if PPSSPP_ARCH(AMD64) - MOVSS(XMM0, fpr.V(sreg)); - // TODO: This reg might be different on Linux... -#ifdef _WIN32 - LEA(64, RDX, MIPSSTATE_VAR(sincostemp[0])); -#else - LEA(64, RDI, MIPSSTATE_VAR(sincostemp[0])); -#endif - ABI_CallFunction(thunks.ProtectFunction((const void *)specialFunc, 0)); -#else - // Sigh, passing floats with cdecl isn't pretty, ends up on the stack. - if (fpr.V(sreg).IsSimpleReg()) { - MOVD_xmm(R(EAX), fpr.VX(sreg)); - } else { - MOV(32, R(EAX), fpr.V(sreg)); - } - CallProtectedFunction((const void *)specialFunc, R(EAX), Imm32((uint32_t)(uintptr_t)&mips_->sincostemp[0])); -#endif - }; - - // Pre-processing: Eliminate silly no-op VMOVs, common in Wipeout Pure - if (((op >> 16) & 0x1f) == 0 && _VS == _VD && js.HasNoPrefix()) { - return; - } - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, sz, _VD); - - bool canSIMD = false; - // Some can be SIMD'd. - switch ((op >> 16) & 0x1f) { - case 0: // vmov - case 1: // vabs - case 2: // vneg - canSIMD = true; - break; - } - - if (canSIMD && fpr.TryMapDirtyInVS(dregs, sz, sregs, sz)) { - switch ((op >> 16) & 0x1f) { - case 0: // vmov - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - break; - case 1: // vabs - if (dregs[0] != sregs[0]) - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - if (RipAccessible(&noSignMask)) { - ANDPS(fpr.VSX(dregs), M(&noSignMask)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&noSignMask)); - ANDPS(fpr.VSX(dregs), MatR(TEMPREG)); - } - break; - case 2: // vneg - if (dregs[0] != sregs[0]) - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - if (RipAccessible(&signBitAll)) { - XORPS(fpr.VSX(dregs), M(&signBitAll)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&signBitAll)); - XORPS(fpr.VSX(dregs), MatR(TEMPREG)); - } - break; - } - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) - { - if (!IsOverlapSafeAllowS(dregs[i], i, n, sregs)) - { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } - else - { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - -#if PPSSPP_ARCH(AMD64) - float (*mathFunc)(float) = VV2OpMathFunc((op >> 16) & 0x1f); -#endif - - // Warning: sregs[i] and tempxregs[i] may be the same reg. - // Helps for vmov, hurts for vrcp, etc. - for (int i = 0; i < n; ++i) - { -#if PPSSPP_ARCH(AMD64) - if (mathFunc) { - MOVSS(XMM0, fpr.V(sregs[i])); - CallProtectedLeaf((const void *)mathFunc); - MOVSS(tempxregs[i], R(XMM0)); - continue; - } -#endif - switch ((op >> 16) & 0x1f) - { - case 0: // d[i] = s[i]; break; //vmov - // Probably for swizzle. - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - break; - case 1: // d[i] = fabsf(s[i]); break; //vabs - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - if (RipAccessible(&noSignMask)) { - ANDPS(tempxregs[i], M(&noSignMask)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&noSignMask)); - ANDPS(tempxregs[i], MatR(TEMPREG)); - } - break; - case 2: // d[i] = -s[i]; break; //vneg - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - if (RipAccessible(&signBitLower)) { - XORPS(tempxregs[i], M(&signBitLower)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&signBitLower)); - XORPS(tempxregs[i], MatR(TEMPREG)); - } - break; - case 4: // if (s[i] < 0) d[i] = 0; else {if(s[i] > 1.0f) d[i] = 1.0f; else d[i] = s[i];} break; // vsat0 - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - - // Zero out XMM0 if it was <= +0.0f (but skip NAN.) - MOVSS(R(XMM0), tempxregs[i]); - XORPS(XMM1, R(XMM1)); - CMPLESS(XMM0, R(XMM1)); - ANDNPS(XMM0, R(tempxregs[i])); - - // Retain a NAN in XMM0 (must be second operand.) - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(tempxregs[i], MatR(TEMPREG)); - MINSS(tempxregs[i], R(XMM0)); - break; - case 5: // if (s[i] < -1.0f) d[i] = -1.0f; else {if(s[i] > 1.0f) d[i] = 1.0f; else d[i] = s[i];} break; // vsat1 - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - - // Check for < -1.0f, but careful of NANs. - MOV(PTRBITS, R(TEMPREG), ImmPtr(&minus_one)); - MOVSS(XMM1, MatR(TEMPREG)); - MOVSS(R(XMM0), tempxregs[i]); - CMPLESS(XMM0, R(XMM1)); - // If it was NOT less, the three ops below do nothing. - // Otherwise, they replace the value with -1.0f. - ANDPS(XMM1, R(XMM0)); - ANDNPS(XMM0, R(tempxregs[i])); - ORPS(XMM0, R(XMM1)); - - // Retain a NAN in XMM0 (must be second operand.) - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(tempxregs[i], MatR(TEMPREG)); - MINSS(tempxregs[i], R(XMM0)); - break; - case 16: // d[i] = 1.0f / s[i]; break; //vrcp - specialFuncCallHelper(&VRcp, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 17: // d[i] = 1.0f / sqrtf(s[i]); break; //vrsq - specialFuncCallHelper(&VRSqrt, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 18: // d[i] = sinf((float)M_PI_2 * s[i]); break; //vsin - specialFuncCallHelper(&SinOnly, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 19: // d[i] = cosf((float)M_PI_2 * s[i]); break; //vcos - specialFuncCallHelper(&CosOnly, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[1])); - break; - case 20: // d[i] = powf(2.0f, s[i]); break; //vexp2 - specialFuncCallHelper(&Exp2, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 21: // d[i] = logf(s[i])/log(2.0f); break; //vlog2 - specialFuncCallHelper(&Log2, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 22: // d[i] = sqrtf(s[i]); break; //vsqrt - specialFuncCallHelper(&VSqrt, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 23: // d[i] = asinf(s[i]) / M_PI_2; break; //vasin - specialFuncCallHelper(&ASinScaled, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 24: // d[i] = -1.0f / s[i]; break; // vnrcp - specialFuncCallHelper(&VNRcp, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 26: // d[i] = -sinf((float)M_PI_2 * s[i]); break; // vnsin - specialFuncCallHelper(&NegSinOnly, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - case 28: // d[i] = 1.0f / expf(s[i] * (float)M_LOG2E); break; // vrexp2 - specialFuncCallHelper(&RExp2, sregs[i]); - MOVSS(tempxregs[i], MIPSSTATE_VAR(sincostemp[0])); - break; - } - } - for (int i = 0; i < n; ++i) - { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Mftv(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - int imm = op & 0xFF; - MIPSGPReg rt = _RT; - switch ((op >> 21) & 0x1f) - { - case 3: //mfv / mfvc - // rt = 0, imm = 255 appears to be used as a CPU interlock by some games. - if (rt != MIPS_REG_ZERO) { - if (imm < 128) { //R(rt) = VI(imm); - fpr.SimpleRegV(imm, 0); - if (fpr.V(imm).IsSimpleReg()) { - fpr.MapRegV(imm, 0); - gpr.MapReg(rt, false, true); - MOVD_xmm(gpr.R(rt), fpr.VX(imm)); - } else { - // Let's not bother mapping the vreg. - gpr.MapReg(rt, false, true); - MOV(32, gpr.R(rt), fpr.V(imm)); - } - } else if (imm < 128 + VFPU_CTRL_MAX) { //mfvc - if (imm - 128 == VFPU_CTRL_CC) { - if (gpr.IsImm(MIPS_REG_VFPUCC)) { - gpr.SetImm(rt, gpr.GetImm(MIPS_REG_VFPUCC)); - } else { - gpr.Lock(rt, MIPS_REG_VFPUCC); - gpr.MapReg(rt, false, true); - gpr.MapReg(MIPS_REG_VFPUCC, true, false); - MOV(32, gpr.R(rt), gpr.R(MIPS_REG_VFPUCC)); - gpr.UnlockAll(); - } - } else { - // In case we have a saved prefix. - FlushPrefixV(); - gpr.MapReg(rt, false, true); - MOV(32, gpr.R(rt), MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm - 128)); - } - } else { - //ERROR - maybe need to make this value too an "interlock" value? - _dbg_assert_msg_(false,"mfv - invalid register"); - } - } - break; - - case 7: //mtv - if (imm < 128) { // VI(imm) = R(rt); - fpr.MapRegV(imm, MAP_DIRTY | MAP_NOINIT); - // Let's not bother mapping rt if we don't have to. - if (gpr.IsImm(rt) && gpr.GetImm(rt) == 0) { - XORPS(fpr.VX(imm), fpr.V(imm)); - } else { - gpr.KillImmediate(rt, true, false); - MOVD_xmm(fpr.VX(imm), gpr.R(rt)); - } - } else if (imm < 128 + VFPU_CTRL_MAX) { //mtvc //currentMIPS->vfpuCtrl[imm - 128] = R(rt); - // Only some of the bits stick: six for CC, the low 20 of a prefix, and the RNG state - // keeps 0x3F8 on top (cpu/vfpu/prefix_ctrl, cpu/vfpu/vrnd). Same as the IR does it. - u32 mask; - u32 setBits = GetVFPUCtrlSetBits(imm - 128); - if (imm - 128 == VFPU_CTRL_CC) { - if (gpr.IsImm(rt)) { - gpr.SetImm(MIPS_REG_VFPUCC, gpr.GetImm(rt) & 0x3F); - } else { - gpr.Lock(rt, MIPS_REG_VFPUCC); - gpr.MapReg(rt, true, false); - gpr.MapReg(MIPS_REG_VFPUCC, false, true); - MOV(32, gpr.R(MIPS_REG_VFPUCC), gpr.R(rt)); - AND(32, gpr.R(MIPS_REG_VFPUCC), Imm32(0x3F)); - gpr.UnlockAll(); - } - } else if (!GetVFPUCtrlMask(imm - 128, &mask)) { - // Read-only or unknown register: nothing is written. - } else { - gpr.MapReg(rt, true, false); - MOV(32, MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm - 128), gpr.R(rt)); - if (mask != 0xFFFFFFFF) - AND(32, MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm - 128), Imm32(mask)); - if (setBits != 0) - OR(32, MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm - 128), Imm32(setBits)); - } - - // TODO: Optimization if rt is Imm? - if (imm - 128 == VFPU_CTRL_SPREFIX) { - js.prefixSFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm - 128 == VFPU_CTRL_TPREFIX) { - js.prefixTFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm - 128 == VFPU_CTRL_DPREFIX) { - js.prefixDFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } - } else { - //ERROR - _dbg_assert_msg_(false,"mtv - invalid register"); - } - break; - - default: - DISABLE; - } -} - -void Jit::Comp_Vmfvc(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - int vd = _VD; - int imm = (op >> 8) & 0x7F; - if (imm < VFPU_CTRL_MAX) { - fpr.MapRegV(vd, MAP_DIRTY | MAP_NOINIT); - if (imm == VFPU_CTRL_CC) { - gpr.MapReg(MIPS_REG_VFPUCC, true, false); - MOVD_xmm(fpr.VX(vd), gpr.R(MIPS_REG_VFPUCC)); - } else { - // In case we have a saved prefix. - FlushPrefixV(); - MOVSS(fpr.VX(vd), MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm)); - } - fpr.ReleaseSpillLocks(); - } else { - fpr.MapRegV(vd, MAP_DIRTY | MAP_NOINIT); - XORPS(fpr.VX(vd), fpr.V(vd)); - fpr.ReleaseSpillLocks(); - } -} - -void Jit::Comp_Vmtvc(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - int vs = _VS; - int imm = op & 0x7F; - if (imm < VFPU_CTRL_MAX) { - u32 mask; - u32 setBits = GetVFPUCtrlSetBits(imm); - fpr.MapRegV(vs, 0); - if (imm == VFPU_CTRL_CC) { - gpr.MapReg(MIPS_REG_VFPUCC, false, true); - MOVD_xmm(gpr.R(MIPS_REG_VFPUCC), fpr.VX(vs)); - AND(32, gpr.R(MIPS_REG_VFPUCC), Imm32(0x3F)); - } else if (GetVFPUCtrlMask(imm, &mask)) { - MOVSS(MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm), fpr.VX(vs)); - if (mask != 0xFFFFFFFF) - AND(32, MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm), Imm32(mask)); - if (setBits != 0) - OR(32, MIPSSTATE_VAR_ELEM32(vfpuCtrl[0], imm), Imm32(setBits)); - } - fpr.ReleaseSpillLocks(); - - if (imm == VFPU_CTRL_SPREFIX) { - js.prefixSFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm == VFPU_CTRL_TPREFIX) { - js.prefixTFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } else if (imm == VFPU_CTRL_DPREFIX) { - js.prefixDFlag = JitState::PREFIX_UNKNOWN; - js.blockWrotePrefixes = true; - } - } -} - -void Jit::Comp_VMatrixInit(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - if (!js.HasNoPrefix()) - DISABLE; - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - // Not really about trying here, it will work if enabled. - if (jo.enableVFPUSIMD) { - VectorSize vsz = GetVectorSize(sz); - u8 vecs[4]; - GetMatrixColumns(_VD, sz, vecs); - switch ((op >> 16) & 0xF) { - case 3: - MOV(PTRBITS, R(TEMPREG), ImmPtr(&identityMatrix[0])); - break; - case 7: - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - MOVAPS(XMM0, MatR(TEMPREG)); - break; - } - - for (int i = 0; i < n; i++) { - u8 vec[4]; - GetVectorRegs(vec, vsz, vecs[i]); - fpr.MapRegsVS(vec, vsz, MAP_NOINIT | MAP_DIRTY); - switch ((op >> 16) & 0xF) { - case 3: - MOVAPS(fpr.VSX(vec), MDisp(TEMPREG, 16 * i)); - break; - case 6: - XORPS(fpr.VSX(vec), fpr.VS(vec)); - break; - case 7: - MOVAPS(fpr.VSX(vec), R(XMM0)); - break; - } - } - fpr.ReleaseSpillLocks(); - return; - } - - u8 dregs[16]; - GetMatrixRegs(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - switch ((op >> 16) & 0xF) { - case 3: // vmidt - XORPS(XMM0, R(XMM0)); - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(XMM1, MatR(TEMPREG)); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - MOVSS(fpr.V(dregs[a * 4 + b]), a == b ? XMM1 : XMM0); - } - } - break; - case 6: // vmzero - XORPS(XMM0, R(XMM0)); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - MOVSS(fpr.V(dregs[a * 4 + b]), XMM0); - } - } - break; - case 7: // vmone - MOV(PTRBITS, R(TEMPREG), ImmPtr(&one)); - MOVSS(XMM0, MatR(TEMPREG)); - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - MOVSS(fpr.V(dregs[a * 4 + b]), XMM0); - } - } - break; - } - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vmmov(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMMOV); - - // TODO: This probably ignores prefixes? - if (!js.HasNoPrefix()) - DISABLE; - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - if (jo.enableVFPUSIMD) { - VectorSize vsz = GetVectorSize(sz); - u8 dest[4][4]; - MatrixOverlapType overlap = GetMatrixOverlap(_VD, _VS, sz); - - u8 vecs[4]; - if (overlap == OVERLAP_NONE) { - GetMatrixColumns(_VD, sz, vecs); - for (int i = 0; i < n; ++i) { - GetVectorRegs(dest[i], vsz, vecs[i]); - } - } else { - for (int i = 0; i < n; ++i) { - fpr.GetTempVS(dest[i], vsz); - } - } - - GetMatrixColumns(_VS, sz, vecs); - for (int i = 0; i < n; i++) { - u8 vec[4]; - GetVectorRegs(vec, vsz, vecs[i]); - fpr.MapRegsVS(vec, vsz, 0); - fpr.MapRegsVS(dest[i], vsz, MAP_NOINIT); - MOVAPS(fpr.VSX(dest[i]), fpr.VS(vec)); - fpr.ReleaseSpillLocks(); - } - - if (overlap != OVERLAP_NONE) { - // Okay, move from the temps to VD now. - GetMatrixColumns(_VD, sz, vecs); - for (int i = 0; i < n; i++) { - u8 vec[4]; - GetVectorRegs(vec, vsz, vecs[i]); - fpr.MapRegsVS(vec, vsz, MAP_NOINIT); - fpr.MapRegsVS(dest[i], vsz, 0); - MOVAPS(fpr.VSX(vec), fpr.VS(dest[i])); - fpr.ReleaseSpillLocks(); - } - } - - fpr.ReleaseSpillLocks(); - return; - } - - u8 sregs[16], dregs[16]; - GetMatrixRegs(sregs, sz, _VS); - GetMatrixRegs(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // TODO: gas doesn't allow overlap, what does the PSP do? - // Potentially detect overlap or the safe direction to move in, or just DISABLE? - // This is very not optimal, blows the regcache everytime. - u8 tempregs[16]; - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - u8 temp = (u8) fpr.GetTempV(); - fpr.MapRegV(temp, MAP_NOINIT | MAP_DIRTY); - MOVSS(fpr.VX(temp), fpr.V(sregs[a * 4 + b])); - fpr.StoreFromRegisterV(temp); - tempregs[a * 4 + b] = temp; - } - } - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - u8 temp = tempregs[a * 4 + b]; - fpr.MapRegV(temp, 0); - MOVSS(fpr.V(dregs[a * 4 + b]), fpr.VX(temp)); - } - } - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VScl(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - // The T prefix is applied oddly here, the interpreter knows how. - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[4], scale; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixT(&scale, V_Single, _VT); - GetVectorRegsPrefixD(dregs, sz, _VD); - - if (fpr.TryMapDirtyInInVS(dregs, sz, sregs, sz, &scale, V_Single, true)) { - MOVSS(XMM0, fpr.VS(&scale)); - if (sz != V_Single) - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(0, 0, 0, 0)); - if (dregs[0] != sregs[0]) { - MOVAPS(fpr.VSX(dregs), fpr.VS(sregs)); - } - MULPS(fpr.VSX(dregs), R(XMM0)); - ApplyPrefixD(dregs, sz); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(&scale, V_Single, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // Move to XMM0 early, so we don't have to worry about overlap with scale. - MOVSS(XMM0, fpr.V(scale)); - - X64Reg tempxregs[4]; - for (int i = 0; i < n; ++i) { - if (dregs[i] != scale || !IsOverlapSafeAllowS(dregs[i], i, n, sregs)) { - int reg = fpr.GetTempV(); - fpr.MapRegV(reg, MAP_NOINIT | MAP_DIRTY); - fpr.SpillLockV(reg); - tempxregs[i] = fpr.VX(reg); - } else { - fpr.MapRegV(dregs[i], dregs[i] == sregs[i] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[i]); - tempxregs[i] = fpr.VX(dregs[i]); - } - } - for (int i = 0; i < n; ++i) { - if (!fpr.V(sregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(tempxregs[i], fpr.V(sregs[i])); - MULSS(tempxregs[i], R(XMM0)); - } - for (int i = 0; i < n; ++i) { - if (!fpr.V(dregs[i]).IsSimpleReg(tempxregs[i])) - MOVSS(fpr.V(dregs[i]), tempxregs[i]); - } - ApplyPrefixD(dregs, sz); - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vmmul(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMMUL); - if (!js.HasNoPrefix()) { - DISABLE; - } - - if (PSP_CoreParameter().compat.flags().MoreAccurateVMMUL) { - // Fall back to interpreter, which has the accurate implementation. - // Later we might do something more optimized here. - DISABLE; - } - - MatrixSize sz = GetMtxSize(op); - VectorSize vsz = GetVectorSize(sz); - int n = GetMatrixSide(sz); - // vmmul.s isn't a real op, and the code below doesn't handle it. - if (n == 1) { - DISABLE; - } - - MatrixOverlapType soverlap = GetMatrixOverlap(_VS, _VD, sz); - MatrixOverlapType toverlap = GetMatrixOverlap(_VT, _VD, sz); - // If these overlap, we won't be able to map T as singles. - MatrixOverlapType stoverlap = GetMatrixOverlap(_VS, _VT, sz); - - const bool wraps = MatrixWrapsAround(_VS, sz) || MatrixWrapsAround(_VT, sz) || MatrixWrapsAround(_VD, sz); - if (jo.enableVFPUSIMD && !soverlap && !toverlap && !stoverlap && !wraps) { - u8 scols[4], dcols[4], tregs[16]; - - int vs = _VS; - int vd = _VD; - int vt = _VT; - - bool transposeDest = false; - bool transposeS = false; - - if ((vd & 0x20) && sz == M_4x4) { - vd ^= 0x20; - transposeDest = true; - } - - // Our algorithm needs a transposed S (which is the usual). - if (!(vs & 0x20) && sz == M_4x4) { - vs ^= 0x20; - transposeS = true; - } - - // The T matrix we will address individually. - GetMatrixColumns(vd, sz, dcols); - GetMatrixRows(vs, sz, scols); - memset(tregs, 255, sizeof(tregs)); - GetMatrixRegs(tregs, sz, vt); - for (int i = 0; i < 16; i++) { - if (tregs[i] != 255) - fpr.StoreFromRegisterV(tregs[i]); - } - - u8 scol[4][4]; - - // Map all of S's columns into registers. - for (int i = 0; i < n; i++) { - if (transposeS){ - fpr.StoreFromRegisterV(scols[i]); - } - GetVectorRegs(scol[i], vsz, scols[i]); - fpr.MapRegsVS(scol[i], vsz, 0); - fpr.SpillLockV(scols[i], vsz); - } - - // Shorter than manually stuffing the registers. But it feels like ther'es room for optimization here... - auto transposeInPlace = [=](u8 col[4][4]) { - MOVAPS(XMM0, fpr.VS(col[0])); - UNPCKLPS(fpr.VSX(col[0]), fpr.VS(col[2])); - UNPCKHPS(XMM0, fpr.VS(col[2])); - - MOVAPS(fpr.VSX(col[2]), fpr.VS(col[1])); - UNPCKLPS(fpr.VSX(col[1]), fpr.VS(col[3])); - UNPCKHPS(fpr.VSX(col[2]), fpr.VS(col[3])); - - MOVAPS(fpr.VSX(col[3]), fpr.VS(col[0])); - UNPCKLPS(fpr.VSX(col[0]), fpr.VS(col[1])); - UNPCKHPS(fpr.VSX(col[3]), fpr.VS(col[1])); - - MOVAPS(fpr.VSX(col[1]), R(XMM0)); - UNPCKLPS(fpr.VSX(col[1]), fpr.VS(col[2])); - UNPCKHPS(XMM0, fpr.VS(col[2])); - - MOVAPS(fpr.VSX(col[2]), fpr.VS(col[1])); - MOVAPS(fpr.VSX(col[1]), fpr.VS(col[3])); - MOVAPS(fpr.VSX(col[3]), R(XMM0)); - }; - - // Some games pass in S as an E matrix (transposed). Let's just transpose the data before we do the multiplication instead. - // This is shorter than trying to combine a discontinous matrix with lots of shufps. - if (transposeS) { - transposeInPlace(scol); - } - - // Now, work our way through the matrix, loading things as we go. - // TODO: With more temp registers, can generate much more efficient code. - for (int i = 0; i < n; i++) { - MOVSS(XMM1, fpr.V(tregs[4 * i])); // TODO: AVX broadcastss to replace this and the SHUFPS - MOVSS(XMM0, fpr.V(tregs[4 * i + 1])); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(0, 0, 0, 0)); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(0, 0, 0, 0)); - MULPS(XMM1, fpr.VS(scol[0])); - MULPS(XMM0, fpr.VS(scol[1])); - ADDPS(XMM1, R(XMM0)); - for (int j = 2; j < n; j++) { - MOVSS(XMM0, fpr.V(tregs[4 * i + j])); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(0, 0, 0, 0)); - MULPS(XMM0, fpr.VS(scol[j])); - ADDPS(XMM1, R(XMM0)); - } - // Map the D column. - u8 dcol[4]; - GetVectorRegs(dcol, vsz, dcols[i]); -#if !PPSSPP_ARCH(AMD64) - fpr.MapRegsVS(dcol, vsz, MAP_DIRTY | MAP_NOINIT | MAP_NOLOCK); -#else - fpr.MapRegsVS(dcol, vsz, MAP_DIRTY | MAP_NOINIT); -#endif - MOVAPS(fpr.VS(dcol), XMM1); - } - if (transposeS){ - for (int i = 0; i < n; i++){ - fpr.DiscardVS(scols[i]); - } - } - -#if !PPSSPP_ARCH(AMD64) - fpr.ReleaseSpillLocks(); -#endif - if (transposeDest) { - u8 dcol[4][4]; - for (int i = 0; i < n; i++) { - GetVectorRegs(dcol[i], vsz, dcols[i]); - fpr.MapRegsVS(dcol[i], vsz, MAP_DIRTY); - } - transposeInPlace(dcol); - } - fpr.ReleaseSpillLocks(); - return; - } - - u8 sregs[16], tregs[16], dregs[16]; - GetMatrixRegs(sregs, sz, _VS); - GetMatrixRegs(tregs, sz, _VT); - GetMatrixRegs(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // Rough overlap check. - bool overlap = false; - if (GetMtx(_VS) == GetMtx(_VD) || GetMtx(_VT) == GetMtx(_VD)) { - // Potential overlap (guaranteed for 3x3 or more). - overlap = true; - } - - if (overlap) { - u8 tempregs[16]; - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - MOVSS(XMM0, fpr.V(sregs[b * 4])); - MULSS(XMM0, fpr.V(tregs[a * 4])); - for (int c = 1; c < n; c++) { - MOVSS(XMM1, fpr.V(sregs[b * 4 + c])); - MULSS(XMM1, fpr.V(tregs[a * 4 + c])); - ADDSS(XMM0, R(XMM1)); - } - u8 temp = (u8) fpr.GetTempV(); - fpr.MapRegV(temp, MAP_NOINIT | MAP_DIRTY); - MOVSS(fpr.VX(temp), R(XMM0)); - fpr.StoreFromRegisterV(temp); - tempregs[a * 4 + b] = temp; - } - } - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - u8 temp = tempregs[a * 4 + b]; - fpr.MapRegV(temp, 0); - MOVSS(fpr.V(dregs[a * 4 + b]), fpr.VX(temp)); - } - } - } else { - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - MOVSS(XMM0, fpr.V(sregs[b * 4])); - MULSS(XMM0, fpr.V(tregs[a * 4])); - for (int c = 1; c < n; c++) { - MOVSS(XMM1, fpr.V(sregs[b * 4 + c])); - MULSS(XMM1, fpr.V(tregs[a * 4 + c])); - ADDSS(XMM0, R(XMM1)); - } - MOVSS(fpr.V(dregs[a * 4 + b]), XMM0); - } - } - } - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vmscl(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VMSCL); - - // TODO: This op probably ignores prefixes? - if (!js.HasNoPrefix()) - DISABLE; - - MatrixSize sz = GetMtxSize(op); - int n = GetMatrixSide(sz); - - u8 sregs[16], dregs[16], scale; - GetMatrixRegs(sregs, sz, _VS); - GetVectorRegs(&scale, V_Single, _VT); - GetMatrixRegs(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(&scale, V_Single, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // Move to XMM0 early, so we don't have to worry about overlap with scale. - MOVSS(XMM0, fpr.V(scale)); - - // TODO: test overlap, optimize. - u8 tempregs[16]; - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - u8 temp = (u8) fpr.GetTempV(); - fpr.MapRegV(temp, MAP_NOINIT | MAP_DIRTY); - MOVSS(fpr.VX(temp), fpr.V(sregs[a * 4 + b])); - MULSS(fpr.VX(temp), R(XMM0)); - fpr.StoreFromRegisterV(temp); - tempregs[a * 4 + b] = temp; - } - } - for (int a = 0; a < n; a++) { - for (int b = 0; b < n; b++) { - u8 temp = tempregs[a * 4 + b]; - fpr.MapRegV(temp, 0); - MOVSS(fpr.V(dregs[a * 4 + b]), fpr.VX(temp)); - } - } - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vtfm(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_MTX_VTFM); - - // TODO: This probably ignores prefixes? Or maybe uses D? - // Vertex transform, "weird prefixes" per the IR frontend, which hands them to the interpreter too. - if (!js.HasNoPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - MatrixSize msz = GetMtxSize(op); - int n = GetNumVectorElements(sz); - int ins = (op >> 23) & 7; - - bool homogenous = false; - if (n == ins) { - n++; - sz = (VectorSize)((int)(sz)+1); - msz = (MatrixSize)((int)(msz)+1); - homogenous = true; - } - // Otherwise, n should already be ins + 1. - else if (n != ins + 1) { - DISABLE; - } - - if (jo.enableVFPUSIMD) { - u8 scols[4], dcol[4], tregs[4]; - - int vs = _VS; - int vd = _VD; - int vt = _VT; // vector! - - // The T matrix we will address individually. - GetVectorRegs(dcol, sz, vd); - GetMatrixRows(vs, msz, scols); - GetVectorRegs(tregs, sz, vt); - for (int i = 0; i < n; i++) { - fpr.StoreFromRegisterV(tregs[i]); - } - - // We need the T regs in individual regs, but they could overlap with S regs. - // If that happens, we copy the T reg to a temp. - auto flushConflictingTRegsToTemps = [&](u8 regs[4]) { - for (int i = 0; i < n; ++i) { - for (int j = 0; j < n; ++j) { - if (regs[i] != tregs[j]) { - continue; - } - - // They match. Let's replace this treg with a temp reg. - // Note that it will spill if there's contention, unfortunately... - tregs[j] = fpr.GetTempV(); - fpr.MapRegV(tregs[j], MAP_NOINIT); - MOVSS(fpr.VX(tregs[j]), fpr.V(regs[i])); - } - } - }; - - u8 scol[4][4]; - - // Map all of S's columns into registers. - for (int i = 0; i < n; i++) { - GetVectorRegs(scol[i], sz, scols[i]); - flushConflictingTRegsToTemps(scol[i]); - fpr.MapRegsVS(scol[i], sz, 0); - } - - // Now, work our way through the matrix, loading things as we go. - // TODO: With more temp registers, can generate much more efficient code. - MOVSS(XMM1, fpr.V(tregs[0])); // TODO: AVX broadcastss to replace this and the SHUFPS (but take care of temps, unless we force store them.) - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(0, 0, 0, 0)); - MULPS(XMM1, fpr.VS(scol[0])); - for (int j = 1; j < n; j++) { - if (!homogenous || j != n - 1) { - MOVSS(XMM0, fpr.V(tregs[j])); - SHUFPS(XMM0, R(XMM0), _MM_SHUFFLE(0, 0, 0, 0)); - MULPS(XMM0, fpr.VS(scol[j])); - ADDPS(XMM1, R(XMM0)); - } else { - ADDPS(XMM1, fpr.VS(scol[j])); - } - } - // Map the D column. Release first in case of overlap. - for (int i = 0; i < n; i++) { - fpr.ReleaseSpillLockV(scol[i], sz); - } - fpr.MapRegsVS(dcol, sz, MAP_DIRTY | MAP_NOINIT); - MOVAPS(fpr.VS(dcol), XMM1); - fpr.ReleaseSpillLocks(); - return; - } - - u8 sregs[16], dregs[4], tregs[4]; - GetMatrixRegs(sregs, msz, _VS); - GetVectorRegs(tregs, sz, _VT); - GetVectorRegs(dregs, sz, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, msz, 0); - fpr.SimpleRegsV(tregs, sz, 0); - fpr.SimpleRegsV(dregs, sz, MAP_NOINIT | MAP_DIRTY); - - // TODO: test overlap, optimize. - u8 tempregs[4]; - for (int i = 0; i < n; i++) { - MOVSS(XMM0, fpr.V(sregs[i * 4])); - MULSS(XMM0, fpr.V(tregs[0])); - for (int k = 1; k < n; k++) - { - MOVSS(XMM1, fpr.V(sregs[i * 4 + k])); - if (!homogenous || k != n - 1) - MULSS(XMM1, fpr.V(tregs[k])); - ADDSS(XMM0, R(XMM1)); - } - - u8 temp = (u8) fpr.GetTempV(); - fpr.MapRegV(temp, MAP_NOINIT | MAP_DIRTY); - MOVSS(fpr.VX(temp), R(XMM0)); - fpr.StoreFromRegisterV(temp); - tempregs[i] = temp; - } - for (int i = 0; i < n; i++) { - u8 temp = tempregs[i]; - fpr.MapRegV(temp, 0); - MOVSS(fpr.V(dregs[i]), fpr.VX(temp)); - } - - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_VCrs(MIPSOpcode op) { - DISABLE; -} - -void Jit::Comp_VDet(MIPSOpcode op) { - DISABLE; -} - -// The goal is to map (reversed byte order for clarity): -// 000000AA 000000BB 000000CC 000000DD -> AABBCCDD -alignas(16) static const s8 vi2xc_shuffle[16] = { 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; -// 0000AAAA 0000BBBB 0000CCCC 0000DDDD -> AAAABBBB CCCCDDDD -alignas(16) static const s8 vi2xs_shuffle[16] = { 2, 3, 6, 7, 10, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1 }; - -void Jit::Comp_Vi2x(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || js.HasSPrefix()) - DISABLE; - - int bits = ((op >> 16) & 2) == 0 ? 8 : 16; // vi2uc/vi2c (0/1), vi2us/vi2s (2/3) - bool unsignedOp = ((op >> 16) & 1) == 0; // vi2uc (0), vi2us (2) - - // These instructions pack pairs or quads of integers into 32 bits. - // The unsigned (u) versions skip the sign bit when packing. - - VectorSize sz = GetVecSize(op); - VectorSize outsize; - if (bits == 8) { - outsize = V_Single; - if (sz != V_Quad) { - DISABLE; - } - } else { - switch (sz) { - case V_Pair: - outsize = V_Single; - break; - case V_Quad: - outsize = V_Pair; - break; - default: - DISABLE; - } - } - - u8 sregs[4], dregs[4]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, outsize, _VD); - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, outsize, MAP_NOINIT | MAP_DIRTY); - - // First, let's assemble the sregs into lanes of a single xmm reg. - // For quad inputs, we need somewhere for the bottom regs. Ideally dregs[0]. - X64Reg dst0 = XMM0; - if (sz == V_Quad) { - int vreg = dregs[0]; - if (!IsOverlapSafeAllowS(dregs[0], 0, 4, sregs)) { - // Will be discarded on release. - vreg = fpr.GetTempV(); - } - fpr.MapRegV(vreg, vreg == sregs[0] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(vreg); - dst0 = fpr.VX(vreg); - } else { - // Pair, let's check if we should use dregs[0] directly. No temp needed. - int vreg = dregs[0]; - if (IsOverlapSafeAllowS(dregs[0], 0, 2, sregs)) { - fpr.MapRegV(vreg, vreg == sregs[0] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(vreg); - dst0 = fpr.VX(vreg); - } - } - - if (!fpr.V(sregs[0]).IsSimpleReg(dst0)) { - MOVSS(dst0, fpr.V(sregs[0])); - } - MOVSS(XMM1, fpr.V(sregs[1])); - // With this, we have the lower half in dst0. - PUNPCKLDQ(dst0, R(XMM1)); - if (sz == V_Quad) { - MOVSS(XMM0, fpr.V(sregs[2])); - MOVSS(XMM1, fpr.V(sregs[3])); - PUNPCKLDQ(XMM0, R(XMM1)); - // Now we need to combine XMM0 into dst0. - PUNPCKLQDQ(dst0, R(XMM0)); - } else { - // Otherwise, we need to zero out the top 2. - // We expect XMM1 to be zero below. - PXOR(XMM1, R(XMM1)); - PUNPCKLQDQ(dst0, R(XMM1)); - } - - // For "u" type ops, we clamp to zero and shift off the sign bit first. - if (unsignedOp) { - if (cpu_info.bSSE4_1) { - if (sz == V_Quad) { - // Zeroed in the other case above. - PXOR(XMM1, R(XMM1)); - } - PMAXSD(dst0, R(XMM1)); - PSLLD(dst0, 1); - } else { - // Get a mask of the sign bit in dst0, then and in the values. This clamps to 0. - MOVDQA(XMM1, R(dst0)); - PSRAD(dst0, 31); - PSLLD(XMM1, 1); - PANDN(dst0, R(XMM1)); - } - } - - // At this point, everything is aligned in the high bits of our lanes. - if (cpu_info.bSSSE3) { - if (RipAccessible(vi2xc_shuffle)) { - PSHUFB(dst0, bits == 8 ? M(vi2xc_shuffle) : M(vi2xs_shuffle)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), bits == 8 ? ImmPtr(vi2xc_shuffle) : ImmPtr(vi2xs_shuffle)); - PSHUFB(dst0, MatR(TEMPREG)); - } - } else { - // Let's *arithmetically* shift in the sign so we can use saturating packs. - PSRAD(dst0, 32 - bits); - // XMM1 used for the high part just so there's no dependency. It contains garbage or 0. - PACKSSDW(dst0, R(XMM1)); - if (bits == 8) { - PACKSSWB(dst0, R(XMM1)); - } - } - - if (!fpr.V(dregs[0]).IsSimpleReg(dst0)) { - MOVSS(fpr.V(dregs[0]), dst0); - } - if (outsize == V_Pair) { - fpr.MapRegV(dregs[1], MAP_NOINIT | MAP_DIRTY); - MOVDQA(fpr.V(dregs[1]), dst0); - // Shift out the lower result to get the result we want. - PSRLDQ(fpr.VX(dregs[1]), 4); - } - - ApplyPrefixD(dregs, outsize); - fpr.ReleaseSpillLocks(); -} - -alignas(16) static const float vavg_table[4] = { 1.0f, 1.0f / 2.0f, 1.0f / 3.0f, 1.0f / 4.0f }; - -void Jit::Comp_Vhoriz(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) - DISABLE; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 sregs[4], dregs[1]; - GetVectorRegsPrefixS(sregs, sz, _VS); - GetVectorRegsPrefixD(dregs, V_Single, _VD); - if (fpr.TryMapDirtyInVS(dregs, V_Single, sregs, sz)) { - if (cpu_info.bSSE4_1) { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&oneOneOneOne)); - switch (sz) { - case V_Pair: - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, MatR(TEMPREG), 0x31); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - case V_Triple: - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, MatR(TEMPREG), 0x71); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - case V_Quad: - XORPS(XMM1, R(XMM1)); - MOVAPS(XMM0, fpr.VS(sregs)); - DPPS(XMM0, MatR(TEMPREG), 0xF1); - // In every other case, +0.0 is selected by the mask and added. - // But, here we need to manually add it to the result. - ADDPS(XMM0, R(XMM1)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - default: - DISABLE; - } - } else { - switch (sz) { - case V_Pair: - XORPS(XMM1, R(XMM1)); - MOVAPS(XMM0, fpr.VS(sregs)); - ADDPS(XMM1, R(XMM0)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 2, 1, 1)); - ADDPS(XMM0, R(XMM1)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - case V_Triple: - XORPS(XMM1, R(XMM1)); - MOVAPS(XMM0, fpr.VS(sregs)); - ADDPS(XMM1, R(XMM0)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 2, 1, 1)); - ADDPS(XMM0, R(XMM1)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(3, 2, 1, 2)); - ADDPS(XMM0, R(XMM1)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - case V_Quad: - XORPS(XMM1, R(XMM1)); - MOVAPS(XMM0, fpr.VS(sregs)); - // This flips the sign of any -0.000. - ADDPS(XMM0, R(XMM1)); - MOVHLPS(XMM1, XMM0); - ADDPS(XMM0, R(XMM1)); - MOVAPS(XMM1, R(XMM0)); - SHUFPS(XMM1, R(XMM1), _MM_SHUFFLE(1, 1, 1, 1)); - ADDPS(XMM0, R(XMM1)); - MOVAPS(fpr.VSX(dregs), R(XMM0)); - break; - default: - DISABLE; - } - } - if (((op >> 16) & 31) == 7) { // vavg - MOV(PTRBITS, R(TEMPREG), ImmPtr(&vavg_table[n - 1])); - MULSS(fpr.VSX(dregs), MatR(TEMPREG)); - } - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocks(); - return; - } - - // Flush SIMD. - fpr.SimpleRegsV(sregs, sz, 0); - fpr.SimpleRegsV(dregs, V_Single, MAP_NOINIT | MAP_DIRTY); - - X64Reg reg = XMM0; - if (IsOverlapSafe(dregs[0], 0, n, sregs)) { - fpr.MapRegV(dregs[0], dregs[0] == sregs[0] ? MAP_DIRTY : MAP_NOINIT); - fpr.SpillLockV(dregs[0]); - reg = fpr.VX(dregs[0]); - } - - // We have to start zt +0.000 in case any values are -0.000. - XORPS(reg, R(reg)); - for (int i = 0; i < n; ++i) { - ADDSS(reg, fpr.V(sregs[i])); - } - - switch ((op >> 16) & 31) { - case 6: // vfad - break; - case 7: // vavg - MOV(PTRBITS, R(TEMPREG), ImmPtr(&vavg_table[n - 1])); - MULSS(reg, MatR(TEMPREG)); - break; - } - - if (reg == XMM0) { - MOVSS(fpr.V(dregs[0]), XMM0); - } - - ApplyPrefixD(dregs, V_Single); - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Viim(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - if (js.HasUnknownPrefix()) - DISABLE; - - u8 dreg; - GetVectorRegs(&dreg, V_Single, _VT); - - // Flush SIMD. - fpr.SimpleRegsV(&dreg, V_Single, MAP_NOINIT | MAP_DIRTY); - - s32 imm = SignExtend16ToS32(op); - FP32 fp; - fp.f = (float)imm; - MOV(32, R(TEMPREG), Imm32(fp.u)); - fpr.MapRegV(dreg, MAP_DIRTY | MAP_NOINIT); - MOVD_xmm(fpr.VX(dreg), R(TEMPREG)); - - ApplyPrefixD(&dreg, V_Single); - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_Vfim(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_XFER); - - if (js.HasUnknownPrefix()) - DISABLE; - - u8 dreg; - GetVectorRegs(&dreg, V_Single, _VT); - - // Flush SIMD. - fpr.SimpleRegsV(&dreg, V_Single, MAP_NOINIT | MAP_DIRTY); - - FP16 half; - half.u = op & 0xFFFF; - FP32 fval = half_to_float_fast5(half); - MOV(32, R(TEMPREG), Imm32(fval.u)); - fpr.MapRegV(dreg, MAP_DIRTY | MAP_NOINIT); - MOVD_xmm(fpr.VX(dreg), R(TEMPREG)); - - ApplyPrefixD(&dreg, V_Single); - fpr.ReleaseSpillLocks(); -} - -void Jit::CompVrotShuffle(u8 *dregs, int imm, int n, bool negSin) { - char what[4] = { '0', '0', '0', '0' }; - if (((imm >> 2) & 3) == (imm & 3)) { - for (int i = 0; i < 4; i++) - what[i] = 'S'; - } - what[(imm >> 2) & 3] = 'S'; - what[imm & 3] = 'C'; - - // TODO: shufps SIMD version - - for (int i = 0; i < n; i++) { - fpr.MapRegV(dregs[i], MAP_DIRTY | MAP_NOINIT); - switch (what[i]) { - case 'C': MOVSS(fpr.V(dregs[i]), XMM1); break; - case 'S': - MOVSS(fpr.V(dregs[i]), XMM0); - if (negSin) { - if (RipAccessible(&signBitLower)) { - XORPS(fpr.VX(dregs[i]), M(&signBitLower)); // rip accessible - } else { - MOV(PTRBITS, R(TEMPREG), ImmPtr(&signBitLower)); - XORPS(fpr.VX(dregs[i]), MatR(TEMPREG)); - } - } - break; - case '0': - { - XORPS(fpr.VX(dregs[i]), fpr.V(dregs[i])); - break; - } - default: - ERROR_LOG(Log::JIT, "Bad what in vrot"); - break; - } - } -} - -// Very heavily used by FF:CC -void Jit::Comp_VRot(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - // The prefixes apply to the sine but never to the cosine; leave that to the interpreter. - if (!js.HasNoPrefix()) { - DISABLE; - } - if (!js.HasNoPrefix()) { - // Prefixes work strangely for this, see IRCompVFPU. - WARN_LOG_REPORT(Log::JIT, "vrot instruction using prefixes at %08x", GetCompilerPC()); - DISABLE; - } - - int vd = _VD; - int vs = _VS; - - VectorSize sz = GetVecSize(op); - int n = GetNumVectorElements(sz); - - u8 dregs[4]; - u8 dregs2[4]; - - MIPSOpcode nextOp = GetOffsetInstruction(1); - int vd2 = -1; - int imm2 = -1; - if ((nextOp >> 26) == 60 && ((nextOp >> 21) & 0x1F) == 29 && _VS == MIPS_GET_VS(nextOp)) { - // Pair of vrot with the same angle argument. Let's join them (can share sin/cos results). - vd2 = MIPS_GET_VD(nextOp); - imm2 = (nextOp >> 16) & 0x1f; - // NOTICE_LOG(Log::JIT, "Joint VFPU at %08x", js.blockStart); - } - - u8 sreg; - GetVectorRegs(dregs, sz, vd); - if (vd2 >= 0) - GetVectorRegs(dregs2, sz, vd2); - GetVectorRegs(&sreg, V_Single, vs); - // With the angle in a destination lane, the cosine is taken of what was written there. - // The assembler refuses that, so leave it to the interpreter, and don't pair such a vrot. - for (int i = 0; i < n; i++) { - if (dregs[i] == sreg) { - DISABLE; - } - if (vd2 >= 0 && dregs2[i] == sreg) { - vd2 = -1; - } - } - - // Flush SIMD. - fpr.SimpleRegsV(&sreg, V_Single, 0); - - int imm = (op >> 16) & 0x1f; - - bool negSin1 = (imm & 0x10) ? true : false; - -#if PPSSPP_ARCH(AMD64) -#ifdef _WIN32 - LEA(64, RDX, MIPSSTATE_VAR(sincostemp)); -#else - LEA(64, RDI, MIPSSTATE_VAR(sincostemp)); -#endif - MOVSS(XMM0, fpr.V(sreg)); - CallProtectedLeaf(negSin1 ? (const void *)&SinCosNegSin : (const void *)&SinCos); -#else - gpr.FlushBeforeCall(); - fpr.Flush(); - - // Sigh, passing floats with cdecl isn't pretty, ends up on the stack. - ABI_CallFunctionAC(negSin1 ? (const void *)&SinCosNegSin : (const void *)&SinCos, fpr.V(sreg), (uintptr_t)mips_->sincostemp); -#endif - - MOVSS(XMM0, MIPSSTATE_VAR(sincostemp[0])); - MOVSS(XMM1, MIPSSTATE_VAR(sincostemp[1])); - - CompVrotShuffle(dregs, imm, n, false); - if (vd2 != -1) { - // If the negsin setting differs between the two joint invocations, we need to flip the second one. - bool negSin2 = (imm2 & 0x10) ? true : false; - CompVrotShuffle(dregs2, imm2, n, negSin1 != negSin2); - EatInstruction(nextOp); - } - fpr.ReleaseSpillLocks(); -} - -void Jit::Comp_ColorConv(MIPSOpcode op) { - CONDITIONAL_DISABLE(VFPU_VEC); - if (js.HasUnknownPrefix() || !IsPrefixWithinSize(js.prefixS, op) || js.HasTPrefix()) - DISABLE; - - int vd = _VD; - int vs = _VS; - - DISABLE; -#if 0 - VectorSize sz = V_Quad; - int n = GetNumVectorElements(sz); - - switch ((op >> 16) & 3) { - case 1: - break; - default: - DISABLE; - } - - u8 sregs[4]; - u8 dregs[1]; - // WARNING: Prefixes. - GetVectorRegs(sregs, sz, vs); - GetVectorRegs(dregs, V_Pair, vd); - - if (fpr.TryMapDirtyInVS(dregs, V_Single, sregs, sz)) { - switch ((op >> 16) & 3) { - case 1: // 4444 - { - //int a = ((in >> 24) & 0xFF) >> 4; - //int b = ((in >> 16) & 0xFF) >> 4; - //int g = ((in >> 8) & 0xFF) >> 4; - //int r = ((in)& 0xFF) >> 4; - //col = (a << 12) | (b << 8) | (g << 4) | (r); - //PACKUSW - break; - } - case 2: // 5551 - { - //int a = ((in >> 24) & 0xFF) >> 7; - //int b = ((in >> 16) & 0xFF) >> 3; - //int g = ((in >> 8) & 0xFF) >> 3; - //int r = ((in)& 0xFF) >> 3; - //col = (a << 15) | (b << 10) | (g << 5) | (r); - break; - } - case 3: // 565 - { - //int b = ((in >> 16) & 0xFF) >> 3; - //int g = ((in >> 8) & 0xFF) >> 2; - //int r = ((in)& 0xFF) >> 3; - //col = (b << 11) | (g << 5) | (r); - break; - } - } - DISABLE; - - // Flush SIMD. - fpr.SimpleRegsV(&sreg, V_Pair, MAP_NOINIT | MAP_DIRTY); - fpr.SimpleRegsV(&dreg, V_Pair, MAP_NOINIT | MAP_DIRTY); -#endif - -} -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/Jit.cpp b/Core/MIPS/x86/Jit.cpp deleted file mode 100644 index 5526a91584..0000000000 --- a/Core/MIPS/x86/Jit.cpp +++ /dev/null @@ -1,971 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include -#include - -#include "Common/Math/math_util.h" -#include "Common/Profiler/Profiler.h" - -#include "Common/Serialize/Serializer.h" -#include "Common/Serialize/SerializeFuncs.h" -#include "Core/Core.h" -#include "Core/MemMap.h" -#include "Core/System.h" -#include "Core/CoreTiming.h" -#include "Core/Config.h" -#include "Core/Reporting.h" -#include "Core/Debugger/SymbolMap.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/MIPSCodeUtils.h" -#include "Core/MIPS/Interpreter.h" -#include "Core/MIPS/MIPSTables.h" -#include "Core/HLE/ReplaceTables.h" - -#include "RegCache.h" -#include "Jit.h" - -#include "Core/Debugger/Breakpoints.h" - -namespace MIPSComp -{ -using namespace Gen; - -const bool USE_JIT_MISSMAP = false; -static std::map notJitOps; - -template -std::pair flip_pair(const std::pair &p) { - return std::pair(p.second, p.first); -} - -// This is called when Jit hits a breakpoint. Returns 1 when hit. -u32 JitBreakpoint(uint32_t addr) -{ - // Should we skip this breakpoint? - if (g_breakpoints.CheckSkipFirst() == currentMIPS->pc || g_breakpoints.CheckSkipFirst() == addr) - return 0; - - BreakAction result = g_breakpoints.ExecBreakPoint(addr); - if ((result & BREAK_ACTION_PAUSE) == 0) - return 0; - - // There's probably a better place for this. - if (USE_JIT_MISSMAP) { - std::map notJitSorted; - std::transform(notJitOps.begin(), notJitOps.end(), std::inserter(notJitSorted, notJitSorted.begin()), flip_pair); - - std::string message; - char temp[256]; - int remaining = 15; - for (auto it = notJitSorted.rbegin(), end = notJitSorted.rend(); it != end && --remaining >= 0; ++it) - { - snprintf(temp, 256, " (%d), ", it->first); - message += it->second + temp; - } - - if (message.size() > 2) - message.resize(message.size() - 2); - - NOTICE_LOG(Log::JIT, "Top ops compiled to interpreter: %s", message.c_str()); - } - - return 1; -} - -static u32 JitMemCheck(u32 addr, u32 pc) { - // Should we skip this breakpoint? - if (g_breakpoints.CheckSkipFirst() == currentMIPS->pc) - return 0; - - // Did we already hit one? - if (coreState != CORE_RUNNING_CPU && coreState != CORE_NEXTFRAME) - return 1; - - // Note: pc may be the delay slot. - g_breakpoints.ExecOpMemCheck(addr, pc); - return coreState == CORE_RUNNING_CPU || coreState == CORE_NEXTFRAME ? 0 : 1; -} - -static void JitLogMiss(MIPSOpcode op) -{ - if (USE_JIT_MISSMAP) - notJitOps[MIPSGetName(op)]++; - - MIPSInterpretFunc func = MIPSGetInterpretFunc(op); - func(currentMIPS, op); -} - -#ifdef _MSC_VER -// JitBlockCache doesn't use this, just stores it. -#pragma warning(disable:4355) -#endif -Jit::Jit(MIPSState *mipsState) - : blocks(mipsState, this), mips_(mipsState) { - blocks.Init(); - gpr.SetEmitter(this); - fpr.SetEmitter(this); - AllocCodeSpace(1024 * 1024 * 16); - GenerateFixedCode(jo); - - safeMemFuncs.Init(&thunks); - - js.startDefaultPrefix = mips_->HasDefaultPrefix(); - - // The debugger sets this so that "go" on a breakpoint will actually... go. - // But if they reset, we can end up hitting it by mistake, since it's based on PC and ticks. - g_breakpoints.SetSkipFirst(0); -} - -Jit::~Jit() { -} - -void Jit::DoState(PointerWrap &p) { - auto s = p.Section("Jit", 1, 2); - if (!s) - return; - - Do(p, js.startDefaultPrefix); - if (p.mode == PointerWrap::MODE_READ && !js.startDefaultPrefix) { - WARN_LOG(Log::CPU, "Jit: An uneaten prefix was previously detected. Jitting in unknown-prefix mode."); - } - if (s >= 2) { - Do(p, js.hasSetRounding); - if (p.mode == PointerWrap::MODE_READ) { - js.lastSetRounding = 0; - } - } else { - js.hasSetRounding = 1; - } - - // The debugger sets this so that "go" on a breakpoint will actually... go. - // But if they load a state, we can end up hitting it by mistake, since it's based on PC and ticks. - g_breakpoints.SetSkipFirst(0); -} - -void Jit::UpdateFCR31() { - // fcr31 can reach a non-default mode without a compiled ctc1 (an interpreted one, a savestate, - // a core switch, a thread switch). Blocks compiled before that don't switch modes around - // syscalls or conversions, so they're thrown away, at the next safe point. - if ((mips_->fcr31 & 0x01000003) && !js.hasSetRounding) { - js.hasSetRounding = true; - js.lastSetRounding = true; - mips_->ClearJitCacheDeferred(); - } -} - -void Jit::GetStateAndFlushAll(RegCacheState &state) { - gpr.GetState(state.gpr); - fpr.GetState(state.fpr); - FlushAll(); -} - -void Jit::RestoreState(const RegCacheState& state) { - gpr.RestoreState(state.gpr); - fpr.RestoreState(state.fpr); -} - -void Jit::FlushAll() { - gpr.Flush(); - fpr.Flush(); - FlushPrefixV(); -} - -void Jit::FlushPrefixV() { - if (js.startDefaultPrefix && !js.blockWrotePrefixes && js.HasNoPrefix()) { - // They started default, we never modified in memory, and they're default now. - // No reason to modify memory. This is common at end of blocks. Just clear dirty. - js.prefixSFlag = (JitState::PrefixState)(js.prefixSFlag & ~JitState::PREFIX_DIRTY); - js.prefixTFlag = (JitState::PrefixState)(js.prefixTFlag & ~JitState::PREFIX_DIRTY); - js.prefixDFlag = (JitState::PrefixState)(js.prefixDFlag & ~JitState::PREFIX_DIRTY); - return; - } - - if ((js.prefixSFlag & JitState::PREFIX_DIRTY) != 0) { - MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_SPREFIX]), Imm32(js.prefixS)); - js.prefixSFlag = (JitState::PrefixState) (js.prefixSFlag & ~JitState::PREFIX_DIRTY); - } - - if ((js.prefixTFlag & JitState::PREFIX_DIRTY) != 0) { - MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_TPREFIX]), Imm32(js.prefixT)); - js.prefixTFlag = (JitState::PrefixState) (js.prefixTFlag & ~JitState::PREFIX_DIRTY); - } - - if ((js.prefixDFlag & JitState::PREFIX_DIRTY) != 0) { - MOV(32, MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_DPREFIX]), Imm32(js.prefixD)); - js.prefixDFlag = (JitState::PrefixState) (js.prefixDFlag & ~JitState::PREFIX_DIRTY); - } - - // If we got here, we must've written prefixes to memory in this block. - js.blockWrotePrefixes = true; -} - -void Jit::WriteDowncount(int offset) { - const int downcount = js.downcountAmount + offset; - SUB(32, MIPSSTATE_VAR(downcount), downcount > 127 ? Imm32(downcount) : Imm8(downcount)); -} - -void Jit::RestoreRoundingMode(bool force) { - // If the game has never set an interesting rounding mode, we can safely skip this. - if (force || js.hasSetRounding) { - CALL(restoreRoundingMode); - } -} - -void Jit::ApplyRoundingMode(bool force) { - // If the game has never set an interesting rounding mode, we can safely skip this. - if (force || js.hasSetRounding) { - CALL(applyRoundingMode); - } -} - -void Jit::UpdateRoundingMode(u32 fcr31) { - // We must set js.hasSetRounding at compile time, or this block will use the wrong rounding mode. - // The fcr31 parameter is -1 when not known at compile time, so we just assume it was changed. - if (fcr31 & 0x01000003) { - js.hasSetRounding = true; - } -} - -void Jit::ClearCache() -{ - blocks.Clear(); - ClearCodeSpace(0); - GenerateFixedCode(jo); - // The compile path may have moved, and replacement hooks' block shadow entries point to it. - Replacement_SetBlockShadowHook(GetBlockShadowHookValue()); -} - -void Jit::SaveFlags() { - PUSHF(); -#if PPSSPP_ARCH(AMD64) - // On X64, the above misaligns the stack. However there might be a cheaper solution than this. - POP(64, R(EAX)); - MOV(64, MIPSSTATE_VAR(saved_flags), R(EAX)); -#endif -} - -void Jit::LoadFlags() { -#if PPSSPP_ARCH(AMD64) - MOV(64, R(EAX), MIPSSTATE_VAR(saved_flags)); - PUSH(64, R(EAX)); -#endif - POPF(); -} - -void Jit::CompileDelaySlot(int flags, RegCacheState *state) { - // Need to offset the downcount which was already incremented for the branch + delay slot. - CheckJitBreakpoint(GetCompilerPC() + 4, -2); - - if (flags & DELAYSLOT_SAFE) - SaveFlags(); // preserve flag around the delay slot! - - js.inDelaySlot = true; - MIPSOpcode op = GetOffsetInstruction(1); - MIPSCompileOp(op, this); - js.inDelaySlot = false; - - if (flags & DELAYSLOT_FLUSH) { - if (state != NULL) - GetStateAndFlushAll(*state); - else - FlushAll(); - } - if (flags & DELAYSLOT_SAFE) - LoadFlags(); // restore flag! -} - -void Jit::EatInstruction(MIPSOpcode op) { - MIPSInfo info = MIPSGetInfo(op); - if (info & DELAYSLOT) { - ERROR_LOG_REPORT_ONCE(ateDelaySlot, Log::JIT, "Ate a branch op."); - } - if (js.inDelaySlot) { - ERROR_LOG_REPORT_ONCE(ateInDelaySlot, Log::JIT, "Ate an instruction inside a delay slot."); - } - - CheckJitBreakpoint(GetCompilerPC() + 4, 0); - js.numInstructions++; - js.compilerPC += 4; - js.downcountAmount += MIPSGetInstructionCycleEstimate(op); -} - -void Jit::Compile(u32 em_address) { - PROFILE_THIS_SCOPE("jitc"); - - if (!Memory::IsValid4AlignedAddress(em_address)) { - Core_ExecException(em_address, em_address, ExecExceptionType::JUMP); - return; - } - - if (GetSpaceLeft() < 0x10000 || blocks.IsFull()) { - ClearCache(); - } - - // Sometimes we compile fairly large blocks, although it's uncommon. - BeginWrite(JitBlockCache::MAX_BLOCK_INSTRUCTIONS * 16); - - int block_num = blocks.AllocateBlock(em_address); - JitBlock *b = blocks.GetBlock(block_num); - b->DoIntegrityCheck(em_address, block_num, "BeforeJit"); - DoJit(em_address, b); - b->DoIntegrityCheck(em_address, block_num, "BeforeFinalize"); - blocks.FinalizeBlock(block_num, jo.enableBlocklink); - b->DoIntegrityCheck(em_address, block_num, "AfterFinalize"); - _dbg_assert_(js.nextExit <= 2); - - EndWrite(); - - bool cleanSlate = false; - - if (js.hasSetRounding && !js.lastSetRounding) { - WARN_LOG(Log::JIT, "Detected rounding mode usage, rebuilding jit with checks"); - // Won't loop, since hasSetRounding is only ever set to 1. - js.lastSetRounding = js.hasSetRounding; - cleanSlate = true; - } - - // Drat. The VFPU hit an uneaten prefix at the end of a block. - if (js.startDefaultPrefix && js.MayHavePrefix()) { - WARN_LOG_REPORT(Log::JIT, "An uneaten prefix at end of block: %08x", GetCompilerPC() - 4); - js.LogPrefix(); - - // Let's try that one more time. We won't get back here because we toggled the value. - js.startDefaultPrefix = false; - cleanSlate = true; - } - - if (cleanSlate) { - // Our assumptions are all wrong so it's clean-slate time. - ClearCache(); - Compile(em_address); - } -} - -void Jit::RunLoopUntil(u64 globalticks) { - PROFILE_THIS_SCOPE("jit"); - ((void (*)())enterDispatcher)(); -} - -u32 Jit::GetCompilerPC() { - return js.compilerPC; -} - -MIPSOpcode Jit::GetOffsetInstruction(int offset) { - const u32 addr = GetCompilerPC() + 4 * offset; - return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); -} - -void Jit::DoJit(u32 em_address, JitBlock *b) { - js.Begin(b); - - // We add a check before the block, used when entering from a linked block. - b->checkedEntry = GetCodePtr(); - // Downcount flag check. The last block decremented downcounter, and the flag should still be available. - FixupBranch skip = J_CC(CC_NS); - MOV(32, MIPSSTATE_VAR(pc), Imm32(js.blockStart)); - JMP(outerLoop, true); // downcount hit zero - go advance. - SetJumpTarget(skip); - - b->normalEntry = GetCodePtr(); - - gpr.Start(mips_, &js, &jo); - fpr.Start(mips_, &js, &jo, RipAccessible(&mips_->v[0])); - - js.numInstructions = 0; - while (js.compiling) { - // Jit breakpoints are quite fast, so let's do them in release too. - CheckJitBreakpoint(GetCompilerPC(), 0); - - MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); - js.downcountAmount += MIPSGetInstructionCycleEstimate(inst); - - MIPSCompileOp(inst, this); - - if (js.afterOp & JitState::AFTER_CORE_STATE) { - // CORE_RUNNING is <= CORE_NEXTFRAME. - if (RipAccessible((const void *)&coreState)) { - CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible - } else { - MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState)); - CMP(32, MatR(RAX), Imm32(CORE_NEXTFRAME)); - } - FixupBranch skipCheck = J_CC(CC_LE, true); - // All cases of AFTER_CORE_STATE should update PC. We don't update here. - RegCacheState state; - GetStateAndFlushAll(state); - WriteSyscallExit(); - - SetJumpTarget(skipCheck); - // If we didn't jump, we can keep our regs as they were. - RestoreState(state); - - js.afterOp = JitState::AFTER_NONE; - } - - js.compilerPC += 4; - js.numInstructions++; - - if (jo.Disabled(JitDisable::REGALLOC_GPR)) { - gpr.Flush(); - } - if (jo.Disabled(JitDisable::REGALLOC_FPR)) { - fpr.Flush(); - FlushPrefixV(); - } - - // Safety check, in case we get a bunch of really large jit ops without a lot of branching. - if (GetSpaceLeft() < 0x800 || js.numInstructions >= JitBlockCache::MAX_BLOCK_INSTRUCTIONS) { - FlushAll(); - WriteExit(GetCompilerPC(), js.nextExit++); - js.compiling = false; - } - } - - b->codeSize = (u32)(GetCodePtr() - b->normalEntry); - NOP(); - AlignCode4(); - - b->originalSize = js.numInstructions; -} - -bool Jit::DescribeCodePtr(const u8 *ptr, std::string &name) { - if (ptr == applyRoundingMode) - name = "applyRoundingMode"; - else if (ptr == dispatcher) - name = "dispatcher"; - else if (ptr == dispatcherInEAXNoCheck) - name = "dispatcher (PC in EAX)"; - else if (ptr == dispatcherNoCheck) - name = "dispatcherNoCheck"; - else if (ptr == dispatcherCheckCoreState) - name = "dispatcherCheckCoreState"; - else if (ptr == enterDispatcher) - name = "enterDispatcher"; - else if (ptr == restoreRoundingMode) - name = "restoreRoundingMode"; - else if (ptr == crashHandler) - name = "crashHandler"; - else { - u32 jitAddr = blocks.GetAddressFromBlockPtr(ptr); - - // Returns 0 when it's valid, but unknown. - if (jitAddr == 0) { - name = "UnknownOrDeletedBlock"; - } else if (jitAddr != (u32)-1) { - char temp[1024]; - const std::string label = g_symbolMap ? g_symbolMap->GetDescription(jitAddr) : ""; - if (!label.empty()) - snprintf(temp, sizeof(temp), "%08x_%s", jitAddr, label.c_str()); - else - snprintf(temp, sizeof(temp), "%08x", jitAddr); - name = temp; - } else if (IsInSpace(ptr)) { - if (ptr < endOfPregeneratedCode) { - name = "PreGenCode"; - } else { - name = "Unknown"; - } - } else if (thunks.IsInSpace(ptr)) { - name = "Thunk"; - } else if (safeMemFuncs.IsInSpace(ptr)) { - name = "JitSafeMem"; - } else { - // Not anywhere in jit, then. - return false; - } - } - // If we got here, one of the above cases matched. - return true; -} - -void Jit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) { - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE); - } - XEmitter emit(exitPoint); - // Okay, this is a bit ugly, but we check here if it already has a JMP. - // That means it doesn't have a full exit to pad with INT 3. - bool prelinked = *emit.GetCodePointer() == 0xE9; - emit.JMP(checkedEntry, true); - if (!prelinked) { - ptrdiff_t actualSize = emit.GetWritableCodePtr() - exitPoint; - int pad = JitBlockCache::GetBlockExitSize() - (int)actualSize; - for (int i = 0; i < pad; ++i) { - emit.INT3(); - } - } - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_EXEC); - } -} - -void Jit::UnlinkBlock(u8 *checkedEntry, u32 originalAddress) { - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_WRITE); - } - // Send anyone who tries to run this block back to the dispatcher. - // Not entirely ideal, but .. pretty good. - // Spurious entrances from previously linked blocks can only come through checkedEntry - XEmitter emit(checkedEntry); - emit.MOV(32, MIPSSTATE_VAR(pc), Imm32(originalAddress)); - emit.JMP(MIPSComp::jit->GetDispatcher(), true); - if (PlatformIsWXExclusive()) { - ProtectMemoryPages(checkedEntry, 16, MEM_PROT_READ | MEM_PROT_EXEC); - } -} - -void Jit::Comp_ReplacementFunc(MIPSOpcode op) { - // We get here if we execute the first instruction of a replaced function. This means - // that we do need to return to RA. - - // Inlined function calls (caught in jal) are handled differently. - - // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. - if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { - Comp_Generic(op); - return; - } - - int index = op.encoding & MIPS_REPLACEMENT_INDEX_MASK; - - const ReplacementTableEntry *entry = GetReplacementFunc(index); - if (!entry) { - ERROR_LOG_REPORT_ONCE(replFunc, Log::HLE, "Invalid replacement op %08x at %08x", op.encoding, js.compilerPC); - return; - } - - u32 funcSize = g_symbolMap->GetFunctionSize(GetCompilerPC()); - bool disabled = (entry->flags & REPFLAG_DISABLED) != 0; - if (!disabled && funcSize != SymbolMap::INVALID_ADDRESS && funcSize > sizeof(u32)) { - // We don't need to disable hooks, the code will still run. - if ((entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) == 0) { - // Any breakpoint at the func entry was already tripped, so we can still run the replacement. - // That's a common case - just to see how often the replacement hits. - disabled = g_breakpoints.RangeContainsBreakPoint(GetCompilerPC() + sizeof(u32), funcSize - sizeof(u32)); - } - } - - Memory::Opcode origInstruction = Memory::Read_Instruction(GetCompilerPC()); - if (disabled) { - MIPSCompileOp(origInstruction, this); - } else if (entry->jitReplaceFunc) { - MIPSReplaceFunc repl = entry->jitReplaceFunc; - int cycles = (this->*repl)(); - - if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { - // Compile the original instruction at this address. We ignore cycles for hooks. - MIPSCompileOp(origInstruction, this); - } else { - FlushAll(); - MOV(32, R(ECX), MIPSSTATE_VAR(r[MIPS_REG_RA])); - js.downcountAmount += cycles; - WriteExitDestInReg(ECX); - js.compiling = false; - } - } else if (entry->replaceFunc) { - FlushAll(); - - // Standard function call, nothing fancy. - // The function returns the number of cycles it took in EAX. - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - RestoreRoundingMode(); - ABI_CallFunction(entry->replaceFunc); - - if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { - // Compile the original instruction at this address. We ignore cycles for hooks. - ApplyRoundingMode(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); - } else { - CMP(32, R(EAX), Imm32(0)); - FixupBranch positive = J_CC(CC_GE); - - MOV(32, R(ECX), MIPSSTATE_VAR(pc)); - ADD(32, MIPSSTATE_VAR(downcount), R(EAX)); - FixupBranch done = J(); - - SetJumpTarget(positive); - MOV(32, R(ECX), MIPSSTATE_VAR(r[MIPS_REG_RA])); - SUB(32, MIPSSTATE_VAR(downcount), R(EAX)); - - SetJumpTarget(done); - ApplyRoundingMode(); - // Need to set flags again, ApplyRoundingMode destroyed them (and EAX.) - SUB(32, MIPSSTATE_VAR(downcount), Imm8(0)); - WriteExitDestInReg(ECX); - js.compiling = false; - } - } else { - ERROR_LOG(Log::HLE, "Replacement function %s has neither jit nor regular impl", entry->name); - } -} - -void Jit::Comp_Generic(MIPSOpcode op) { - FlushAll(); - MIPSInterpretFunc func = MIPSGetInterpretFunc(op); - _dbg_assert_msg_((MIPSGetInfo(op) & DELAYSLOT) == 0, "Cannot use interpreter for branch ops."); - - const MIPSInfo info = MIPSGetInfo(op); - if (func) - { - // Interpreted ops run in the guest's rounding mode, like on the interpreter. A syscall is host code - // (like a compiled one), and only ctc1 to fcr31 changes the mode, so only those need it switched. - const bool isSyscall = (info & IS_SYSCALL) != 0; - const bool setsMode = MIPS_IS_CTC1_FCR31(op.encoding); - if (isSyscall) - RestoreRoundingMode(); - if (setsMode) - UpdateRoundingMode(); - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - if (USE_JIT_MISSMAP) - ABI_CallFunctionC(&JitLogMiss, op.encoding); - else - ABI_CallFunctionC(&MIPSInterpretTrampoline, op.encoding); - if (setsMode) - RestoreRoundingMode(true); - if (isSyscall || setsMode) - ApplyRoundingMode(setsMode); - } else { - // These are basically always due to some kind of crash or corruption now. - ERROR_LOG(Log::JIT, "Trying to compile instruction %08x that can't be interpreted", op.encoding); - } - - if ((info & IS_VFPU) != 0 && (info & VFPU_NO_PREFIX) == 0) - { - // If it does eat them, it'll happen in MIPSCompileOp(). - if ((info & OUT_EAT_PREFIX) == 0) - js.PrefixUnknown(); - - // Even if DISABLE'd, we want to set this flag so we overwrite. - if ((info & OUT_VFPU_PREFIX) != 0) - js.blockWrotePrefixes = true; - } -} - -static void HitInvalidBranch(uint32_t dest) { - Core_ExecException(dest, currentMIPS->pc, ExecExceptionType::JUMP); -} - -void Jit::WriteExit(u32 destination, int exit_num) { - _assert_msg_(exit_num < MAX_JIT_BLOCK_EXITS, "Expected a valid exit_num. dest=%08x", destination); - - if (!Memory::IsValidAddress(destination) || (destination & 3) != 0) { - ERROR_LOG_REPORT(Log::JIT, "Trying to write block exit to illegal destination %08x: pc = %08x", destination, currentMIPS->pc); - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - ABI_CallFunctionC(&HitInvalidBranch, destination); - js.afterOp |= JitState::AFTER_CORE_STATE; - } - // If we need to verify coreState, we may not jump yet. - if (js.afterOp & JitState::AFTER_CORE_STATE) { - // CORE_RUNNING is <= CORE_NEXTFRAME. - if (RipAccessible((const void *)&coreState)) { - CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible - } else { - MOV(PTRBITS, R(RAX), ImmPtr((const void *)&coreState)); - CMP(32, MatR(RAX), Imm32(CORE_NEXTFRAME)); - } - FixupBranch skipCheck = J_CC(CC_LE); - // All cases of AFTER_CORE_STATE should update PC. We don't update here. - WriteSyscallExit(); - SetJumpTarget(skipCheck); - } - - WriteDowncount(); - - //If nobody has taken care of this yet (this can be removed when all branches are done) - JitBlock *b = js.curBlock; - b->exitAddress[exit_num] = destination; - b->exitPtrs[exit_num] = GetWritableCodePtr(); - - // Link opportunity! - int block = blocks.GetBlockNumberFromStartAddress(destination); - if (block >= 0 && jo.enableBlocklink) { - // It exists! Joy of joy! - JMP(blocks.GetBlock(block)->checkedEntry, true); - b->linkStatus[exit_num] = true; - } else { - // No blocklinking. - MOV(32, MIPSSTATE_VAR(pc), Imm32(destination)); - JMP(dispatcher, true); - - // Normally, exits are 15 bytes (MOV + &pc + dest + JMP + dest) on 64 or 32 bit. - // But just in case we somehow optimized, pad. - ptrdiff_t actualSize = GetWritableCodePtr() - b->exitPtrs[exit_num]; - int pad = JitBlockCache::GetBlockExitSize() - (int)actualSize; - for (int i = 0; i < pad; ++i) { - INT3(); - } - } -} - -static u32 IsValidJumpTarget(uint32_t addr) { - if (Memory::IsValidAddress(addr) && (addr & 3) == 0) - return 1; - return 0; -} - -static void HitInvalidJumpReg(uint32_t source) { - Core_ExecException(currentMIPS->pc, source, ExecExceptionType::JUMP); - currentMIPS->pc = source + 8; -} - -void Jit::WriteExitDestInReg(X64Reg reg) { - // If we need to verify coreState, we may not jump yet. - if (js.afterOp & JitState::AFTER_CORE_STATE) { - // CORE_RUNNING is <= CORE_NEXTFRAME. - if (RipAccessible((const void *)&coreState)) { - CMP(32, M(&coreState), Imm32(CORE_NEXTFRAME)); // rip accessible - } else { - X64Reg temp = reg == RAX ? RDX : RAX; - MOV(PTRBITS, R(temp), ImmPtr((const void *)&coreState)); - CMP(32, MatR(temp), Imm32(CORE_NEXTFRAME)); - } - FixupBranch skipCheck = J_CC(CC_LE); - // All cases of AFTER_CORE_STATE should update PC. We don't update here. - WriteSyscallExit(); - SetJumpTarget(skipCheck); - } - - MOV(32, MIPSSTATE_VAR(pc), R(reg)); - WriteDowncount(); - - // Validate the jump to avoid a crash? - if (!g_Config.bFastMemory) { - CMP(32, R(reg), Imm32(PSP_GetKernelMemoryBase())); - FixupBranch tooLow = J_CC(CC_B); - CMP(32, R(reg), Imm32(PSP_GetUserMemoryEnd())); - FixupBranch tooHigh = J_CC(CC_AE); - - // Need to set neg flag again. - SUB(32, MIPSSTATE_VAR(downcount), Imm8(0)); - if (reg == EAX) - J_CC(CC_NS, dispatcherInEAXNoCheck, true); - JMP(dispatcher, true); - - SetJumpTarget(tooLow); - SetJumpTarget(tooHigh); - - ABI_CallFunctionA((const void *)&IsValidJumpTarget, R(reg)); - - // If we're ignoring, coreState didn't trip - so trip it now. - CMP(32, R(EAX), Imm32(0)); - FixupBranch skip = J_CC(CC_NE); - ABI_CallFunctionC(&HitInvalidJumpReg, GetCompilerPC()); - SetJumpTarget(skip); - - SUB(32, MIPSSTATE_VAR(downcount), Imm8(0)); - JMP(dispatcherCheckCoreState, true); - } else if (reg == EAX) { - J_CC(CC_NS, dispatcherInEAXNoCheck, true); - JMP(dispatcher, true); - } else { - JMP(dispatcher, true); - } -} - -void Jit::WriteSyscallExit() { - WriteDowncount(); - JMP(dispatcherCheckCoreState, true); -} - -bool Jit::CheckJitBreakpoint(u32 addr, int downcountOffset) { - if (g_breakpoints.NeedsBreakCheckAt(addr)) { - SaveFlags(); - FlushAll(); - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - RestoreRoundingMode(); - ABI_CallFunctionC(&JitBreakpoint, addr); - - // If 0, the conditional breakpoint wasn't taken. - CMP(32, R(EAX), Imm32(0)); - FixupBranch skip = J_CC(CC_Z); - WriteDowncount(downcountOffset); - ApplyRoundingMode(); - // Just to fix the stack. - LoadFlags(); - JMP(dispatcherCheckCoreState, true); - SetJumpTarget(skip); - - ApplyRoundingMode(); - LoadFlags(); - return true; - } - - return false; -} - -void Jit::CheckMemoryBreakpoint(int instructionOffset, MIPSGPReg rs, int offset) { - if (!g_breakpoints.HasMemChecks()) - return; - - int totalInstructionOffset = instructionOffset + (js.inDelaySlot ? 1 : 0); - uint32_t checkedPC = GetCompilerPC() + totalInstructionOffset * 4; - int size = MIPSAnalyst::OpMemoryAccessSize(checkedPC); - bool isWrite = MIPSAnalyst::IsOpMemoryWrite(checkedPC); - - // 0 because we normally execute before increasing. - int downcountOffset = js.inDelaySlot ? -2 : -1; - // TODO: In likely branches, downcount will be incorrect. This might make resume fail. - if (js.downcountAmount + downcountOffset < 0) { - downcountOffset = 0; - } - - if (gpr.IsImm(rs)) { - uint32_t iaddr = gpr.GetImm(rs) + offset; - MemCheck check; - if (g_breakpoints.GetMemCheckInRange(iaddr, size, &check)) { - if (!(check.cond & MEMCHECK_READ) && !isWrite) - return; - if (!(check.cond & MEMCHECK_WRITE) && isWrite) - return; - - // We need to flush, or conditions and log expressions will see old register values. - FlushAll(); - - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - CallProtectedFunction(&JitMemCheck, iaddr, checkedPC); - - CMP(32, R(RAX), Imm32(0)); - FixupBranch skipCheck = J_CC(CC_E); - WriteDowncount(downcountOffset); - JMP(dispatcherCheckCoreState, true); - - SetJumpTarget(skipCheck); - } - } else { - const auto memchecks = g_breakpoints.GetMemCheckRanges(isWrite); - bool possible = !memchecks.empty(); - if (!possible) - return; - - gpr.Lock(rs); - gpr.MapReg(rs, true, false); - LEA(32, RAX, MDisp(gpr.RX(rs), offset)); - gpr.UnlockAll(); - - // We need to flush, or conditions and log expressions will see old register values. - FlushAll(); - - std::vector hitChecks; - hitChecks.reserve(memchecks.size()); - for (auto it = memchecks.begin(), end = memchecks.end(); it != end; ++it) { - if (it->end != 0) { - CMP(32, R(RAX), Imm32(it->start - size)); - FixupBranch skipNext = J_CC(CC_BE); - - CMP(32, R(RAX), Imm32(it->end)); - hitChecks.push_back(J_CC(CC_B, true)); - - SetJumpTarget(skipNext); - } else { - CMP(32, R(RAX), Imm32(it->start)); - hitChecks.push_back(J_CC(CC_E, true)); - } - } - - FixupBranch noHits = J(true); - - // Okay, now land any hit here. - for (auto &fixup : hitChecks) - SetJumpTarget(fixup); - hitChecks.clear(); - - MOV(32, MIPSSTATE_VAR(pc), Imm32(GetCompilerPC())); - CallProtectedFunction(&JitMemCheck, R(RAX), checkedPC); - - CMP(32, R(RAX), Imm32(0)); - FixupBranch skipCheck = J_CC(CC_E); - WriteDowncount(downcountOffset); - JMP(dispatcherCheckCoreState, true); - - SetJumpTarget(skipCheck); - SetJumpTarget(noHits); - } -} - -void Jit::CallProtectedFunction(const void *func, const OpArg &arg1) { - // We don't regcache RCX, so the below is safe (and also faster, maybe branch prediction?) - ABI_CallFunctionA(thunks.ProtectFunction(func, 1), arg1); -} - -void Jit::CallProtectedFunction(const void *func, const OpArg &arg1, const OpArg &arg2) { - // We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?) - ABI_CallFunctionAA(thunks.ProtectFunction(func, 2), arg1, arg2); -} - -void Jit::CallProtectedFunction(const void *func, const u32 arg1, const u32 arg2) { - // We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?) - ABI_CallFunctionCC(thunks.ProtectFunction(func, 2), arg1, arg2); -} - -void Jit::CallProtectedFunction(const void *func, const OpArg &arg1, const u32 arg2) { - // We don't regcache RCX/RDX, so the below is safe (and also faster, maybe branch prediction?) - ABI_CallFunctionAC(thunks.ProtectFunction(func, 2), arg1, arg2); -} - -#if PPSSPP_ARCH(AMD64) -void Jit::CallProtectedLeaf(const void *func) { - // The registers a call may clobber, apart from RAX and XMM0-1, which are scratch. -#ifdef _WIN32 - static const X64Reg callerSavedGPRs[] = { RCX, RDX, R8, R9, R10, R11 }; - const int lastXMM = 5; - const int shadowSpace = 32; -#else - static const X64Reg callerSavedGPRs[] = { RCX, RDX, R8, R9, R10, R11, RSI, RDI }; - const int lastXMM = 15; - const int shadowSpace = 0; -#endif - X64Reg xmms[16]; - X64Reg gprs[ARRAY_SIZE(callerSavedGPRs)]; - int numXMMs = 0, numGPRs = 0; - for (int i = 2; i <= lastXMM; i++) { - if (fpr.IsXRegInUse((X64Reg)(XMM0 + i))) - xmms[numXMMs++] = (X64Reg)(XMM0 + i); - } - for (X64Reg reg : callerSavedGPRs) { - if (gpr.IsXRegInUse(reg)) - gprs[numGPRs++] = reg; - } - - // The JIT runs with RSP 16-byte aligned, so this keeps it aligned for the call. - const int gprBase = shadowSpace + numXMMs * 16; - const int frameSize = (gprBase + numGPRs * 8 + 15) & ~15; - if (frameSize) - SUB(64, R(RSP), Imm32(frameSize)); - for (int i = 0; i < numXMMs; i++) - MOVAPS(MDisp(RSP, shadowSpace + i * 16), xmms[i]); - for (int i = 0; i < numGPRs; i++) - MOV(64, MDisp(RSP, gprBase + i * 8), R(gprs[i])); - ABI_CallFunction(func); - for (int i = 0; i < numXMMs; i++) - MOVAPS(xmms[i], MDisp(RSP, shadowSpace + i * 16)); - for (int i = 0; i < numGPRs; i++) - MOV(64, R(gprs[i]), MDisp(RSP, gprBase + i * 8)); - if (frameSize) - ADD(64, R(RSP), Imm32(frameSize)); -} -#endif - -void Jit::Comp_DoNothing(MIPSOpcode op) { } - -} // namespace - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/Jit.h b/Core/MIPS/x86/Jit.h deleted file mode 100644 index 4f255b4d50..0000000000 --- a/Core/MIPS/x86/Jit.h +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - - -// TODO: Implement https://github.com/dolphin-emu/dolphin/pull/1025/commits/b597ec3e081a289d9ac782586617a876535183d6 . - -#pragma once - -#include "Common/CommonTypes.h" -#include "Common/Thunk.h" -#include "Common/x64Emitter.h" - -#include "Common/x64Emitter.h" -#include "Core/MIPS/JitCommon/JitBlockCache.h" -#include "Core/MIPS/JitCommon/JitState.h" -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/x86/JitSafeMem.h" -#include "Core/MIPS/x86/RegCache.h" -#include "Core/MIPS/x86/RegCacheFPU.h" - -class PointerWrap; - -namespace MIPSComp { - -struct RegCacheState { - GPRRegCacheState gpr; - FPURegCacheState fpr; -}; - -class Jit : public Gen::XCodeBlock, public JitInterface, public MIPSFrontendInterface { -public: - Jit(MIPSState *mipsState); - ~Jit(); - - const JitOptions &GetJitOptions() { return jo; } - - void DoState(PointerWrap &p) override; - - // Compiled ops should ignore delay slots - // the compiler will take care of them by itself - // OR NOT - void Comp_Generic(MIPSOpcode op) override; - - void RunLoopUntil(u64 globalticks) override; - - void Compile(u32 em_address) override; // Compiles a block at current MIPS PC - void DoJit(u32 em_address, JitBlock *b); - - const u8 *GetCrashHandler() const override { return crashHandler; } - bool CodeInRange(const u8 *ptr) const override { return IsInSpace(ptr); } - bool DescribeCodePtr(const u8 *ptr, std::string &name) override; - - void Comp_ReplacementFunc(MIPSOpcode op) override; - - // Ops - void Comp_ITypeMem(MIPSOpcode op) override; - void Comp_StoreSync(MIPSOpcode op) override; - void Comp_Cache(MIPSOpcode op) override; - - void Comp_RelBranch(MIPSOpcode op) override; - void Comp_RelBranchRI(MIPSOpcode op) override; - void Comp_FPUBranch(MIPSOpcode op) override; - void Comp_FPULS(MIPSOpcode op) override; - void Comp_FPUComp(MIPSOpcode op) override; - void Comp_Jump(MIPSOpcode op) override; - void Comp_JumpReg(MIPSOpcode op) override; - void Comp_Syscall(MIPSOpcode op) override; - void Comp_Break(MIPSOpcode op) override; - - void Comp_IType(MIPSOpcode op) override; - void Comp_RType2(MIPSOpcode op) override; - void Comp_RType3(MIPSOpcode op) override; - void Comp_ShiftType(MIPSOpcode op) override; - void Comp_Allegrex(MIPSOpcode op) override; - void Comp_Allegrex2(MIPSOpcode op) override; - void Comp_VBranch(MIPSOpcode op) override; - void Comp_MulDivType(MIPSOpcode op) override; - void Comp_Special3(MIPSOpcode op) override; - - void Comp_FPU3op(MIPSOpcode op) override; - void Comp_FPU2op(MIPSOpcode op) override; - void Comp_mxc1(MIPSOpcode op) override; - - void Comp_SV(MIPSOpcode op) override; - void Comp_SVQ(MIPSOpcode op) override; - void Comp_VPFX(MIPSOpcode op) override; - void Comp_VVectorInit(MIPSOpcode op) override; - void Comp_VMatrixInit(MIPSOpcode op) override; - void Comp_VDot(MIPSOpcode op) override; - void Comp_VecDo3(MIPSOpcode op) override; - void Comp_VV2Op(MIPSOpcode op) override; - void Comp_Mftv(MIPSOpcode op) override; - void Comp_Vmfvc(MIPSOpcode op) override; - void Comp_Vmtvc(MIPSOpcode op) override; - void Comp_Vmmov(MIPSOpcode op) override; - void Comp_VScl(MIPSOpcode op) override; - void Comp_Vmmul(MIPSOpcode op) override; - void Comp_Vmscl(MIPSOpcode op) override; - void Comp_Vtfm(MIPSOpcode op) override; - void Comp_VHdp(MIPSOpcode op) override; - void Comp_VCrs(MIPSOpcode op) override; - void Comp_VDet(MIPSOpcode op) override; - void Comp_Vi2x(MIPSOpcode op) override; - void Comp_Vx2i(MIPSOpcode op) override; - void Comp_Vf2i(MIPSOpcode op) override; - void Comp_Vi2f(MIPSOpcode op) override; - void Comp_Vh2f(MIPSOpcode op) override; - void Comp_Vcst(MIPSOpcode op) override; - void Comp_Vhoriz(MIPSOpcode op) override; - void Comp_VRot(MIPSOpcode op) override; - void Comp_VIdt(MIPSOpcode op) override; - void Comp_Vcmp(MIPSOpcode op) override; - void Comp_Vcmov(MIPSOpcode op) override; - void Comp_Viim(MIPSOpcode op) override; - void Comp_Vfim(MIPSOpcode op) override; - void Comp_VCrossQuat(MIPSOpcode op) override; - void Comp_Vsgn(MIPSOpcode op) override; - void Comp_Vocp(MIPSOpcode op) override; - void Comp_ColorConv(MIPSOpcode op) override; - void Comp_Vbfy(MIPSOpcode op) override; - - void Comp_DoNothing(MIPSOpcode op) override; - - int Replace_fabsf() override; - - void ApplyPrefixST(u8 *vregs, u32 prefix, VectorSize sz); - void ApplyPrefixD(const u8 *vregs, VectorSize sz); - void GetVectorRegsPrefixS(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixSFlag & JitState::PREFIX_KNOWN); - GetVectorRegs(regs, sz, vectorReg); - ApplyPrefixST(regs, js.prefixS, sz); - } - void GetVectorRegsPrefixT(u8 *regs, VectorSize sz, int vectorReg) { - _assert_(js.prefixTFlag & JitState::PREFIX_KNOWN); - GetVectorRegs(regs, sz, vectorReg); - ApplyPrefixST(regs, js.prefixT, sz); - } - void GetVectorRegsPrefixD(u8 *regs, VectorSize sz, int vectorReg); - void EatPrefix() override { js.EatPrefix(); } - - void RestoreRoundingMode(bool force = false); - void ApplyRoundingMode(bool force = false); - void UpdateRoundingMode(u32 fcr31 = -1); - - JitBlockCache *GetBlockCache() override { return &blocks; } - JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - - void ClearCache() override; - void InvalidateCacheAt(u32 em_address, int length = 4) override { - if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { - blocks.InvalidateICache(em_address, length); - } - } - void UpdateFCR31() override; - - const u8 *GetDispatcher() const override { - return dispatcher; - } - - void LinkBlock(u8 *exitPoint, const u8 *checkedEntry) override; - void UnlinkBlock(u8 *checkedEntry, u32 originalAddress) override; - - const u8 *GetCodeBase() const override { return GetCodePtr(); } - -private: - void GenerateFixedCode(JitOptions &jo); - void GetStateAndFlushAll(RegCacheState &state); - void RestoreState(const RegCacheState& state); - void FlushAll(); - void FlushPrefixV(); - void WriteDowncount(int offset = 0); - - u32 GetCompilerPC(); - // See CompileDelaySlotFlags for flags. - void CompileDelaySlot(int flags, RegCacheState *state = NULL); - void CompileDelaySlot(int flags, RegCacheState &state) { - CompileDelaySlot(flags, &state); - } - void EatInstruction(MIPSOpcode op); - MIPSOpcode GetOffsetInstruction(int offset); - - void WriteExit(u32 destination, int exit_num); - void WriteExitDestInReg(Gen::X64Reg reg); - -// void WriteRfiExitDestInEAX(); - void WriteSyscallExit(); - bool CheckJitBreakpoint(u32 addr, int downcountOffset); - void CheckMemoryBreakpoint(int instructionOffset, MIPSGPReg rs, int offset); - - // Utility compilation functions - void BranchFPFlag(MIPSOpcode op, Gen::CCFlags cc, bool likely); - void BranchVFPUFlag(MIPSOpcode op, Gen::CCFlags cc, bool likely); - void BranchRSZeroComp(MIPSOpcode op, Gen::CCFlags cc, bool andLink, bool likely); - void BranchRSRTComp(MIPSOpcode op, Gen::CCFlags cc, bool likely); - void BranchLog(MIPSOpcode op); - void BranchLogExit(MIPSOpcode op, u32 dest, bool useEAX); - - // Utilities to reduce duplicated code - void CompImmLogic(MIPSOpcode op, void (XEmitter::*arith)(int, const Gen::OpArg &, const Gen::OpArg &)); - void CompTriArith(MIPSOpcode op, void (XEmitter::*arith)(int, const Gen::OpArg &, const Gen::OpArg &), u32 (*doImm)(const u32, const u32), bool invertResult = false); - void CompShiftImm(MIPSOpcode op, void (XEmitter::*shift)(int, Gen::OpArg, Gen::OpArg), u32 (*doImm)(const u32, const u32)); - void CompShiftVar(MIPSOpcode op, void (XEmitter::*shift)(int, Gen::OpArg, Gen::OpArg), u32 (*doImm)(const u32, const u32)); - void CompITypeMemRead(MIPSOpcode op, u32 bits, void (XEmitter::*mov)(int, int, Gen::X64Reg, Gen::OpArg), const void *safeFunc); - template - void CompITypeMemRead(MIPSOpcode op, u32 bits, void (XEmitter::*mov)(int, int, Gen::X64Reg, Gen::OpArg), T (*safeFunc)(u32 addr)) { - CompITypeMemRead(op, bits, mov, (const void *)safeFunc); - } - void CompITypeMemWrite(MIPSOpcode op, u32 bits, const void *safeFunc, bool makeRTWritable = false); - template - void CompITypeMemWrite(MIPSOpcode op, u32 bits, void (*safeFunc)(T val, u32 addr)) { - CompITypeMemWrite(op, bits, (const void *)safeFunc); - } - void CompITypeMemUnpairedLR(MIPSOpcode op, bool isStore); - void CompITypeMemUnpairedLRInner(MIPSOpcode op, Gen::X64Reg shiftReg); - void CompBranchExits(Gen::CCFlags cc, u32 targetAddr, u32 notTakenAddr, const BranchInfo &branchInfo); - void CompBranchExit(bool taken, u32 targetAddr, u32 notTakenAddr, const BranchInfo &branchInfo); - static Gen::CCFlags FlipCCFlag(Gen::CCFlags flag); - static Gen::CCFlags SwapCCFlag(Gen::CCFlags flag); - - void CopyFPReg(Gen::X64Reg dst, Gen::OpArg src); - void CompFPTriArith(MIPSOpcode op, void (XEmitter::*arith)(Gen::X64Reg reg, Gen::OpArg), bool orderMatters); - void CompFPComp(int lhs, int rhs, u8 compare, bool allowNaN = false); - void CompVrotShuffle(u8 *dregs, int imm, int n, bool negSin); - - void CallProtectedFunction(const void *func, const Gen::OpArg &arg1); - void CallProtectedFunction(const void *func, const Gen::OpArg &arg1, const Gen::OpArg &arg2); - void CallProtectedFunction(const void *func, const Gen::OpArg &arg1, const u32 arg2); - void CallProtectedFunction(const void *func, const u32 arg1, const u32 arg2); -#if PPSSPP_ARCH(AMD64) - // A lighter CallProtectedFunction for a function that leaves MXCSR alone: saves only the - // caller-saved registers the caches are using. Set up the arguments first - the argument - // registers are never cached. - void CallProtectedLeaf(const void *func); -#endif - - template - void CallProtectedFunction(Tr (*func)(T1), const Gen::OpArg &arg1) { - CallProtectedFunction((const void *)func, arg1); - } - - template - void CallProtectedFunction(Tr (*func)(T1, T2), const Gen::OpArg &arg1, const Gen::OpArg &arg2) { - CallProtectedFunction((const void *)func, arg1, arg2); - } - - template - void CallProtectedFunction(Tr(*func)(T1, T2), const Gen::OpArg &arg1, const u32 arg2) { - CallProtectedFunction((const void *)func, arg1, arg2); - } - - template - void CallProtectedFunction(Tr(*func)(T1, T2), const u32 arg1, const u32 arg2) { - CallProtectedFunction((const void *)func, arg1, arg2); - } - - bool PredictTakeBranch(u32 targetAddr, bool likely); - u32 GetBlockShadowHookValue() const override { - return (u32)(blockShadowHook - GetBasePtr()); - } - bool IsAtDispatchFetch(const u8 *codePtr) const override { - return codePtr == dispatcherFetch; - } - - void SaveFlags(); - void LoadFlags(); - - JitBlockCache blocks; - JitOptions jo; - JitState js; - - GPRRegCache gpr; - FPURegCache fpr; - - ThunkManager thunks; - JitSafeMemFuncs safeMemFuncs; - - MIPSState *mips_; - - - const u8 *enterDispatcher; - - const u8 *outerLoop; - const u8 *dispatcher; - const u8 *dispatcherCheckCoreState; - const u8 *dispatcherNoCheck; - const u8 *dispatcherInEAXNoCheck; - const u8 *dispatcherFetch; - - const u8 *restoreRoundingMode; - const u8 *applyRoundingMode; - - const u8 *endOfPregeneratedCode; - - const u8 *crashHandler; - const u8 *blockShadowHook; - - friend class JitSafeMem; - friend class JitSafeMemFuncs; -}; - -} // namespace MIPSComp diff --git a/Core/MIPS/x86/JitSafeMem.cpp b/Core/MIPS/x86/JitSafeMem.cpp deleted file mode 100644 index 91ef1200b4..0000000000 --- a/Core/MIPS/x86/JitSafeMem.cpp +++ /dev/null @@ -1,488 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" - -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include "Core/Config.h" -#include "Core/Debugger/Breakpoints.h" -#include "Core/MemMap.h" -#include "Core/MIPS/JitCommon/JitCommon.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/JitSafeMem.h" -#include "Core/System.h" - -namespace MIPSComp -{ -using namespace Gen; -using namespace X64JitConstants; - -JitSafeMem::JitSafeMem(Jit *jit, MIPSGPReg raddr, s32 offset, u32 alignMask) - : jit_(jit), raddr_(raddr), offset_(offset), needsCheck_(false), needsSkip_(false), alignMask_(alignMask) -{ - // Mask out the kernel RAM bit, because we'll end up with a negative offset to MEMBASEREG. - if (jit_->gpr.IsImm(raddr_)) - iaddr_ = (jit_->gpr.GetImm(raddr_) + offset_) & 0x7FFFFFFF; - else - iaddr_ = (u32) -1; - - fast_ = g_Config.bFastMemory || raddr == MIPS_REG_SP; - - // If raddr_ is going to get loaded soon, load it now for more optimal code. - // We assume that it was already locked. - const int LOOKAHEAD_OPS = 3; - if (!jit_->gpr.R(raddr_).IsImm() && MIPSAnalyst::IsRegisterUsed(raddr_, jit_->GetCompilerPC() + 4, LOOKAHEAD_OPS)) - jit_->gpr.MapReg(raddr_, true, false); -} - -bool JitSafeMem::PrepareWrite(OpArg &dest, int size) -{ - size_ = size; - // If it's an immediate, we can do the write if valid. - if (iaddr_ != (u32) -1) - { - if (ImmValid()) - { - u32 addr = (iaddr_ & alignMask_); -#ifdef MASKED_PSP_MEMORY - addr &= Memory::MEMVIEW32_MASK; -#endif - -#if PPSSPP_ARCH(32BIT) - dest = M(Memory::base + addr); // 32-bit only -#else - dest = MDisp(MEMBASEREG, addr); -#endif - return true; - } - else - return false; - } - // Otherwise, we always can do the write (conditionally.) - else - dest = PrepareMemoryOpArg(MEM_WRITE); - return true; -} - -bool JitSafeMem::PrepareRead(OpArg &src, int size) -{ - size_ = size; - if (iaddr_ != (u32) -1) - { - if (ImmValid()) - { - u32 addr = (iaddr_ & alignMask_); -#ifdef MASKED_PSP_MEMORY - addr &= Memory::MEMVIEW32_MASK; -#endif - -#if PPSSPP_ARCH(32BIT) - src = M(Memory::base + addr); // 32-bit only -#else - src = MDisp(MEMBASEREG, addr); -#endif - return true; - } - else - return false; - } - else - src = PrepareMemoryOpArg(MEM_READ); - return true; -} - -OpArg JitSafeMem::NextFastAddress(int suboffset) -{ - if (iaddr_ != (u32) -1) - { - u32 addr = (iaddr_ + suboffset) & alignMask_; -#ifdef MASKED_PSP_MEMORY - addr &= Memory::MEMVIEW32_MASK; -#endif - -#if PPSSPP_ARCH(32BIT) - return M(Memory::base + addr); // 32-bit only -#else - return MDisp(MEMBASEREG, addr); -#endif - } - - _dbg_assert_msg_((suboffset & alignMask_) == suboffset, "suboffset must be aligned"); - -#if PPSSPP_ARCH(32BIT) - return MDisp(xaddr_, (u32) Memory::base + offset_ + suboffset); -#else - return MComplex(MEMBASEREG, xaddr_, SCALE_1, offset_ + suboffset); -#endif -} - -OpArg JitSafeMem::PrepareMemoryOpArg(MemoryOpType type) -{ - // We may not even need to move into EAX as a temporary. - bool needTemp = alignMask_ != 0xFFFFFFFF; - -#ifdef MASKED_PSP_MEMORY - bool needMask = true; // raddr_ != MIPS_REG_SP; // Commented out this speedhack due to low impact - // We always mask on 32 bit in fast memory mode. - needTemp = needTemp || (fast_ && needMask); -#endif - - if (jit_->gpr.R(raddr_).IsSimpleReg() && !needTemp) - { - jit_->gpr.MapReg(raddr_, true, false); - xaddr_ = jit_->gpr.RX(raddr_); - } - else - { - jit_->MOV(32, R(EAX), jit_->gpr.R(raddr_)); - xaddr_ = EAX; - } - - if (!fast_) - { - // Is it in physical ram? - jit_->CMP(32, R(xaddr_), Imm32(PSP_GetKernelMemoryBase() - offset_)); - tooLow_ = jit_->J_CC(CC_B); - jit_->CMP(32, R(xaddr_), Imm32(PSP_GetUserMemoryEnd() - offset_ - (size_ - 1))); - tooHigh_ = jit_->J_CC(CC_AE); - - // We may need to jump back up here. - safe_ = jit_->GetCodePtr(); - } - else - { -#ifdef MASKED_PSP_MEMORY - if (needMask) { - jit_->AND(32, R(EAX), Imm32(Memory::MEMVIEW32_MASK)); - } -#endif - } - - // TODO: This could be more optimal, but the common case is that we want xaddr_ not to include offset_. - // Since we need to align them after add, we add and subtract. - if (alignMask_ != 0xFFFFFFFF) - { - jit_->ADD(32, R(xaddr_), Imm32(offset_)); - jit_->AND(32, R(xaddr_), Imm32(alignMask_)); - jit_->SUB(32, R(xaddr_), Imm32(offset_)); - } - -#if PPSSPP_ARCH(32BIT) - return MDisp(xaddr_, (u32) Memory::base + offset_); -#else - return MComplex(MEMBASEREG, xaddr_, SCALE_1, offset_); -#endif -} - -void JitSafeMem::PrepareSlowAccess() -{ - // Skip the fast path (which the caller wrote just now.) - skip_ = jit_->J(true); - needsSkip_ = true; - jit_->SetJumpTarget(tooLow_); - jit_->SetJumpTarget(tooHigh_); - - // Might also be the scratchpad. - jit_->CMP(32, R(xaddr_), Imm32(PSP_GetScratchpadMemoryBase() - offset_)); - FixupBranch tooLow = jit_->J_CC(CC_B); - jit_->CMP(32, R(xaddr_), Imm32(PSP_GetScratchpadMemoryEnd() - offset_ - (size_ - 1))); - jit_->J_CC(CC_B, safe_); - jit_->SetJumpTarget(tooLow); -} - -bool JitSafeMem::PrepareSlowWrite() -{ - // If it's immediate, we only need a slow write on invalid. - if (iaddr_ != (u32) -1) - return !fast_ && !ImmValid(); - - if (!fast_) - { - PrepareSlowAccess(); - return true; - } - else - return false; -} - -void JitSafeMem::DoSlowWrite(const void *safeFunc, const OpArg &src, int suboffset) { - _dbg_assert_msg_(safeFunc != nullptr, "Safe func cannot be null"); - - if (iaddr_ != (u32) -1) - jit_->MOV(32, R(EAX), Imm32((iaddr_ + suboffset) & alignMask_)); - else - { - jit_->LEA(32, EAX, MDisp(xaddr_, offset_ + suboffset)); - if (alignMask_ != 0xFFFFFFFF) - jit_->AND(32, R(EAX), Imm32(alignMask_)); - } - -#if PPSSPP_ARCH(32BIT) - jit_->PUSH(EDX); -#endif - if (!src.IsSimpleReg(EDX)) { - jit_->MOV(32, R(EDX), src); - } - if (!g_Config.bIgnoreBadMemAccess) { - jit_->MOV(32, MIPSSTATE_VAR(pc), Imm32(jit_->GetCompilerPC())); - } - // This is a special jit-ABI'd function. - if (jit_->CanCALLDirect(safeFunc)) { - jit_->CALL(safeFunc); - } else { - // We can't safely flush a reg, but this shouldn't be normal. - IndirectCALL(safeFunc); - } -#if PPSSPP_ARCH(32BIT) - jit_->POP(EDX); -#endif - needsCheck_ = true; -} - -bool JitSafeMem::PrepareSlowRead(const void *safeFunc) { - _dbg_assert_msg_(safeFunc != nullptr, "Safe func cannot be null"); - if (!fast_) { - if (iaddr_ != (u32) -1) { - // No slow read necessary. - if (ImmValid()) - return false; - jit_->MOV(32, R(EAX), Imm32(iaddr_ & alignMask_)); - } else { - PrepareSlowAccess(); - jit_->LEA(32, EAX, MDisp(xaddr_, offset_)); - if (alignMask_ != 0xFFFFFFFF) - jit_->AND(32, R(EAX), Imm32(alignMask_)); - } - - if (!g_Config.bIgnoreBadMemAccess) { - jit_->MOV(32, MIPSSTATE_VAR(pc), Imm32(jit_->GetCompilerPC())); - } - // This is a special jit-ABI'd function. - if (jit_->CanCALLDirect(safeFunc)) { - jit_->CALL(safeFunc); - } else { - // We can't safely flush a reg, but this shouldn't be normal. - IndirectCALL(safeFunc); - } - needsCheck_ = true; - return true; - } - else - return false; -} - -void JitSafeMem::NextSlowRead(const void *safeFunc, int suboffset) { - _dbg_assert_msg_(safeFunc != nullptr, "Safe func cannot be null"); - _dbg_assert_msg_(!fast_, "NextSlowRead() called in fast memory mode?"); - - // For simplicity, do nothing for 0. We already read in PrepareSlowRead(). - if (suboffset == 0) - return; - - if (jit_->gpr.IsImm(raddr_)) - { - _dbg_assert_msg_(!Memory::IsValidAddress(iaddr_ + suboffset), "NextSlowRead() for an invalid immediate address?"); - - jit_->MOV(32, R(EAX), Imm32((iaddr_ + suboffset) & alignMask_)); - } - // For GPR, if xaddr_ was the dest register, this will be wrong. Don't use in GPR. - else - { - jit_->LEA(32, EAX, MDisp(xaddr_, offset_ + suboffset)); - if (alignMask_ != 0xFFFFFFFF) - jit_->AND(32, R(EAX), Imm32(alignMask_)); - } - - if (!g_Config.bIgnoreBadMemAccess) { - jit_->MOV(32, MIPSSTATE_VAR(pc), Imm32(jit_->GetCompilerPC())); - } - // This is a special jit-ABI'd function. - if (jit_->CanCALLDirect(safeFunc)) { - jit_->CALL(safeFunc); - } else { - // We can't safely flush a reg, but this shouldn't be normal. - IndirectCALL(safeFunc); - } -} - -bool JitSafeMem::ImmValid() -{ - return iaddr_ != (u32) -1 && Memory::IsValidAddress(iaddr_) && Memory::IsValidAddress(iaddr_ + size_ - 1); -} - -void JitSafeMem::IndirectCALL(const void *safeFunc) { -#if PPSSPP_ARCH(32BIT) - jit_->PUSH(ECX); - jit_->SUB(PTRBITS, R(ESP), Imm8(16 - 4)); - jit_->MOV(PTRBITS, R(ECX), ImmPtr(safeFunc)); - jit_->CALLptr(R(RCX)); - jit_->ADD(PTRBITS, R(ESP), Imm8(16 - 4)); - jit_->POP(ECX); -#else - jit_->PUSH(RCX); - jit_->SUB(PTRBITS, R(RSP), Imm8(8)); - jit_->MOV(PTRBITS, R(RCX), ImmPtr(safeFunc)); - jit_->CALLptr(R(RCX)); - jit_->ADD(64, R(RSP), Imm8(8)); - jit_->POP(RCX); -#endif -} - -void JitSafeMem::Finish() -{ - // Memory::ReadOrException_U32/etc. may have tripped coreState. - if (needsCheck_ && !g_Config.bIgnoreBadMemAccess) - jit_->js.afterOp |= JitState::AFTER_CORE_STATE; - if (needsSkip_) - jit_->SetJumpTarget(skip_); - for (auto it = skipChecks_.begin(), end = skipChecks_.end(); it != end; ++it) - jit_->SetJumpTarget(*it); -} - -static const int FUNCS_ARENA_SIZE = 512 * 1024; - -void JitSafeMemFuncs::Init(ThunkManager *thunks) { - using namespace Gen; - - AllocCodeSpace(FUNCS_ARENA_SIZE); - thunks_ = thunks; - - BeginWrite(1024); - readU32 = GetCodePtr(); - CreateReadFunc(32, (const void *)&Memory::ReadOrException_U32); - readU16 = GetCodePtr(); - CreateReadFunc(16, (const void *)&Memory::ReadOrException_U16); - readU8 = GetCodePtr(); - CreateReadFunc(8, (const void *)&Memory::ReadOrException_U8); - - writeU32 = GetCodePtr(); - CreateWriteFunc(32, (const void *)&Memory::WriteOrException_U32); - writeU16 = GetCodePtr(); - CreateWriteFunc(16, (const void *)&Memory::WriteOrException_U16); - writeU8 = GetCodePtr(); - CreateWriteFunc(8, (const void *)&Memory::WriteOrException_U8); - EndWrite(); -} - -void JitSafeMemFuncs::Shutdown() { - ResetCodePtr(0); - FreeCodeSpace(); - - readU32 = nullptr; - readU16 = nullptr; - readU8 = nullptr; - writeU32 = nullptr; - writeU16 = nullptr; - writeU8 = nullptr; -} - -// Mini ABI: -// Read funcs take address in EAX, return in RAX. -// Write funcs take address in EAX, data in RDX. -// On x86-32, Write funcs also have an extra 4 bytes on the stack. - -void JitSafeMemFuncs::CreateReadFunc(int bits, const void *fallbackFunc) { - CheckDirectEAX(); - - // Since we were CALLed, we need to align the stack before calling C++. -#if PPSSPP_ARCH(32BIT) - SUB(32, R(ESP), Imm8(16 - 4)); - ABI_CallFunctionA(thunks_->ProtectFunction(fallbackFunc, 1), R(EAX)); - ADD(32, R(ESP), Imm8(16 - 4)); -#else - SUB(64, R(RSP), Imm8(0x28)); - ABI_CallFunctionA(thunks_->ProtectFunction(fallbackFunc, 1), R(EAX)); - ADD(64, R(RSP), Imm8(0x28)); -#endif - - RET(); - - StartDirectAccess(); - -#if PPSSPP_ARCH(32BIT) - MOVZX(32, bits, EAX, MDisp(EAX, (u32)Memory::base)); -#else - MOVZX(32, bits, EAX, MRegSum(MEMBASEREG, EAX)); -#endif - - RET(); -} - -void JitSafeMemFuncs::CreateWriteFunc(int bits, const void *fallbackFunc) { - CheckDirectEAX(); - - // Since we were CALLed, we need to align the stack before calling C++. -#if PPSSPP_ARCH(32BIT) - // 4 for return, 4 for saved reg on stack. - SUB(32, R(ESP), Imm8(16 - 4 - 4)); - ABI_CallFunctionAA(thunks_->ProtectFunction(fallbackFunc, 2), R(EDX), R(EAX)); - ADD(32, R(ESP), Imm8(16 - 4 - 4)); -#else - SUB(64, R(RSP), Imm8(0x28)); - ABI_CallFunctionAA(thunks_->ProtectFunction(fallbackFunc, 2), R(EDX), R(EAX)); - ADD(64, R(RSP), Imm8(0x28)); -#endif - - RET(); - - StartDirectAccess(); - -#if PPSSPP_ARCH(32BIT) - MOV(bits, MDisp(EAX, (u32)Memory::base), R(EDX)); -#else - MOV(bits, MRegSum(MEMBASEREG, EAX), R(EDX)); -#endif - - RET(); -} - -void JitSafeMemFuncs::CheckDirectEAX() { - // Clear any cache/kernel bits. - AND(32, R(EAX), Imm32(0x3FFFFFFF)); - - CMP(32, R(EAX), Imm32(PSP_GetUserMemoryEnd())); - FixupBranch tooHighRAM = J_CC(CC_AE); - CMP(32, R(EAX), Imm32(PSP_GetKernelMemoryBase())); - skips_.push_back(J_CC(CC_AE)); - - CMP(32, R(EAX), Imm32(PSP_GetVidMemEnd())); - FixupBranch tooHighVid = J_CC(CC_AE); - CMP(32, R(EAX), Imm32(PSP_GetVidMemBase())); - skips_.push_back(J_CC(CC_AE)); - - CMP(32, R(EAX), Imm32(PSP_GetScratchpadMemoryEnd())); - FixupBranch tooHighScratch = J_CC(CC_AE); - CMP(32, R(EAX), Imm32(PSP_GetScratchpadMemoryBase())); - skips_.push_back(J_CC(CC_AE)); - - SetJumpTarget(tooHighRAM); - SetJumpTarget(tooHighVid); - SetJumpTarget(tooHighScratch); -} - -void JitSafeMemFuncs::StartDirectAccess() { - for (auto it = skips_.begin(), end = skips_.end(); it != end; ++it) { - SetJumpTarget(*it); - } - skips_.clear(); -} - -}; - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/JitSafeMem.h b/Core/MIPS/x86/JitSafeMem.h deleted file mode 100644 index 4dae2bdff3..0000000000 --- a/Core/MIPS/x86/JitSafeMem.h +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include - -class ThunkManager; - -namespace MIPSComp { - -class Jit; - -class JitSafeMem { -public: - JitSafeMem(Jit *jit, MIPSGPReg raddr, s32 offset, u32 alignMask = 0xFFFFFFFF); - - // Emit code necessary for a memory write, returns true if MOV to dest is needed. - bool PrepareWrite(Gen::OpArg &dest, int size); - // Emit code proceeding a slow write call, returns true if slow write is needed. - bool PrepareSlowWrite(); - // Emit a slow write from src. - void DoSlowWrite(const void *safeFunc, const Gen::OpArg& src, int suboffset = 0); - template - void DoSlowWrite(void (*safeFunc)(T val, u32 addr), const Gen::OpArg& src, int suboffset = 0) { - DoSlowWrite((const void *)safeFunc, src, suboffset); - } - - // Emit code necessary for a memory read, returns true if MOV from src is needed. - bool PrepareRead(Gen::OpArg &src, int size); - // Emit code for a slow read call, and returns true if result is in EAX. - bool PrepareSlowRead(const void *safeFunc); - template - bool PrepareSlowRead(T (*safeFunc)(u32 addr)) { - return PrepareSlowRead((const void *)safeFunc); - } - - // Cleans up final code for the memory access. - void Finish(); - - // WARNING: Only works for non-GPR. Do not use for reads into GPR. - Gen::OpArg NextFastAddress(int suboffset); - // WARNING: Only works for non-GPR. Do not use for reads into GPR. - void NextSlowRead(const void *safeFunc, int suboffset); - template - void NextSlowRead(T (*safeFunc)(u32 addr), int suboffset) { - NextSlowRead((const void *)safeFunc, suboffset); - } - -private: - enum MemoryOpType { - MEM_READ, - MEM_WRITE, - }; - - Gen::OpArg PrepareMemoryOpArg(MemoryOpType type); - void PrepareSlowAccess(); - bool ImmValid(); - void IndirectCALL(const void *safeFunc); - - Jit *jit_; - MIPSGPReg raddr_; - s32 offset_; - int size_; - bool needsCheck_; - bool needsSkip_; - bool fast_; - u32 alignMask_; - u32 iaddr_; - Gen::X64Reg xaddr_; - Gen::FixupBranch tooLow_, tooHigh_, skip_; - std::vector skipChecks_; - const u8 *safe_; -}; - -// Kept separate to avoid mistakes in the above class not using jit_. -class JitSafeMemFuncs : public Gen::XCodeBlock -{ -public: - JitSafeMemFuncs() { - } - ~JitSafeMemFuncs() { - Shutdown(); - } - - void Init(ThunkManager *thunks); - void Shutdown(); - - const u8 *readU32; - const u8 *readU16; - const u8 *readU8; - const u8 *writeU32; - const u8 *writeU16; - const u8 *writeU8; - -private: - void CreateReadFunc(int bits, const void *fallbackFunc); - void CreateWriteFunc(int bits, const void *fallbackFunc); - - void CheckDirectEAX(); - void StartDirectAccess(); - - std::vector skips_; - ThunkManager *thunks_; -}; - -}; diff --git a/Core/MIPS/x86/RegCache.cpp b/Core/MIPS/x86/RegCache.cpp deleted file mode 100644 index 06a298bf46..0000000000 --- a/Core/MIPS/x86/RegCache.cpp +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include - -#include "Common/x64Emitter.h" -#include "Core/Reporting.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSTables.h" -#include "Core/MIPS/MIPSAnalyst.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" - -using namespace Gen; -using namespace X64JitConstants; - -static const X64Reg allocationOrder[] = { - // R12, when used as base register, for example in a LEA, can generate bad code! Need to look into this. - // On x64, RCX and RDX are the first args. CallProtectedFunction() assumes they're not regcached. -#if PPSSPP_ARCH(AMD64) -#ifdef _WIN32 - RSI, RDI, R8, R9, R10, R11, R12, R13, -#else - RBP, R8, R9, R10, R11, R12, R13, -#endif -#elif PPSSPP_ARCH(X86) - ESI, EDI, EDX, ECX, EBX, -#endif -}; - -#if PPSSPP_ARCH(AMD64) -static X64Reg allocationOrderR15[ARRAY_SIZE(allocationOrder) + 1] = {INVALID_REG}; -#endif - -void GPRRegCache::FlushBeforeCall() { - // TODO: Only flush the non-preserved-by-callee registers. - Flush(); -} - -GPRRegCache::GPRRegCache() { -} - -void GPRRegCache::Start(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo) { -#if PPSSPP_ARCH(AMD64) - if (allocationOrderR15[0] == INVALID_REG) { - memcpy(allocationOrderR15, allocationOrder, sizeof(allocationOrder)); - allocationOrderR15[ARRAY_SIZE(allocationOrderR15) - 1] = R15; - } -#endif - - mips_ = mipsState; - for (int i = 0; i < NUM_X_REGS; i++) { - xregs[i].free = true; - xregs[i].dirty = false; - xregs[i].allocLocked = false; - } - memset(regs, 0, sizeof(regs)); - OpArg base = GetDefaultLocation(MIPS_REG_ZERO); - for (int i = 0; i < 32; i++) { - regs[i].location = base; - base.IncreaseOffset(sizeof(u32)); - } - for (int i = 32; i < NUM_MIPS_GPRS; i++) { - regs[i].location = GetDefaultLocation(MIPSGPReg(i)); - } - SetImm(MIPS_REG_ZERO, 0); - - js_ = js; - jo_ = jo; -} - - -// these are MIPS reg indices -void GPRRegCache::Lock(MIPSGPReg p1, MIPSGPReg p2, MIPSGPReg p3, MIPSGPReg p4) { - regs[p1].locked = true; - if (p2 != MIPS_REG_INVALID) regs[p2].locked = true; - if (p3 != MIPS_REG_INVALID) regs[p3].locked = true; - if (p4 != MIPS_REG_INVALID) regs[p4].locked = true; -} - -// these are x64 reg indices -void GPRRegCache::LockX(int x1, int x2, int x3, int x4) { - _assert_msg_(!xregs[x1].allocLocked, "RegCache: x %d already locked!", x1); - xregs[x1].allocLocked = true; - if (x2 != 0xFF) xregs[x2].allocLocked = true; - if (x3 != 0xFF) xregs[x3].allocLocked = true; - if (x4 != 0xFF) xregs[x4].allocLocked = true; -} - -void GPRRegCache::UnlockAll() { - for (int i = 0; i < NUM_MIPS_GPRS; i++) - regs[i].locked = false; - // In case it was stored, discard it now. - SetImm(MIPS_REG_ZERO, 0); -} - -void GPRRegCache::UnlockAllX() { - for (int i = 0; i < NUM_X_REGS; i++) - xregs[i].allocLocked = false; -} - -X64Reg GPRRegCache::FindBestToSpill(bool unusedOnly, bool *clobbered) { - int allocCount; - const X64Reg *allocOrder = GetAllocationOrder(allocCount); - - static const int UNUSED_LOOKAHEAD_OPS = 30; - - *clobbered = false; - for (int i = 0; i < allocCount; i++) { - X64Reg reg = allocOrder[i]; - if (xregs[reg].allocLocked) - continue; - if (xregs[reg].mipsReg != MIPS_REG_INVALID && regs[xregs[reg].mipsReg].locked) - continue; - - // Awesome, a clobbered reg. Let's use it. - if (MIPSAnalyst::IsRegisterClobbered(xregs[reg].mipsReg, js_->compilerPC, UNUSED_LOOKAHEAD_OPS)) { - *clobbered = true; - return reg; - } - - // Not awesome. A used reg. Let's try to avoid spilling. - if (unusedOnly && MIPSAnalyst::IsRegisterUsed(xregs[reg].mipsReg, js_->compilerPC, UNUSED_LOOKAHEAD_OPS)) { - continue; - } - - return reg; - } - - return INVALID_REG; -} - -X64Reg GPRRegCache::GetFreeXReg() -{ - int aCount; - const X64Reg *aOrder = GetAllocationOrder(aCount); - for (int i = 0; i < aCount; i++) - { - X64Reg xr = aOrder[i]; - if (!xregs[xr].allocLocked && xregs[xr].free) - { - return xr; - } - } - - //Okay, not found :( Force grab one - bool clobbered; - X64Reg bestToSpill = FindBestToSpill(true, &clobbered); - if (bestToSpill == INVALID_REG) { - bestToSpill = FindBestToSpill(false, &clobbered); - } - - if (bestToSpill != INVALID_REG) { - // TODO: Broken somehow in Dante's Inferno, but most games work. Bad flags in MIPSTables somewhere? - if (clobbered) { - DiscardRegContentsIfCached(xregs[bestToSpill].mipsReg); - } else { - StoreFromRegister(xregs[bestToSpill].mipsReg); - } - return bestToSpill; - } - - // Still no dice? Give up. - _assert_msg_(false, "Regcache ran out of regs"); - return (X64Reg)-1; -} - -void GPRRegCache::FlushR(X64Reg reg) -{ - if (reg >= NUM_X_REGS) { - _assert_msg_(false, "Flushing non existent reg"); - } else if (!xregs[reg].free) { - StoreFromRegister(xregs[reg].mipsReg); - } -} - -void GPRRegCache::FlushRemap(MIPSGPReg oldreg, MIPSGPReg newreg) { - OpArg oldLocation = regs[oldreg].location; - _assert_msg_(oldLocation.IsSimpleReg(), "FlushRemap: Must already be in an x86 register"); - - X64Reg xr = oldLocation.GetSimpleReg(); - - if (oldreg == newreg) { - xregs[xr].dirty = true; - return; - } - - StoreFromRegister(oldreg); - - // Now, if newreg already was mapped somewhere, get rid of that. - DiscardRegContentsIfCached(newreg); - - // Now, take over the old register. - regs[newreg].location = oldLocation; - regs[newreg].away = true; - regs[newreg].locked = true; - xregs[xr].mipsReg = newreg; - xregs[xr].dirty = true; - xregs[xr].free = false; -} - -int GPRRegCache::SanityCheck() const { - for (int i = 0; i < NUM_MIPS_GPRS; i++) { - const MIPSGPReg r = MIPSGPReg(i); - if (regs[i].away) { - if (regs[i].location.IsSimpleReg()) { - Gen::X64Reg simple = regs[i].location.GetSimpleReg(); - if (xregs[simple].allocLocked) - return 1; - if (xregs[simple].mipsReg != r) - return 2; - } - else if (regs[i].location.IsImm()) - return 3; - } - } - return 0; -} - -void GPRRegCache::DiscardRegContentsIfCached(MIPSGPReg preg) { - if (regs[preg].away && regs[preg].location.IsSimpleReg()) { - X64Reg xr = regs[preg].location.GetSimpleReg(); - xregs[xr].free = true; - xregs[xr].dirty = false; - xregs[xr].mipsReg = MIPS_REG_INVALID; - regs[preg].away = false; - if (preg == MIPS_REG_ZERO) { - regs[preg].location = Imm32(0); - } else { - regs[preg].location = GetDefaultLocation(preg); - } - } -} - -void GPRRegCache::DiscardR(MIPSGPReg preg) { - if (regs[preg].away) { - if (regs[preg].location.IsSimpleReg()) { - DiscardRegContentsIfCached(preg); - } else { - regs[preg].away = false; - if (preg == MIPS_REG_ZERO) { - regs[preg].location = Imm32(0); - } else { - regs[preg].location = GetDefaultLocation(preg); - } - } - } -} - - -void GPRRegCache::SetImm(MIPSGPReg preg, u32 immValue) { - // ZERO is always zero. Let's just make sure. - if (preg == MIPS_REG_ZERO) - immValue = 0; - - DiscardRegContentsIfCached(preg); - regs[preg].away = true; - regs[preg].location = Imm32(immValue); -} - -bool GPRRegCache::IsImm(MIPSGPReg preg) const { - // Note that ZERO is generally always imm. - return regs[preg].location.IsImm(); -} - -u32 GPRRegCache::GetImm(MIPSGPReg preg) const { - _dbg_assert_msg_(IsImm(preg), "Reg %d must be an immediate.", preg); - // Always 0 for ZERO. - if (preg == MIPS_REG_ZERO) - return 0; - return regs[preg].location.GetImmValue(); -} - -const X64Reg *GPRRegCache::GetAllocationOrder(int &count) { -#if PPSSPP_ARCH(AMD64) - if (!jo_->reserveR15ForAsm) { - count = ARRAY_SIZE(allocationOrderR15); - return allocationOrderR15; - } -#endif - count = ARRAY_SIZE(allocationOrder); - return allocationOrder; -} - - -OpArg GPRRegCache::GetDefaultLocation(MIPSGPReg reg) const { - if (reg < 32) { - return MDisp(CTXREG, -128 + reg * 4); - } - switch (reg) { - case MIPS_REG_HI: - return MIPSSTATE_VAR(hi); - case MIPS_REG_LO: - return MIPSSTATE_VAR(lo); - case MIPS_REG_FPCOND: - return MIPSSTATE_VAR(fpcond); - case MIPS_REG_VFPUCC: - return MIPSSTATE_VAR(vfpuCtrl[VFPU_CTRL_CC]); - default: - ERROR_LOG_REPORT(Log::JIT, "Bad mips register %d", reg); - return MIPSSTATE_VAR(r[0]); - } -} - - -void GPRRegCache::KillImmediate(MIPSGPReg preg, bool doLoad, bool makeDirty) { - if (regs[preg].away) { - if (regs[preg].location.IsImm()) - MapReg(preg, doLoad, makeDirty); - else if (regs[preg].location.IsSimpleReg()) - xregs[RX(preg)].dirty |= makeDirty; - } -} - -void GPRRegCache::MapReg(MIPSGPReg i, bool doLoad, bool makeDirty) { - if (!regs[i].away && regs[i].location.IsImm()) { - _assert_msg_(false, "Bad immediate"); - } - if (!regs[i].away || regs[i].location.IsImm()) { - X64Reg xr = GetFreeXReg(); - _assert_msg_(!xregs[xr].dirty, "Xreg already dirty"); - _assert_msg_(!xregs[xr].allocLocked, "GetFreeXReg returned locked register"); - xregs[xr].free = false; - xregs[xr].mipsReg = i; - xregs[xr].dirty = makeDirty || regs[i].location.IsImm(); - OpArg newloc = ::Gen::R(xr); - if (doLoad) { - // Force ZERO to be 0. - if (i == MIPS_REG_ZERO) - emit->MOV(32, newloc, Imm32(0)); - else - emit->MOV(32, newloc, regs[i].location); - } - for (int j = 0; j < 32; j++) { - if (i != MIPSGPReg(j) && regs[j].location.IsSimpleReg(xr)) { - _assert_msg_(false, "BindToRegister: Strange condition"); - } - } - regs[i].away = true; - regs[i].location = newloc; - } else { - // reg location must be simplereg; memory locations - // and immediates are taken care of above. - xregs[RX(i)].dirty |= makeDirty; - } - - _assert_msg_(!xregs[RX(i)].allocLocked, "This reg should have been flushed (r%d)", i); -} - -void GPRRegCache::StoreFromRegister(MIPSGPReg i) { - if (regs[i].away) { - bool doStore; - if (regs[i].location.IsSimpleReg()) { - X64Reg xr = RX(i); - xregs[xr].free = true; - xregs[xr].mipsReg = MIPS_REG_INVALID; - doStore = xregs[xr].dirty; - xregs[xr].dirty = false; - } else { - //must be immediate - do nothing - doStore = true; - } - OpArg newLoc = GetDefaultLocation(i); - // But never store to ZERO. - if (doStore && i != MIPS_REG_ZERO) - emit->MOV(32, newLoc, regs[i].location); - regs[i].location = newLoc; - regs[i].away = false; - } -} - -void GPRRegCache::Flush() { - for (int i = 0; i < NUM_X_REGS; i++) { - _assert_msg_(!xregs[i].allocLocked, "Someone forgot to unlock X64 reg %d.", i); - } - SetImm(MIPS_REG_ZERO, 0); - for (int i = 1; i < NUM_MIPS_GPRS; i++) { - const MIPSGPReg r = MIPSGPReg(i); - _assert_msg_(!regs[i].locked, "Somebody forgot to unlock MIPS reg %d.", i); - if (regs[i].away) { - if (regs[i].location.IsSimpleReg()) { - X64Reg xr = RX(r); - StoreFromRegister(r); - xregs[xr].dirty = false; - } - else if (regs[i].location.IsImm()) { - StoreFromRegister(r); - } else { - _assert_msg_(false, "Jit64 - Flush unhandled case, reg %d PC: %08x", i, mips_->pc); - } - } - } -} - -void GPRRegCache::GetState(GPRRegCacheState &state) const { - memcpy(state.regs, regs, sizeof(regs)); - memcpy(state.xregs, xregs, sizeof(xregs)); -} - -void GPRRegCache::RestoreState(const GPRRegCacheState& state) { - memcpy(regs, state.regs, sizeof(regs)); - memcpy(xregs, state.xregs, sizeof(xregs)); -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/RegCache.h b/Core/MIPS/x86/RegCache.h deleted file mode 100644 index 85b381225a..0000000000 --- a/Core/MIPS/x86/RegCache.h +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include "ppsspp_config.h" -#include "Common/x64Emitter.h" -#include "Core/MIPS/MIPS.h" - -namespace X64JitConstants { -#if PPSSPP_ARCH(AMD64) - const Gen::X64Reg MEMBASEREG = Gen::RBX; - const Gen::X64Reg CTXREG = Gen::R14; - const Gen::X64Reg JITBASEREG = Gen::R15; -#else - const Gen::X64Reg CTXREG = Gen::EBP; -#endif - - // This must be one of EAX, EBX, ECX, EDX as they have 8-bit subregisters. - const Gen::X64Reg TEMPREG = Gen::EAX; - const int NUM_MIPS_GPRS = 36; - -#if PPSSPP_ARCH(AMD64) - const u32 NUM_X_REGS = 16; -#elif PPSSPP_ARCH(X86) - const u32 NUM_X_REGS = 8; -#endif -} - -struct MIPSCachedReg { - Gen::OpArg location; - bool away; // value not in source register - bool locked; -}; - -struct X64CachedReg { - MIPSGPReg mipsReg; - bool dirty; - bool free; - bool allocLocked; -}; - -struct GPRRegCacheState { - MIPSCachedReg regs[X64JitConstants::NUM_MIPS_GPRS]; - X64CachedReg xregs[X64JitConstants::NUM_X_REGS]; -}; - -namespace MIPSComp { - struct JitOptions; - struct JitState; -} - -class GPRRegCache -{ -public: - GPRRegCache(); - ~GPRRegCache() {} - void Start(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo); - - void DiscardRegContentsIfCached(MIPSGPReg preg); - void DiscardR(MIPSGPReg preg); - void SetEmitter(Gen::XEmitter *emitter) {emit = emitter;} - - void FlushR(Gen::X64Reg reg); - void FlushLockX(Gen::X64Reg reg) { - FlushR(reg); - LockX(reg); - } - void FlushLockX(Gen::X64Reg reg1, Gen::X64Reg reg2) { - FlushR(reg1); FlushR(reg2); - LockX(reg1); LockX(reg2); - } - void Flush(); - void FlushBeforeCall(); - - // Flushes one register and reuses the register for another one. Dirtyness is implied. - void FlushRemap(MIPSGPReg oldreg, MIPSGPReg newreg); - - int SanityCheck() const; - void KillImmediate(MIPSGPReg preg, bool doLoad, bool makeDirty); - - void MapReg(MIPSGPReg preg, bool doLoad = true, bool makeDirty = true); - void StoreFromRegister(MIPSGPReg preg); - - const Gen::OpArg &R(MIPSGPReg preg) const {return regs[preg].location;} - Gen::X64Reg RX(MIPSGPReg preg) const - { - if (regs[preg].away && regs[preg].location.IsSimpleReg()) - return regs[preg].location.GetSimpleReg(); - _assert_msg_(false, "Not so simple - %d", preg); - return (Gen::X64Reg)-1; - } - Gen::OpArg GetDefaultLocation(MIPSGPReg reg) const; - - // Register locking. - void Lock(MIPSGPReg p1, MIPSGPReg p2 = MIPS_REG_INVALID, MIPSGPReg p3 = MIPS_REG_INVALID, MIPSGPReg p4 = MIPS_REG_INVALID); - void LockX(int x1, int x2=0xff, int x3=0xff, int x4=0xff); - void UnlockAll(); - void UnlockAllX(); - - void SetImm(MIPSGPReg preg, u32 immValue); - bool IsImm(MIPSGPReg preg) const; - u32 GetImm(MIPSGPReg preg) const; - - void GetState(GPRRegCacheState &state) const; - void RestoreState(const GPRRegCacheState& state); - - // Whether the register holds something the cache cares about: a MIPS register or a locked temp. - bool IsXRegInUse(Gen::X64Reg reg) const { - return !xregs[reg].free || xregs[reg].allocLocked; - } - - MIPSState *mips_ = nullptr; - -private: - Gen::X64Reg GetFreeXReg(); - Gen::X64Reg FindBestToSpill(bool unusedOnly, bool *clobbered); - const Gen::X64Reg *GetAllocationOrder(int &count); - - MIPSCachedReg regs[X64JitConstants::NUM_MIPS_GPRS]{}; - X64CachedReg xregs[X64JitConstants::NUM_X_REGS]{}; - - Gen::XEmitter *emit = nullptr; - MIPSComp::JitState *js_; - MIPSComp::JitOptions *jo_; -}; diff --git a/Core/MIPS/x86/RegCacheFPU.cpp b/Core/MIPS/x86/RegCacheFPU.cpp deleted file mode 100644 index 567b53f901..0000000000 --- a/Core/MIPS/x86/RegCacheFPU.cpp +++ /dev/null @@ -1,1102 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#include "ppsspp_config.h" -#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) - -#include -#include "Common/Math/SIMDHeaders.h" -#include "Common/Log.h" -#include "Common/x64Emitter.h" -#include "Core/MIPS/x86/Jit.h" -#include "Core/MIPS/x86/RegCache.h" -#include "Core/MIPS/x86/RegCacheFPU.h" - -using namespace Gen; -using namespace X64JitConstants; - -FPURegCache::FPURegCache() { - vregs = regs + 32; -} - -void FPURegCache::Start(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo, bool useRip) { - mips_ = mipsState; - useRip_ = useRip; - if (!initialReady) { - SetupInitialRegs(); - initialReady = true; - } - - memcpy(xregs, xregsInitial, sizeof(xregs)); - memcpy(regs, regsInitial, sizeof(regs)); - pendingFlush = false; - - js_ = js; - jo_ = jo; -} - -void FPURegCache::SetupInitialRegs() { - for (int i = 0; i < NUM_X_FPREGS; i++) { - memset(xregsInitial[i].mipsRegs, -1, sizeof(xregsInitial[i].mipsRegs)); - xregsInitial[i].dirty = false; - } - memset(regsInitial, 0, sizeof(regsInitial)); - OpArg base = GetDefaultLocation(0); - for (int i = 0; i < 32; i++) { - regsInitial[i].location = base; - base.IncreaseOffset(sizeof(float)); - } - for (int i = 32; i < 32 + 128; i++) { - regsInitial[i].location = GetDefaultLocation(i); - } - base = GetDefaultLocation(32 + 128); - for (int i = 32 + 128; i < NUM_MIPS_FPRS; i++) { - regsInitial[i].location = base; - base.IncreaseOffset(sizeof(float)); - } -} - -void FPURegCache::SpillLock(int p1, int p2, int p3, int p4) { - regs[p1].locked++; - if (p2 != 0xFF) regs[p2].locked++; - if (p3 != 0xFF) regs[p3].locked++; - if (p4 != 0xFF) regs[p4].locked++; -} - -void FPURegCache::SpillLockV(const u8 *vec, VectorSize sz) { - for (int i = 0; i < GetNumVectorElements(sz); i++) { - vregs[vec[i]].locked++; - } -} - -void FPURegCache::SpillLockV(int vec, VectorSize sz) { - u8 r[4]; - GetVectorRegs(r, sz, vec); - SpillLockV(r, sz); -} - -void FPURegCache::ReleaseSpillLockV(const u8 *vec, VectorSize sz) { - for (int i = 0; i < GetNumVectorElements(sz); i++) { - vregs[vec[i]].locked = 0; - } -} - -void FPURegCache::ReduceSpillLock(int mipsreg) { - regs[mipsreg].locked--; -} - -void FPURegCache::ReduceSpillLockV(const u8 *vec, VectorSize sz) { - for (int i = 0; i < GetNumVectorElements(sz); i++) { - vregs[vec[i]].locked--; - } -} - -void FPURegCache::FlushRemap(int oldreg, int newreg) { - OpArg oldLocation = regs[oldreg].location; - _assert_msg_(oldLocation.IsSimpleReg(), "FlushRemap: Must already be in an x86 SSE register"); - _assert_msg_(regs[oldreg].lane == 0, "FlushRemap only supports FPR registers"); - - X64Reg xr = oldLocation.GetSimpleReg(); - if (oldreg == newreg) { - xregs[xr].dirty = true; - return; - } - - StoreFromRegister(oldreg); - - // Now, if newreg already was mapped somewhere, get rid of that. - DiscardR(newreg); - - // Now, take over the old register. - regs[newreg].location = oldLocation; - regs[newreg].away = true; - regs[newreg].locked = true; - regs[newreg].lane = 0; - xregs[xr].mipsReg = newreg; - xregs[xr].dirty = true; -} - -void FPURegCache::MapRegV(int vreg, int flags) { - MapReg(vreg + 32, (flags & MAP_NOINIT) != MAP_NOINIT, (flags & MAP_DIRTY) != 0); -} - -void FPURegCache::MapRegsV(int vec, VectorSize sz, int flags) { - u8 r[4]; - GetVectorRegs(r, sz, vec); - SpillLockV(r, sz); - for (int i = 0; i < GetNumVectorElements(sz); i++) { - MapReg(r[i] + 32, (flags & MAP_NOINIT) != MAP_NOINIT, (flags & MAP_DIRTY) != 0); - } - if ((flags & MAP_NOLOCK) != 0) { - // We have to lock so the sz won't spill, so we unlock after. - // If they were already locked, we only reduce the lock we added above. - ReduceSpillLockV(r, sz); - } -} - -void FPURegCache::MapRegsV(const u8 *r, VectorSize sz, int flags) { - SpillLockV(r, sz); - for (int i = 0; i < GetNumVectorElements(sz); i++) { - MapReg(r[i] + 32, (flags & MAP_NOINIT) != MAP_NOINIT, (flags & MAP_DIRTY) != 0); - } - if ((flags & MAP_NOLOCK) != 0) { - // We have to lock so the sz won't spill, so we unlock after. - // If they were already locked, we only reduce the lock we added above. - ReduceSpillLockV(r, sz); - } -} - -bool FPURegCache::IsMappedVS(const u8 *v, VectorSize vsz) { - const int n = GetNumVectorElements(vsz); - - // Make sure the first reg is at least mapped in the right place. - if (!IsMappedVS(v[0])) - return false; - if (vregs[v[0]].lane != 1) - return false; - - // And make sure the rest are mapped to the same reg in the right positions. - X64Reg xr = VSX(v); - for (int i = 1; i < n; ++i) { - u8 vi = v[i]; - if (!IsMappedVS(vi) || VSX(&vi) != xr) - return false; - if (vregs[vi].lane != i + 1) - return false; - } - // TODO: Optimize this case? It happens. - for (int i = n; i < 4; ++i) { - if (xregs[xr].mipsRegs[i] != -1) { - return false; - } - } - return true; -} - -void FPURegCache::MapRegsVS(const u8 *r, VectorSize vsz, int flags) { - const int n = GetNumVectorElements(vsz); - - _dbg_assert_msg_(jo_->enableVFPUSIMD, "Should not map simd regs when option is off."); - - if (!TryMapRegsVS(r, vsz, flags)) { - // TODO: Could be more optimal. - for (int i = 0; i < n; ++i) { - StoreFromRegisterV(r[i]); - } - if (!TryMapRegsVS(r, vsz, flags)) { - _dbg_assert_msg_(false, "MapRegsVS() failed on second try."); - } - } -} - -bool FPURegCache::CanMapVS(const u8 *v, VectorSize vsz) { - const int n = GetNumVectorElements(vsz); - - if (!jo_->enableVFPUSIMD) { - return false; - } - - if (IsMappedVS(v, vsz)) { - return true; - } else if (vregs[v[0]].lane != 0) { - const MIPSCachedFPReg &v0 = vregs[v[0]]; - _dbg_assert_msg_(v0.away, "Must be away when lane != 0"); - _dbg_assert_msg_(v0.location.IsSimpleReg(), "Must be is register when lane != 0"); - - // Already in a different simd set. - return false; - } - - if (vregs[v[0]].locked) { - // If it's locked, we can't mess with it. - return false; - } - - // Next, fail if any of the other regs are in simd currently. - // TODO: Only if locked? Not sure if it will be worth breaking them anyway. - for (int i = 1; i < n; ++i) { - if (vregs[v[i]].lane != 0) { - return false; - } - // If it's locked, in simd or not, we can't use it. - if (vregs[v[i]].locked) { - return false; - } - _assert_msg_(!vregs[v[i]].location.IsImm(), "Cannot handle imms in fp cache."); - } - - return true; -} - -bool FPURegCache::TryMapRegsVS(const u8 *v, VectorSize vsz, int flags) { - const int n = GetNumVectorElements(vsz); - - if (!CanMapVS(v, vsz)) { - return false; - } - - if (IsMappedVS(v, vsz)) { - // Already mapped then, perfect. Just mark dirty. - if ((flags & MAP_DIRTY) != 0) - xregs[VSX(v)].dirty = true; - if ((flags & MAP_NOLOCK) == 0) - SpillLockV(v, vsz); - return true; - } - - // At this point, some or all are in single regs or memory, and they're not locked there. - - if (n == 1) { - // Single is easy, just map normally but track as a SIMD reg. - // This way V/VS can warn about improper usage properly. - MapRegV(v[0], flags); - X64Reg vx = VX(v[0]); - if (vx == INVALID_REG) - return false; - - vregs[v[0]].lane = 1; - if ((flags & MAP_DIRTY) != 0) - xregs[vx].dirty = true; - if ((flags & MAP_NOLOCK) == 0) - SpillLockV(v, vsz); - Invariant(); - return true; - } - - X64Reg xr; - if ((flags & MAP_NOINIT) != MAP_NOINIT) { - xr = LoadRegsVS(v, n); - } else { - xr = GetFreeXReg(); - } - - // Victory, now let's clean up everything. - OpArg newloc = Gen::R(xr); - bool dirty = (flags & MAP_DIRTY) != 0; - for (int i = 0; i < n; ++i) { - MIPSCachedFPReg &vr = vregs[v[i]]; - if (vr.away) { - // Clear the xreg it was in before. - X64Reg oldXReg = vr.location.GetSimpleReg(); - if (oldXReg != xr) { - xregs[oldXReg].mipsReg = -1; - } - if (xregs[oldXReg].dirty) { - // Inherit the "dirtiness" (ultimately set below for all regs.) - dirty = true; - xregs[oldXReg].dirty = false; - } - } - xregs[xr].mipsRegs[i] = v[i] + 32; - vr.location = newloc; - vr.lane = i + 1; - vr.away = true; - } - xregs[xr].dirty = dirty; - - if ((flags & MAP_NOLOCK) == 0) { - SpillLockV(v, vsz); - } - - Invariant(); - return true; -} - -X64Reg FPURegCache::LoadRegsVS(const u8 *v, int n) { - int regsAvail = 0; - int regsLoaded = 0; - X64Reg xrs[4] = {INVALID_REG, INVALID_REG, INVALID_REG, INVALID_REG}; - bool xrsLoaded[4] = {false, false, false, false}; - - _dbg_assert_msg_(n >= 2 && n <= 4, "LoadRegsVS is only implemented for simd loads."); - - for (int i = 0; i < n; ++i) { - const MIPSCachedFPReg &mr = vregs[v[i]]; - if (mr.away) { - X64Reg mrx = mr.location.GetSimpleReg(); - // If it's not simd, or lanes 1+ are clear, we can use it. - if (mr.lane == 0 || xregs[mrx].mipsRegs[1] == -1) { - // Okay, there's nothing else in this reg, so we can use it. - xrsLoaded[i] = true; - xrs[i] = mrx; - ++regsLoaded; - ++regsAvail; - } else if (mr.lane != 0) { - _dbg_assert_msg_(false, "LoadRegsVS is not able to handle simd remapping yet, store first."); - } - } - } - - if (regsAvail < n) { - // Try to grab some without spilling. - X64Reg xrFree[4]; - int obtained = GetFreeXRegs(xrFree, n - regsAvail, false); - int pos = 0; - for (int i = 0; i < n && pos < obtained; ++i) { - if (xrs[i] == INVALID_REG) { - // Okay, it's not loaded but we have a reg for this slot. - xrs[i] = xrFree[pos++]; - ++regsAvail; - } - } - } - - // Let's also check if the memory addresses are sequential. - int sequential = 1; - for (int i = 1; i < n; ++i) { - if (v[i] < 128 && v[i - 1] < 128) { - if (voffset[v[i]] != voffset[v[i - 1]] + 1) { - break; - } - } else if (v[i] >= 128 && v[i - 1] >= 128) { - if (v[i] != v[i - 1] + 1) { - break; - } - } else { - // Temps can't be sequential with non-temps. - break; - } - ++sequential; - } - - // Did we end up with enough regs? - // TODO: Not handling the case of some regs avail and some loaded right now. - if (regsAvail < n && (sequential != n || regsLoaded == n || regsAvail == 0)) { - regsAvail = GetFreeXRegs(xrs, 2, true); - _dbg_assert_msg_(regsAvail >= 2, "Ran out of fp regs for loading simd regs with."); - _dbg_assert_msg_(xrs[0] != xrs[1], "Regs for simd load are the same, bad things await."); - // We spilled, so we assume that all our regs are screwed up now anyway. - for (int i = 0; i < 4; ++i) { - xrsLoaded[i] = false; - } - for (int i = 2; i < n; ++i){ - xrs[i] = INVALID_REG; - } - regsLoaded = 0; - } - - // If they're sequential, and we wouldn't need to store them all, use a single load. - // But if they're already loaded, we'd have to store, not worth it. - X64Reg res = INVALID_REG; - if (sequential == n && regsLoaded < n) { - // TODO: What should we do if some are in regs? Better to assemble? - for (int i = 0; i < n; ++i) { - StoreFromRegisterV(v[i]); - } - - // Grab any available reg. - for (int i = 0; i < n; ++i) { - if (xrs[i] != INVALID_REG) { - res = xrs[i]; - break; - } - } - const float *f = v[0] < 128 ? &mips_->v[voffset[v[0]]] : &mips_->tempValues[v[0] - 128]; - if (((intptr_t)f & 0x7) == 0 && n == 2) { - emit->MOVQ_xmm(res, vregs[v[0]].location); - } else if (((intptr_t)f & 0xf) == 0) { - // On modern processors, MOVUPS on aligned is fast, but maybe not on older ones. - emit->MOVAPS(res, vregs[v[0]].location); - } else { - emit->MOVUPS(res, vregs[v[0]].location); - } - } else if (regsAvail >= n) { - // Have enough regs, potentially all in regs. - auto loadXR = [&](int l) { - if (!xrsLoaded[l] && n >= l + 1) { - emit->MOVSS(xrs[l], vregs[v[l]].location); - } - }; - // The order here is intentional. - loadXR(3); - loadXR(1); - loadXR(2); - loadXR(0); - if (n == 4) { - // This gives us [w, y] in the y reg. - emit->UNPCKLPS(xrs[1], Gen::R(xrs[3])); - } - if (n >= 3) { - // This gives us [z, x]. Then we combine with y. - emit->UNPCKLPS(xrs[0], Gen::R(xrs[2])); - } - if (n >= 2) { - emit->UNPCKLPS(xrs[0], Gen::R(xrs[1])); - } - res = xrs[0]; - } else { - _dbg_assert_msg_(n > 2, "2 should not be possible here."); - - // Available regs are less than n, and some may be loaded. - // Let's grab the most optimal unloaded ones. - X64Reg xr1 = n == 3 ? xrs[1] : xrs[3]; - X64Reg xr2 = xrs[2]; - if (xr1 == INVALID_REG) { - // Not one of the available ones. Grab another. - for (int i = n - 1; i >= 0; --i) { - if (xrs[i] != INVALID_REG && xrs[i] != xr2) { - StoreFromRegisterV(v[i]); - xr1 = xrs[i]; - break; - } - } - } - if (xr2 == INVALID_REG) { - // Not one of the available ones. Grab another. - for (int i = n - 1; i >= 0; --i) { - if (xrs[i] != INVALID_REG && xrs[i] != xr1) { - StoreFromRegisterV(v[i]); - xr2 = xrs[i]; - break; - } - } - } - - if (n == 3) { - if (!vregs[v[2]].location.IsSimpleReg(xr2)) - emit->MOVSS(xr2, vregs[v[2]].location); - if (!vregs[v[1]].location.IsSimpleReg(xr1)) - emit->MOVSS(xr1, vregs[v[1]].location); - emit->SHUFPS(xr1, Gen::R(xr2), _MM_SHUFFLE(3, 0, 0, 0)); - emit->MOVSS(xr2, vregs[v[0]].location); - emit->MOVSS(xr1, Gen::R(xr2)); - } else if (n == 4) { - if (!vregs[v[2]].location.IsSimpleReg(xr2)) - emit->MOVSS(xr2, vregs[v[2]].location); - if (!vregs[v[3]].location.IsSimpleReg(xr1)) - emit->MOVSS(xr1, vregs[v[3]].location); - emit->UNPCKLPS(xr2, Gen::R(xr1)); - emit->MOVSS(xr1, vregs[v[1]].location); - emit->SHUFPS(xr1, Gen::R(xr2), _MM_SHUFFLE(1, 0, 0, 3)); - emit->MOVSS(xr2, vregs[v[0]].location); - emit->MOVSS(xr1, Gen::R(xr2)); - } - res = xr1; - } - - return res; -} - -bool FPURegCache::TryMapDirtyInVS(const u8 *vd, VectorSize vdsz, const u8 *vs, VectorSize vssz, bool avoidLoad) { - // Don't waste time mapping if some will for sure fail. - if (!CanMapVS(vd, vdsz) || !CanMapVS(vs, vssz)) { - return false; - } - // But, they could still fail based on overlap. Hopefully not common... - bool success = TryMapRegsVS(vs, vssz, 0); - if (success) { - success = TryMapRegsVS(vd, vdsz, avoidLoad ? MAP_NOINIT : MAP_DIRTY); - } - ReleaseSpillLockV(vs, vssz); - ReleaseSpillLockV(vd, vdsz); - - _dbg_assert_msg_(!success || IsMappedVS(vd, vdsz), "vd should be mapped now"); - _dbg_assert_msg_(!success || IsMappedVS(vs, vssz), "vs should be mapped now"); - - return success; -} - -bool FPURegCache::TryMapDirtyInInVS(const u8 *vd, VectorSize vdsz, const u8 *vs, VectorSize vssz, const u8 *vt, VectorSize vtsz, bool avoidLoad) { - // Don't waste time mapping if some will for sure fail. - if (!CanMapVS(vd, vdsz) || !CanMapVS(vs, vssz) || !CanMapVS(vt, vtsz)) { - return false; - } - - - // But, they could still fail based on overlap. Hopefully not common... - bool success = TryMapRegsVS(vs, vssz, 0); - if (success) { - success = TryMapRegsVS(vt, vtsz, 0); - } - if (success) { - success = TryMapRegsVS(vd, vdsz, avoidLoad ? MAP_NOINIT : MAP_DIRTY); - } - ReleaseSpillLockV(vd, vdsz); - ReleaseSpillLockV(vs, vssz); - ReleaseSpillLockV(vt, vtsz); - - _dbg_assert_msg_(!success || IsMappedVS(vd, vdsz), "vd should be mapped now"); - _dbg_assert_msg_(!success || IsMappedVS(vs, vssz), "vs should be mapped now"); - _dbg_assert_msg_(!success || IsMappedVS(vt, vtsz), "vt should be mapped now"); - - return success; -} - -void FPURegCache::SimpleRegsV(const u8 *v, VectorSize vsz, int flags) { - const int n = GetNumVectorElements(vsz); - // TODO: Could be more optimal (in case of Discard or etc.) - for (int i = 0; i < n; ++i) { - SimpleRegV(v[i], flags); - } -} - -void FPURegCache::SimpleRegsV(const u8 *v, MatrixSize msz, int flags) { - const int n = GetMatrixSide(msz); - // TODO: Could be more optimal (in case of Discard or etc.) - for (int i = 0; i < n; ++i) { - for (int j = 0; j < n; ++j) { - SimpleRegV(v[j * 4 + i], flags); - } - } -} - -void FPURegCache::SimpleRegV(const u8 v, int flags) { - MIPSCachedFPReg &vr = vregs[v]; - // Special optimization: if it's in a single simd, we can keep it there. - if (vr.lane == 1 && xregs[VSX(&v)].mipsRegs[1] == -1) { - if (flags & MAP_DIRTY) { - xregs[VSX(&v)].dirty = true; - } - // Just change the lane to 0. - vr.lane = 0; - } else if (vr.lane != 0) { - // This will never end up in a register this way, so ignore dirty. - if ((flags & MAP_NOINIT) == MAP_NOINIT) { - // This will discard only this reg, and store the others. - DiscardV(v); - } else { - StoreFromRegisterV(v); - } - } else if (vr.away) { - // There are no immediates in the FPR reg file, so we already had this in a register. Make dirty as necessary. - if (flags & MAP_DIRTY) { - xregs[VX(v)].dirty = true; - } - _assert_msg_(vr.location.IsSimpleReg(), "not loaded and not simple."); - } - Invariant(); -} - -void FPURegCache::ReleaseSpillLock(int mipsreg) { - regs[mipsreg].locked = 0; -} - -void FPURegCache::ReleaseSpillLocks() { - for (int i = 0; i < NUM_MIPS_FPRS; i++) - regs[i].locked = 0; - for (int i = TEMP0; i < TEMP0 + MIPSState::NUM_X86_FPU_TEMPS; ++i) - DiscardR(i); -} - -void FPURegCache::MapReg(const int i, bool doLoad, bool makeDirty) { - pendingFlush = true; - _assert_msg_(!regs[i].location.IsImm(), "WTF - FPURegCache::MapReg - imm"); - _assert_msg_(i >= 0 && i < NUM_MIPS_FPRS, "WTF - FPURegCache::MapReg - invalid mips reg %d", i); - - if (!regs[i].away) { - // Reg is at home in the memory register file. Let's pull it out. - X64Reg xr = GetFreeXReg(); - _assert_msg_(xr < NUM_X_FPREGS, "WTF - FPURegCache::MapReg - invalid reg %d", (int)xr); - xregs[xr].mipsReg = i; - xregs[xr].dirty = makeDirty; - OpArg newloc = ::Gen::R(xr); - if (doLoad) { - emit->MOVSS(xr, regs[i].location); - } - regs[i].location = newloc; - regs[i].lane = 0; - regs[i].away = true; - } else if (regs[i].lane != 0) { - // Well, darn. This means we need to flush it. - // TODO: This could be more optimal. Also check flags. - StoreFromRegister(i); - MapReg(i, doLoad, makeDirty); - } else { - // There are no immediates in the FPR reg file, so we already had this in a register. Make dirty as necessary. - xregs[RX(i)].dirty |= makeDirty; - _assert_msg_(regs[i].location.IsSimpleReg(), "not loaded and not simple."); - } - Invariant(); -} - -static int MMShuffleSwapTo0(int lane) { - if (lane == 0) { - return _MM_SHUFFLE(3, 2, 1, 0); - } else if (lane == 1) { - return _MM_SHUFFLE(3, 2, 0, 1); - } else if (lane == 2) { - return _MM_SHUFFLE(3, 0, 1, 2); - } else if (lane == 3) { - return _MM_SHUFFLE(0, 2, 1, 3); - } else { - _assert_msg_(false, "MMShuffleSwapTo0: Invalid lane %d", lane); - return 0; - } -} - -void FPURegCache::StoreFromRegister(int i) { - _assert_msg_(!regs[i].location.IsImm(), "WTF - FPURegCache::StoreFromRegister - it's an imm"); - _assert_msg_(i >= 0 && i < NUM_MIPS_FPRS, "WTF - FPURegCache::StoreFromRegister - invalid mipsreg %i PC=%08x", i, js_->compilerPC); - - if (regs[i].away) { - X64Reg xr = regs[i].location.GetSimpleReg(); - _assert_msg_(xr < NUM_X_FPREGS, "WTF - FPURegCache::StoreFromRegister - invalid reg: x %i (mr: %i). PC=%08x", (int)xr, i, js_->compilerPC); - if (regs[i].lane != 0) { - const int *mri = xregs[xr].mipsRegs; - int seq = 1; - for (int j = 1; j < 4; ++j) { - if (mri[j] == -1) { - break; - } - if (mri[j] - 32 >= 128 && mri[j - 1] - 32 >= 128 && mri[j] == mri[j - 1] + 1) { - seq++; - } else if (mri[j] - 32 < 128 && mri[j - 1] - 32 < 128 && voffset[mri[j] - 32] == voffset[mri[j - 1] - 32] + 1) { - seq++; - } else { - break; - } - } - - const float *f = mri[0] - 32 < 128 ? &mips_->v[voffset[mri[0] - 32]] : &mips_->tempValues[mri[0] - 32 - 128]; - int align = (intptr_t)f & 0xf; - - // If we can do a multistore... - if ((seq == 2 && (align & 0x7) == 0) || seq == 4) { - OpArg newLoc = GetDefaultLocation(mri[0]); - if (xregs[xr].dirty) { - if (seq == 4 && align == 0) - emit->MOVAPS(newLoc, xr); - else if (seq == 4) - emit->MOVUPS(newLoc, xr); - else - emit->MOVQ_xmm(newLoc, xr); - } - for (int j = 0; j < seq; ++j) { - int mr = xregs[xr].mipsRegs[j]; - if (mr == -1) { - continue; - } - OpArg newLoc = GetDefaultLocation(mr); - regs[mr].location = newLoc; - regs[mr].away = false; - regs[mr].lane = 0; - xregs[xr].mipsRegs[j] = -1; - } - } else { - seq = 0; - } - // Store the rest. - for (int j = seq; j < 4; ++j) { - int mr = xregs[xr].mipsRegs[j]; - if (mr == -1) { - continue; - } - if (j != 0 && xregs[xr].dirty) { - emit->SHUFPS(xr, Gen::R(xr), MMShuffleSwapTo0(j)); - } - OpArg newLoc = GetDefaultLocation(mr); - if (xregs[xr].dirty) { - emit->MOVSS(newLoc, xr); - } - regs[mr].location = newLoc; - regs[mr].away = false; - regs[mr].lane = 0; - xregs[xr].mipsRegs[j] = -1; - } - } else { - OpArg newLoc = GetDefaultLocation(i); - xregs[xr].mipsReg = -1; - if (xregs[xr].dirty) { - emit->MOVSS(newLoc, xr); - } - regs[i].location = newLoc; - } - xregs[xr].dirty = false; - regs[i].away = false; - } else { - // _assert_msg_(false,"already stored"); - } - Invariant(); -} - -void FPURegCache::DiscardR(int i) { - _assert_msg_(!regs[i].location.IsImm(), "FPU can't handle imm yet."); - if (regs[i].away) { - X64Reg xr = regs[i].location.GetSimpleReg(); - _assert_msg_(xr < NUM_X_FPREGS, "DiscardR: MipsReg had bad X64Reg"); - // Note that we DO NOT write it back here. That's the whole point of Discard. - if (regs[i].lane != 0) { - // But we can't just discard all of them in SIMD, just the one lane. - // TODO: Potentially this could be more optimal (MOVQ or etc.) - xregs[xr].mipsRegs[regs[i].lane - 1] = -1; - regs[i].lane = 0; - for (int j = 0; j < 4; ++j) { - int mr = xregs[xr].mipsRegs[j]; - if (mr == -1) { - continue; - } - if (j != 0 && xregs[xr].dirty) { - emit->SHUFPS(xr, Gen::R(xr), MMShuffleSwapTo0(j)); - } - - OpArg newLoc = GetDefaultLocation(mr); - if (xregs[xr].dirty) { - emit->MOVSS(newLoc, xr); - } - regs[mr].location = newLoc; - regs[mr].away = false; - regs[mr].lane = 0; - xregs[xr].mipsRegs[j] = -1; - } - } else { - xregs[xr].mipsReg = -1; - } - xregs[xr].dirty = false; - regs[i].location = GetDefaultLocation(i); - regs[i].away = false; - regs[i].tempLocked = false; - } else { - // _assert_msg_(false,"already stored"); - regs[i].tempLocked = false; - } - Invariant(); -} - -void FPURegCache::DiscardVS(int vreg) { - _assert_msg_(!vregs[vreg].location.IsImm(), "FPU can't handle imm yet."); - - if (vregs[vreg].away) { - _assert_msg_(vregs[vreg].lane != 0, "VS expects a SIMD reg."); - X64Reg xr = vregs[vreg].location.GetSimpleReg(); - _assert_msg_(xr < NUM_X_FPREGS, "DiscardR: MipsReg had bad X64Reg"); - // Note that we DO NOT write it back here. That's the whole point of Discard. - for (int i = 0; i < 4; ++i) { - int mr = xregs[xr].mipsRegs[i]; - if (mr != -1) { - regs[mr].location = GetDefaultLocation(mr); - regs[mr].away = false; - regs[mr].tempLocked = false; - regs[mr].lane = 0; - } - xregs[xr].mipsRegs[i] = -1; - } - xregs[xr].dirty = false; - } else { - vregs[vreg].tempLocked = false; - } - Invariant(); -} - -bool FPURegCache::IsTempX(X64Reg xr) { - return xregs[xr].mipsReg >= TEMP0; -} - -int FPURegCache::GetTempR() { - pendingFlush = true; - for (int r = TEMP0; r < TEMP0 + MIPSState::NUM_X86_FPU_TEMPS; ++r) { - if (!regs[r].away && !regs[r].tempLocked) { - regs[r].tempLocked = true; - return r; - } - } - - _assert_msg_(false, "Regcache ran out of temp regs, might need to DiscardR() some."); - return -1; -} - -int FPURegCache::GetTempVS(u8 *v, VectorSize vsz) { - pendingFlush = true; - const int n = GetNumVectorElements(vsz); - - // Let's collect regs as we go, but try for n free in a row. - int found = 0; - for (int r = TEMP0; r <= TEMP0 + MIPSState::NUM_X86_FPU_TEMPS - n; ++r) { - if (regs[r].away || regs[r].tempLocked) { - continue; - } - - // How many free siblings does this have? - int seq = 1; - for (int i = 1; i < n; ++i) { - if (regs[r + i].away || regs[r + i].tempLocked) { - break; - } - ++seq; - } - - if (seq == n) { - // Got 'em. Exacty as many as we need. - for (int i = 0; i < n; ++i) { - v[i] = r + i - 32; - } - found = n; - break; - } - - if (found < n) { - v[found++] = r - 32; - } - } - - if (found != n) { - _assert_msg_(false, "Regcache ran out of temp regs, might need to DiscardR() some."); - return -1; - } - - for (int i = 0; i < n; ++i) { - regs[v[i] + 32].tempLocked = true; - } - - return 0; // ?? -} - -void FPURegCache::Flush() { - if (!pendingFlush) { - return; - } - for (int i = 0; i < NUM_MIPS_FPRS; i++) { - _assert_msg_(!regs[i].locked, "Somebody forgot to unlock MIPS reg %d.", i); - if (regs[i].away) { - if (regs[i].location.IsSimpleReg()) { - X64Reg xr = RX(i); - StoreFromRegister(i); - xregs[xr].dirty = false; - } else if (regs[i].location.IsImm()) { - StoreFromRegister(i); - } else { - _assert_msg_(false, "Jit64 - Flush unhandled case, reg %i PC: %08x", i, mips_->pc); - } - } - } - pendingFlush = false; - Invariant(); -} - -OpArg FPURegCache::GetDefaultLocation(int reg) const { - if (reg < 32) { - // Smaller than RIP addressing since we can use a byte offset. - return MDisp(CTXREG, reg * 4); - } else if (reg < 32 + 128) { - // Here, RIP has the advantage so let's use it when possible - if (useRip_) { - return M(&mips_->v[voffset[reg - 32]]); // rip accessible - } else { - return MIPSSTATE_VAR_ELEM32(v[0], voffset[reg - 32]); - } - } else { - if (useRip_) { - return M(&mips_->tempValues[reg - 32 - 128]); // rip accessible - } else { - return MIPSSTATE_VAR_ELEM32(tempValues[0], reg - 32 - 128); - } - } -} - -void FPURegCache::Invariant() const { -#if 0 - _assert_msg_(SanityCheck() == 0, "Sanity check failed: %d", SanityCheck()); -#endif -} - -static int GetMRMtx(int mr) { - if (mr < 32) - return -1; - if (mr >= 128 + 32) - return -1; - return ((mr - 32) >> 2) & 7; -} - -static int GetMRRow(int mr) { - if (mr < 32) - return -1; - if (mr >= 128 + 32) - return -1; - return ((mr - 32) >> 0) & 3; -} - -static int GetMRCol(int mr) { - if (mr < 32) - return -1; - if (mr >= 128 + 32) - return -1; - return ((mr - 32) >> 5) & 3; -} - -static bool IsMRTemp(int mr) { - return mr >= 128 + 32; -} - -int FPURegCache::SanityCheck() const { - for (int i = 0; i < NUM_MIPS_FPRS; i++) { - const MIPSCachedFPReg &mr = regs[i]; - - // FPR can never have imms. - if (mr.location.IsImm()) - return 1; - - bool reallyAway = mr.location.IsSimpleReg(); - if (reallyAway != mr.away) - return 2; - - if (mr.lane < 0 || mr.lane > 4) - return 3; - if (mr.lane != 0 && !reallyAway) - return 4; - - if (mr.away) { - Gen::X64Reg simple = mr.location.GetSimpleReg(); - if (mr.lane == 0) { - if (xregs[simple].mipsReg != i) - return 5; - for (int j = 1; j < 4; ++j) { - if (xregs[simple].mipsRegs[j] != -1) - return 6; - } - } else { - if (xregs[simple].mipsRegs[mr.lane - 1] != i) - return 7; - } - } - } - - for (int i = 0; i < NUM_X_FPREGS; ++i) { - const X64CachedFPReg &xr = xregs[i]; - bool hasReg = xr.mipsReg != -1; - if (!hasReg && xr.dirty) - return 8; - - bool hasMoreRegs = hasReg; - int mtx = -2; - int row = -2; - int col = -2; - bool rowMatched = true; - bool colMatched = true; - for (int j = 0; j < 4; ++j) { - if (xr.mipsRegs[j] == -1) { - hasMoreRegs = false; - continue; - } - if (xr.mipsRegs[j] >= NUM_MIPS_FPRS) { - return 13; - } - // We can't have a hole in the middle / front. - if (!hasMoreRegs) - return 9; - - const MIPSCachedFPReg &mr = regs[xr.mipsRegs[j]]; - if (!mr.location.IsSimpleReg(X64Reg(i))) - return 10; - - if (!IsMRTemp(xr.mipsRegs[j])) { - if (mtx == -2) - mtx = GetMRMtx(xr.mipsRegs[j]); - else if (mtx != GetMRMtx(xr.mipsRegs[j])) - return 11; - - if (row == -2) - row = GetMRRow(xr.mipsRegs[j]); - else if (row != GetMRRow(xr.mipsRegs[j])) - rowMatched = false; - - if (col == -2) - col = GetMRCol(xr.mipsRegs[j]); - else if (col != GetMRCol(xr.mipsRegs[j])) - colMatched = false; - } - } - if (!rowMatched && !colMatched) { - return 12; - } - } - - return 0; -} - -const int *FPURegCache::GetAllocationOrder(int &count) { - static const int allocationOrder[] = { -#if PPSSPP_ARCH(AMD64) - XMM6, XMM7, XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, XMM2, XMM3, XMM4, XMM5 -#elif PPSSPP_ARCH(X86) - XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, -#endif - }; - count = sizeof(allocationOrder) / sizeof(int); - return allocationOrder; -} - -X64Reg FPURegCache::GetFreeXReg() { - X64Reg res; - int obtained = GetFreeXRegs(&res, 1); - - _assert_msg_(obtained == 1, "Regcache ran out of regs"); - return res; -} - -int FPURegCache::GetFreeXRegs(X64Reg *res, int n, bool spill) { - pendingFlush = true; - int aCount; - const int *aOrder = GetAllocationOrder(aCount); - - _dbg_assert_msg_(n <= NUM_X_FPREGS - 2, "Cannot obtain that many regs."); - - int r = 0; - - for (int i = 0; i < aCount; i++) { - X64Reg xr = (X64Reg)aOrder[i]; - if (xregs[xr].mipsReg == -1) { - res[r++] = (X64Reg)xr; - if (r >= n) { - break; - } - } - } - - if (r < n && spill) { - // Okay, not found :(... Force grab one. - // TODO - add a pass to grab xregs whose mipsreg is not used in the next 3 instructions. - for (int i = 0; i < aCount; i++) { - X64Reg xr = (X64Reg)aOrder[i]; - int preg = xregs[xr].mipsReg; - _assert_msg_(preg >= -1 && preg < NUM_MIPS_FPRS, "WTF - FPURegCache::GetFreeXRegs - invalid mips reg %d in xr %d", preg, (int)xr); - - // We're only spilling here, so don't overlap. - if (preg != -1 && !regs[preg].locked) { - StoreFromRegister(preg); - res[r++] = xr; - if (r >= n) { - break; - } - } - } - } - - for (int i = r; i < n; ++i) { - res[i] = INVALID_REG; - } - return r; -} - -void FPURegCache::FlushX(X64Reg reg) { - if (reg >= NUM_X_FPREGS) { - _assert_msg_(false, "Flushing non existent reg"); - } else if (xregs[reg].mipsReg != -1) { - StoreFromRegister(xregs[reg].mipsReg); - } -} - -void FPURegCache::GetState(FPURegCacheState &state) const { - memcpy(state.regs, regs, sizeof(regs)); - memcpy(state.xregs, xregs, sizeof(xregs)); -} - -void FPURegCache::RestoreState(const FPURegCacheState& state) { - memcpy(regs, state.regs, sizeof(regs)); - memcpy(xregs, state.xregs, sizeof(xregs)); - pendingFlush = true; -} - -#endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/RegCacheFPU.h b/Core/MIPS/x86/RegCacheFPU.h deleted file mode 100644 index 2bd1d5ef49..0000000000 --- a/Core/MIPS/x86/RegCacheFPU.h +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright (c) 2012- PPSSPP Project. - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, version 2.0 or later versions. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License 2.0 for more details. - -// A copy of the GPL 2.0 should have been included with the program. -// If not, see http://www.gnu.org/licenses/ - -// Official git repository and contact information can be found at -// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. - -#pragma once - -#include "Common/x64Emitter.h" -#include "Core/MIPS/MIPS.h" -#include "Core/MIPS/MIPSVFPUUtils.h" - -#undef MAP_NOINIT - -// GPRs are numbered 0 to 31 -// VFPU regs are numbered 32 to 159. -// Then we have some temp regs for VFPU handling from 160 to 175. - -// Temp regs: 4 from S prefix, 4 from T prefix, 4 from D mask, and 4 for work (worst case.) -// But most of the time prefixes aren't used that heavily so we won't use all of them. - -// PLANS FOR PROPER SIMD -// 1, 2, 3, and 4-vectors will be loaded into single XMM registers -// Matrices will be loaded into pairs, triads, or quads of XMM registers - simply by loading -// the columns or the rows one by one. - -// On x86 this means that only one 4x4 matrix can be fully loaded at once but that's alright. -// We might want to keep "linearized" columns in memory. - -// Implement optimized vec/matrix multiplications of all types and transposes that -// take into account in which XMM registers the values are. Fallback: Just dump out the values -// and do it the old way. - -#include "ppsspp_config.h" - -enum { - TEMP0 = 32 + 128, - NUM_MIPS_FPRS = 32 + 128 + MIPSState::NUM_X86_FPU_TEMPS, -}; - -#if PPSSPP_ARCH(AMD64) -#define NUM_X_FPREGS 16 -#elif PPSSPP_ARCH(X86) -#define NUM_X_FPREGS 8 -#endif - -struct X64CachedFPReg { - union { - int mipsReg; - int mipsRegs[4]; - }; - bool dirty; -}; - -struct MIPSCachedFPReg { - Gen::OpArg location; - int lane; - bool away; // value not in source register (memory) - u8 locked; - // Only for temp regs. - bool tempLocked; -}; - -struct FPURegCacheState { - MIPSCachedFPReg regs[NUM_MIPS_FPRS]; - X64CachedFPReg xregs[NUM_X_FPREGS]; -}; - -namespace MIPSComp { - struct JitOptions; - struct JitState; -} - -enum { - MAP_DIRTY = 1, - MAP_NOINIT = 2 | MAP_DIRTY, - // Only for MapRegsV, MapRegsVS. - MAP_NOLOCK = 4, -}; - -// The PSP has 160 FP registers: 32 FPRs + 128 VFPU registers. -// Soon we will support them all. - -class FPURegCache -{ -public: - FPURegCache(); - ~FPURegCache() {} - - void Start(MIPSState *mipsState, MIPSComp::JitState *js, MIPSComp::JitOptions *jo, bool useRip); - void MapReg(int preg, bool doLoad = true, bool makeDirty = true); - void StoreFromRegister(int preg); - void StoreFromRegisterV(int preg) { - StoreFromRegister(preg + 32); - } - Gen::OpArg GetDefaultLocation(int reg) const; - void DiscardR(int freg); - void DiscardV(int vreg) { - DiscardR(vreg + 32); - } - void DiscardVS(int vreg); - bool IsTempX(Gen::X64Reg xreg); - int GetTempR(); - int GetTempV() { - return GetTempR() - 32; - } - int GetTempVS(u8 *v, VectorSize vsz); - - void SetEmitter(Gen::XEmitter *emitter) {emit = emitter;} - - // Flushes one register and reuses the register for another one. Dirtyness is implied. - void FlushRemap(int oldreg, int newreg); - - void Flush(); - int SanityCheck() const; - - const Gen::OpArg &R(int freg) const {return regs[freg].location;} - const Gen::OpArg &V(int vreg) const { - _dbg_assert_msg_(vregs[vreg].lane == 0, "SIMD reg %d used as V reg (use VS instead). pc=%08x", vreg, mips_->pc); - return vregs[vreg].location; - } - const Gen::OpArg &VS(const u8 *vs) const { - _dbg_assert_msg_(vregs[vs[0]].lane != 0, "V reg %d used as VS reg (use V instead). pc=%08x", vs[0], mips_->pc); - return vregs[vs[0]].location; - } - - Gen::X64Reg RX(int freg) const { - if (regs[freg].away && regs[freg].location.IsSimpleReg()) - return regs[freg].location.GetSimpleReg(); - _assert_msg_(false, "Not so simple - f%i", freg); - return (Gen::X64Reg)-1; - } - - Gen::X64Reg VX(int vreg) const { - _dbg_assert_msg_(vregs[vreg].lane == 0, "SIMD reg %d used as V reg (use VSX instead). pc=%08x", vreg, mips_->pc); - if (vregs[vreg].away && vregs[vreg].location.IsSimpleReg()) - return vregs[vreg].location.GetSimpleReg(); - _assert_msg_(false, "Not so simple - v%i", vreg); - return (Gen::X64Reg)-1; - } - - Gen::X64Reg VSX(const u8 *vs) const { - _dbg_assert_msg_(vregs[vs[0]].lane != 0, "V reg %d used as VS reg (use VX instead). pc=%08x", vs[0], mips_->pc); - if (vregs[vs[0]].away && vregs[vs[0]].location.IsSimpleReg()) - return vregs[vs[0]].location.GetSimpleReg(); - _assert_msg_(false, "Not so simple - v%i", vs[0]); - return (Gen::X64Reg)-1; - } - - // Just to avoid coding mistakes, defined here to prevent compilation. - void R(Gen::X64Reg r); - - // Register locking. Prevents them from being spilled. - void SpillLock(int p1, int p2=0xff, int p3=0xff, int p4=0xff); - void ReleaseSpillLock(int mipsreg); - void ReleaseSpillLocks(); - - bool IsMapped(int r) { - return R(r).IsSimpleReg(); - } - bool IsMappedV(int v) { - return vregs[v].lane == 0 && V(v).IsSimpleReg(); - } - // Whether the register holds a MIPS register or a temp. - bool IsXRegInUse(Gen::X64Reg reg) const { - return xregs[reg].mipsReg != -1; - } - bool IsMappedVS(u8 v) { - return vregs[v].lane != 0 && VS(&v).IsSimpleReg(); - } - bool IsMappedVS(const u8 *v, VectorSize vsz); - bool CanMapVS(const u8 *v, VectorSize vsz); - - void MapRegV(int vreg, int flags); - void MapRegsV(int vec, VectorSize vsz, int flags); - void MapRegsV(const u8 *v, VectorSize vsz, int flags); - void SpillLockV(int vreg) { - SpillLock(vreg + 32); - } - void SpillLockV(const u8 *v, VectorSize vsz); - void SpillLockV(int vec, VectorSize vsz); - void ReleaseSpillLockV(int vreg) { - ReleaseSpillLock(vreg + 32); - } - void ReleaseSpillLockV(const u8 *vec, VectorSize sz); - - // TODO: This may trash XMM0/XMM1 some day. - void MapRegsVS(const u8 *v, VectorSize vsz, int flags); - bool TryMapRegsVS(const u8 *v, VectorSize vsz, int flags); - bool TryMapDirtyInVS(const u8 *vd, VectorSize vdsz, const u8 *vs, VectorSize vssz, bool avoidLoad = true); - bool TryMapDirtyInInVS(const u8 *vd, VectorSize vdsz, const u8 *vs, VectorSize vssz, const u8 *vt, VectorSize vtsz, bool avoidLoad = true); - // TODO: If s/t overlap differently, need read-only copies? Maybe finalize d? Major design flaw... - // TODO: Matrix versions? Cols/Rows? - // No MapRegVS, that'd be silly. - - void SimpleRegsV(const u8 *v, VectorSize vsz, int flags); - void SimpleRegsV(const u8 *v, MatrixSize msz, int flags); - void SimpleRegV(const u8 v, int flags); - - void GetState(FPURegCacheState &state) const; - void RestoreState(const FPURegCacheState& state); - - MIPSState *mips_ = nullptr; - - void FlushX(Gen::X64Reg reg); - Gen::X64Reg GetFreeXReg(); - int GetFreeXRegs(Gen::X64Reg *regs, int n, bool spill = true); - - void Invariant() const; - -private: - const int *GetAllocationOrder(int &count); - void SetupInitialRegs(); - - // These are intentionally not public so the interface is "locked" or "unlocked", no levels. - void ReduceSpillLock(int mreg); - void ReduceSpillLockV(int vreg) { - ReduceSpillLock(vreg + 32); - } - void ReduceSpillLockV(const u8 *vec, VectorSize sz); - - Gen::X64Reg LoadRegsVS(const u8 *v, int n); - - MIPSCachedFPReg regs[NUM_MIPS_FPRS]{}; - X64CachedFPReg xregs[NUM_X_FPREGS]{}; - MIPSCachedFPReg *vregs; - - bool useRip_; - bool pendingFlush; - bool initialReady = false; - MIPSCachedFPReg regsInitial[NUM_MIPS_FPRS]; - X64CachedFPReg xregsInitial[NUM_X_FPREGS]; - - Gen::XEmitter *emit = nullptr; - MIPSComp::JitState *js_; - MIPSComp::JitOptions *jo_; -}; diff --git a/UI/DeveloperToolsScreen.cpp b/UI/DeveloperToolsScreen.cpp index c570b671ae..62dd71f78a 100644 --- a/UI/DeveloperToolsScreen.cpp +++ b/UI/DeveloperToolsScreen.cpp @@ -152,13 +152,10 @@ void DeveloperToolsScreen::CreateGeneralTab(UI::LinearLayout *list) { }); if (!canUseJit) { core->HideChoice(1); - core->HideChoice(3); } - // TODO: Enable "JIT using IR" on more architectures. ARM32 needs more testing. - // Note also that Loongarch and RISC-V only have a jit-ir backend, and it's used for the JIT option, so the fourth option isn't shown. -#if !PPSSPP_ARCH(X86) && !PPSSPP_ARCH(AMD64) && !PPSSPP_ARCH(ARM64) + // Every native JIT except ARM32's is IR-based and used for the JIT option, and ARM32 uses its + // legacy JIT for both, so "JIT using IR" is the same as "JIT" everywhere. core->HideChoice(3); -#endif // Only the interpreter raises these, but don't hide it behind the CPU core choice - it'd just // look like the setting had vanished. diff --git a/UI/ImDebugger/ImJitViewer.cpp b/UI/ImDebugger/ImJitViewer.cpp index f10d87bf6b..382562f498 100644 --- a/UI/ImDebugger/ImJitViewer.cpp +++ b/UI/ImDebugger/ImJitViewer.cpp @@ -33,6 +33,14 @@ void ImJitViewerWindow::Draw(ImConfig &cfg, ImControl &control) { return; } + // Every native JIT except ARM32's is IR-based. +#if PPSSPP_ARCH(ARM) + const bool hasIR = core == CPUCore::IR_INTERPRETER; +#else + const bool hasIR = true; +#endif + const bool hasTarget = core == CPUCore::JIT || core == CPUCore::JIT_IR; + // Options menu if (ImGui::BeginPopup("Options")) { // ImGui::Checkbox("Auto-scroll", &AutoScroll); @@ -55,7 +63,7 @@ void ImJitViewerWindow::Draw(ImConfig &cfg, ImControl &control) { // Three or four columns: One table for blocks that can be sorted in various ways, // then the remaining are either MIPS/IR/TARGET or MIPS/TARGET or MIPS/IR depending on the JIT type. int numColumns = 3; - if (core == CPUCore::JIT_IR) { + if (hasIR && hasTarget) { numColumns = 4; } @@ -135,10 +143,10 @@ void ImJitViewerWindow::Draw(ImConfig &cfg, ImControl &control) { if (ImGui::BeginTable("columns", numColumns, ImGuiTableFlags_RowBg | ImGuiTableFlags_BordersH | ImGuiTableFlags_Resizable)) { ImGui::TableSetupColumn("Blocks"); ImGui::TableSetupColumn("MIPS"); - if (core == CPUCore::JIT_IR || core == CPUCore::IR_INTERPRETER) { + if (hasIR) { ImGui::TableSetupColumn("IR"); } - if (core == CPUCore::JIT_IR || core == CPUCore::JIT) { + if (hasTarget) { ImGui::TableSetupColumn(TARGET); } @@ -221,7 +229,7 @@ void ImJitViewerWindow::Draw(ImConfig &cfg, ImControl &control) { } ImGui::EndChild(); - if (core == CPUCore::JIT_IR || core == CPUCore::IR_INTERPRETER) { + if (hasIR) { ImGui::TableNextColumn(); ImGui::BeginChild("IRPane", ImVec2(0, 0), true, ImGuiWindowFlags_HorizontalScrollbar); @@ -232,7 +240,7 @@ void ImJitViewerWindow::Draw(ImConfig &cfg, ImControl &control) { ImGui::EndChild(); } - if (core == CPUCore::JIT_IR || core == CPUCore::JIT) { + if (hasTarget) { ImGui::TableNextColumn(); ImGui::BeginChild("TargetPane", ImVec2(0, 0), true, ImGuiWindowFlags_HorizontalScrollbar); diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index 080025e76f..27bf91e2ff 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -276,9 +276,6 @@ - - - @@ -309,10 +306,6 @@ - - - - @@ -561,16 +554,6 @@ - - - - - - - - - - @@ -620,17 +603,6 @@ - - - - - - - - - - - diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index 3d445e6e01..1d5768e59e 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -183,16 +183,6 @@ - - - - - - - - - - @@ -242,17 +232,6 @@ - - - - - - - - - - - @@ -617,9 +596,6 @@ - - - @@ -650,10 +626,6 @@ - - - - diff --git a/android/jni/Android.mk b/android/jni/Android.mk index c8f689f96c..82486e02ab 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -498,17 +498,6 @@ LOCAL_WHOLE_STATIC_LIBRARIES += ppsspp_common libchdr lua pugixml ifeq ($(TARGET_ARCH_ABI),x86_64) ARCH_FILES := \ - $(SRC)/Core/MIPS/x86/CompALU.cpp \ - $(SRC)/Core/MIPS/x86/CompBranch.cpp \ - $(SRC)/Core/MIPS/x86/CompFPU.cpp \ - $(SRC)/Core/MIPS/x86/CompLoadStore.cpp \ - $(SRC)/Core/MIPS/x86/CompVFPU.cpp \ - $(SRC)/Core/MIPS/x86/CompReplace.cpp \ - $(SRC)/Core/MIPS/x86/Asm.cpp \ - $(SRC)/Core/MIPS/x86/Jit.cpp \ - $(SRC)/Core/MIPS/x86/JitSafeMem.cpp \ - $(SRC)/Core/MIPS/x86/RegCache.cpp \ - $(SRC)/Core/MIPS/x86/RegCacheFPU.cpp \ $(SRC)/Core/MIPS/x86/X64IRAsm.cpp \ $(SRC)/Core/MIPS/x86/X64IRCompALU.cpp \ $(SRC)/Core/MIPS/x86/X64IRCompBranch.cpp \ @@ -535,16 +524,6 @@ ARCH_FILES := \ ArmEmitterTest.cpp else ifeq ($(findstring arm64-v8a,$(TARGET_ARCH_ABI)),arm64-v8a) ARCH_FILES := \ - $(SRC)/Core/MIPS/ARM64/Arm64CompALU.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64CompBranch.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64CompFPU.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64CompLoadStore.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64CompVFPU.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64CompReplace.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64Asm.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64Jit.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64RegCache.cpp \ - $(SRC)/Core/MIPS/ARM64/Arm64RegCacheFPU.cpp \ $(SRC)/Core/MIPS/ARM64/Arm64IRAsm.cpp \ $(SRC)/Core/MIPS/ARM64/Arm64IRCompALU.cpp \ $(SRC)/Core/MIPS/ARM64/Arm64IRCompBranch.cpp \ diff --git a/docs/cpu-equivalence.md b/docs/cpu-equivalence.md index 0d3b1bea7d..af3fd6aa4d 100644 --- a/docs/cpu-equivalence.md +++ b/docs/cpu-equivalence.md @@ -27,7 +27,7 @@ Environment variables: | `PPSSPP_CPUFUZZ_CLASSES` | weighted string of `a` (ALU), `i` (immediate), `f` (FPU), `v` (VFPU), `m` (memory) | | `PPSSPP_CPUFUZZ_CODE` | comma-separated hex instructions to run instead of the generated code | | `PPSSPP_CPUFUZZ_CODE_SEED` | apply `PPSSPP_CPUFUZZ_CODE` to this seed only | -| `PPSSPP_CPUFUZZ_SKIP` | cores to skip, by digit: 1 IR interpreter, 2 JIT, 3 JIT IR | +| `PPSSPP_CPUFUZZ_SKIP` | cores to skip, by digit: 1 IR interpreter, 2 JIT | | `PPSSPP_CPUFUZZ_REPORTS` | how many differences to minimize and print (default 20) | | `PPSSPP_CPUFUZZ_VERBOSE` | print every case before running it, to find one that crashes | | `PPSSPP_CPUFUZZ_DUMP` | write every core's end state of every case to a file | @@ -99,7 +99,8 @@ PSP's. Where matching the PSP is expensive, agreeing with each other comes first ### Answered on hardware Each of these has a pspautotests test recorded on a PSP (6.61). Distinct outputs across the eight -configurations (four cores, ARM64 and x86-64 under Rosetta), and lines that differ from the PSP: +configurations (four cores, ARM64 and x86-64 under Rosetta), and lines that differ from the PSP. +The `jit` columns were measured with the legacy JITs, since removed; `jit` is now `jit-ir` on both. | Test | Distinct outputs | ARM64 interp / ir / jit / jit-ir | x86-64 interp / ir / jit / jit-ir | |---|---|---|---| @@ -159,7 +160,7 @@ sign. `0x7F800001` or `0xFF800001`. The sign is + for add, sub and every summing op, the XOR of the operand signs for mul, div and vscl, the input's sign for vrcp, vrsq, vsin and vasin, and + for vsqrt, vlog2 and vexp2. vsat0/vsat1, vabs and vneg pass NaN bits through. The cores make the - host's NaN instead, so x86-64 and ARM disagree (and the ARM64 JIT's vsat quiets a NaN). The + host's NaN instead, so x86-64 and ARM disagree. The branch `cpu-equiv-4` fixes the single-lane ops (vadd, vsub, vmul, vdiv, vscl, vcrs, vsat0/1, vbfy, vocp, vsocp) on every core, through four new IR ops and checks in the legacy JITs, and is held back until its cost on every multiplication is measured. The horizontal ops wait for the @@ -173,7 +174,7 @@ sign. around, vtfm/vhtfm size mismatches, vmfvc past register 15 reading 0, vcmov with bit 20 set), now including `vcmov` with imm3 = 7, which tests a CC bit that is always clear: vcmovt never moves and vcmovf always does (fixed on every core). The JITs still differ on the single-lane - dot products and vcrsp/vqmul at .s/.p (for the accurate dot product), and the IR and x86 JIT on + dot products and vcrsp/vqmul at .s/.p (for the accurate dot product), and the IR cores on matrix registers whose offset doesn't fit the size. ### Known, no hardware test needed @@ -181,7 +182,6 @@ sign. 8. **Divide-by-zero flags.** Only the interpreter sets the cause and flag bits for `div.s` by zero (see `Int_FPU3op`). The other cores leave fcr31 alone. Making the others match needs a new IR op or an interpreter fallback for div.s, and the latter costs every division. -9. The legacy ARM64 JIT ignores the D prefix's write mask for `viim.s`/`vfim.s`. ### ARM32 @@ -205,9 +205,8 @@ UNITTEST=1 SKIPAPP=1`, run over adb). ARM32 has three cores: `CreateNativeJit` r ### Not looked into yet -- With VFPU code in the mix, the legacy x86-64 JIT differs from the interpreter in 205 of 2000 - default cases and the legacy ARM64 JIT in 127, against 147 for the IR cores. Most of those - looked at are the horizontal ops (items 4 and 6) and item 7. +- With VFPU code in the mix, the IR cores differ from the interpreter in 147 of 2000 default + cases. Most of those looked at are the horizontal ops (items 4 and 6) and item 7. - Branches, delay slots, and VFPU prefixes across blocks: the generated code is straight-line. ## Unrelated diff --git a/libretro/Makefile.common b/libretro/Makefile.common index 4550f50e42..3ddae16cdd 100644 --- a/libretro/Makefile.common +++ b/libretro/Makefile.common @@ -939,16 +939,6 @@ ifeq ($(WITH_DYNAREC),1) else ifeq ($(TARGET_ARCH),arm64) DYNAFLAGS += -D_ARCH_64 SOURCES_CXX += $(COMMONDIR)/Arm64Emitter.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64Asm.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompALU.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompBranch.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompFPU.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompLoadStore.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompVFPU.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64CompReplace.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64Jit.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64RegCache.cpp \ - $(COREDIR)/MIPS/ARM64/Arm64RegCacheFPU.cpp \ $(COREDIR)/MIPS/ARM64/Arm64IRAsm.cpp \ $(COREDIR)/MIPS/ARM64/Arm64IRCompALU.cpp \ $(COREDIR)/MIPS/ARM64/Arm64IRCompBranch.cpp \ @@ -981,17 +971,6 @@ ifeq ($(WITH_DYNAREC),1) $(COMMONDIR)/x64Analyzer.cpp \ $(COMMONDIR)/ABI.cpp \ $(COMMONDIR)/Thunk.cpp \ - $(COREDIR)/MIPS/x86/CompReplace.cpp \ - $(COREDIR)/MIPS/x86/CompBranch.cpp \ - $(COREDIR)/MIPS/x86/Asm.cpp \ - $(COREDIR)/MIPS/x86/CompALU.cpp \ - $(COREDIR)/MIPS/x86/CompVFPU.cpp \ - $(COREDIR)/MIPS/x86/CompLoadStore.cpp \ - $(COREDIR)/MIPS/x86/CompFPU.cpp \ - $(COREDIR)/MIPS/x86/Jit.cpp \ - $(COREDIR)/MIPS/x86/JitSafeMem.cpp \ - $(COREDIR)/MIPS/x86/RegCache.cpp \ - $(COREDIR)/MIPS/x86/RegCacheFPU.cpp \ $(COREDIR)/MIPS/x86/X64IRAsm.cpp \ $(COREDIR)/MIPS/x86/X64IRCompALU.cpp \ $(COREDIR)/MIPS/x86/X64IRCompBranch.cpp \ diff --git a/unittest/JitHarness.cpp b/unittest/JitHarness.cpp index 7e58502e25..b9cdfbdd16 100644 --- a/unittest/JitHarness.cpp +++ b/unittest/JitHarness.cpp @@ -206,7 +206,7 @@ bool TestJit() { printf("\n"); - double jit_speed = 0.0, jit_ir_speed = 0.0, ir_speed = 0.0, interp_speed = 0.0; + double jit_speed = 0.0, ir_speed = 0.0, interp_speed = 0.0; if (compileSuccess) { interp_speed = ExecCPUTest(); checkMemoryClean("Interpreter"); @@ -215,12 +215,8 @@ bool TestJit() { checkMemoryClean("IR interpreter"); MIPSComp::jit->ClearCache(); mipsr4k.UpdateCore(CPUCore::JIT); - jit_speed = ExecCPUTest(false); + jit_speed = ExecCPUTest(false); // not clearing, so the below can do things. checkMemoryClean("JIT"); - MIPSComp::jit->ClearCache(); - mipsr4k.UpdateCore(CPUCore::JIT_IR); - jit_ir_speed = ExecCPUTest(false); // not clearing, so the below can do things. - checkMemoryClean("JIT IR"); // Disassemble JitBlockCacheDebugInterface *cache = MIPSComp::jit->GetBlockCacheDebugInterface(); @@ -235,7 +231,7 @@ bool TestJit() { if (lines.size() > cutoff) printf("...\n"); } - printf("Jit was %fx faster than interp, IR was %fx faster, JIT IR %fx.\n\n", jit_speed / interp_speed, ir_speed / interp_speed, jit_ir_speed / interp_speed); + printf("Jit was %fx faster than interp, IR was %fx faster.\n\n", jit_speed / interp_speed, ir_speed / interp_speed); } printf("\n"); @@ -270,7 +266,6 @@ static bool CheckReplacementHooks(u32 func, u32 size, const std::vector &cod static const CPUCore cores[] = { CPUCore::JIT, CPUCore::IR_INTERPRETER, - CPUCore::JIT_IR, }; for (CPUCore core : cores) { mipsr4k.UpdateCore(core); @@ -921,7 +916,6 @@ bool TestCPUEquivalence() { CPUCore::INTERPRETER, CPUCore::IR_INTERPRETER, CPUCore::JIT, - CPUCore::JIT_IR, }; const int numCores = (int)ARRAY_SIZE(cores); @@ -1096,7 +1090,6 @@ bool TestCPURoundingMode() { static const CPUCore cores[] = { CPUCore::IR_INTERPRETER, CPUCore::JIT, - CPUCore::JIT_IR, }; // No flags, then ctc1 left to the interpreter, then everything the JITs can leave to it. static const u32 disableFlags[] = { 0, (u32)MIPSComp::JitDisable::FPU_XFER, 0xFFFF }; diff --git a/unittest/TestX64Emitter.cpp b/unittest/TestX64Emitter.cpp index 6230742a8e..81efb99e01 100644 --- a/unittest/TestX64Emitter.cpp +++ b/unittest/TestX64Emitter.cpp @@ -5,8 +5,6 @@ #include "Common/CPUDetect.h" #include "Common/x64Analyzer.h" #include "Common/x64Emitter.h" -#include "Core/MIPS/x86/RegCacheFPU.h" -#include "Core/MIPS/x86/Jit.h" #include "Core/MIPS/JitCommon/JitState.h" #include "Core/MIPS/JitCommon/JitCommon.h" #include "Core/MIPS/MIPSVFPUUtils.h"