diff --git a/Common/MemoryUtil.cpp b/Common/MemoryUtil.cpp index d4de2cf131..7817c37db3 100644 --- a/Common/MemoryUtil.cpp +++ b/Common/MemoryUtil.cpp @@ -44,6 +44,12 @@ #ifndef _WIN32 #include #endif + +#ifndef MAP_NORESERVE +// Not implemented on BSDs +#define MAP_NORESERVE 0 +#endif + static int hint_location; #ifdef __APPLE__ #define MEM_PAGE_SIZE (PAGE_SIZE) @@ -254,6 +260,54 @@ void *AllocateMemoryPages(size_t size, uint32_t memProtFlags) { return ptr; } +void *ReserveMemoryPages(size_t size) { +#ifdef _WIN32 + if (sys_info.dwPageSize == 0) + GetSystemInfo(&sys_info); + size = ppsspp_round_page(size); +#if PPSSPP_PLATFORM(UWP) + void *ptr = VirtualAllocFromApp(0, size, MEM_RESERVE, PAGE_NOACCESS); +#else + void *ptr = VirtualAlloc(0, size, MEM_RESERVE, PAGE_NOACCESS); +#endif + if (!ptr) { + ERROR_LOG(Log::MemMap, "Failed to reserve %d bytes of address space", (int)size); + return nullptr; + } +#else + size = ppsspp_round_page(size); + void *ptr = mmap(0, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE, -1, 0); + if (ptr == MAP_FAILED) { + ERROR_LOG(Log::MemMap, "Failed to reserve %d bytes of address space: errno=%d", (int)size, errno); + return nullptr; + } +#endif + return ptr; +} + +bool CommitMemoryPages(void *ptr, size_t size) { + uintptr_t page_size = GetMemoryProtectPageSize(); + uintptr_t start = (uintptr_t)ptr & ~(page_size - 1); + uintptr_t end = ((uintptr_t)ptr + size + page_size - 1) & ~(page_size - 1); +#ifdef _WIN32 +#if PPSSPP_PLATFORM(UWP) + void *result = VirtualAllocFromApp((void *)start, end - start, MEM_COMMIT, PAGE_READWRITE); +#else + void *result = VirtualAlloc((void *)start, end - start, MEM_COMMIT, PAGE_READWRITE); +#endif + if (!result) { + ERROR_LOG(Log::MemMap, "CommitMemoryPages failed!\n%s", GetLastErrorMsg().c_str()); + return false; + } +#else + if (mprotect((void *)start, end - start, PROT_READ | PROT_WRITE) != 0) { + ERROR_LOG(Log::MemMap, "CommitMemoryPages failed (%p)! errno=%d (%s)", (void *)start, errno, strerror(errno)); + return false; + } +#endif + return true; +} + void *AllocateAlignedMemory(size_t size, size_t alignment) { #ifdef _WIN32 void* ptr = _aligned_malloc(size, alignment); diff --git a/Common/MemoryUtil.h b/Common/MemoryUtil.h index 00dd786aa4..478d02270a 100644 --- a/Common/MemoryUtil.h +++ b/Common/MemoryUtil.h @@ -45,6 +45,11 @@ void* AllocateMemoryPages(size_t size, uint32_t memProtFlags); bool ProtectMemoryPages(const void* ptr, size_t size, uint32_t memProtFlags); void FreeMemoryPages(void* ptr, size_t size); +// Reserves address space without backing it. Accessing it faults until CommitMemoryPages +// makes a range readable and writable (zero-filled, backed lazily). Free with FreeMemoryPages. +void *ReserveMemoryPages(size_t size); +bool CommitMemoryPages(void *ptr, size_t size); + // Regular aligned memory. Don't try to apply memory protection willy-nilly to memory allocated this way as in-page alignment is unknown (though could be checked). // No longer asserts, will return nullptr on failure. // WARNING: Not necessarily malloc-compatible! diff --git a/Common/MemoryUtilHorizon.cpp b/Common/MemoryUtilHorizon.cpp index e0ae8090ad..d908663d57 100644 --- a/Common/MemoryUtilHorizon.cpp +++ b/Common/MemoryUtilHorizon.cpp @@ -49,6 +49,20 @@ void *AllocateMemoryPages(size_t size, uint32_t memProtFlags) { return ptr; } +// No lazy commit here, the whole reservation is backed (and zeroed) up front. +void *ReserveMemoryPages(size_t size) { + size = ppsspp_round_page(size); + void *ptr = memalign(MEM_PAGE_SIZE, size); + if (ptr) { + memset(ptr, 0, size); + } + return ptr; +} + +bool CommitMemoryPages(void *ptr, size_t size) { + return true; +} + void *AllocateAlignedMemory(size_t size, size_t alignment) { void* ptr = memalign(alignment, size); diff --git a/Core/CwCheat.cpp b/Core/CwCheat.cpp index a0ae9f8e40..88ffddd2cf 100644 --- a/Core/CwCheat.cpp +++ b/Core/CwCheat.cpp @@ -28,13 +28,8 @@ // Cache invalidation // -// It should be obvious why we need to invalidate the instruction cache (effectively, in PPSSPP's case, -// the JIT translation cache) for writes. But currently we do also need to do it for reads, in case -// cheats check what MIPS opcode is at an address - the JIT sometimes overwrites them. Invalidating -// the cache will restore that. -// -// Long term, we should get rid of the instruction overwriting hack, or we could use Memory::Read_Instruction -// with workarounds for 8 and 16 bit reads. +// Writes invalidate the instruction cache (effectively, in PPSSPP's case, the JIT translation cache), +// so code a cheat patches gets recompiled. Reads don't need to: PSP memory never holds JIT data. static int CheatEvent = -1; static CWCheatEngine *cheatEngine; @@ -768,7 +763,6 @@ void CWCheatEngine::ApplyMemoryOperator(const CheatOperation &op, uint32_t(*oper bool CWCheatEngine::TestIf(const CheatOperation &op, bool(*oper)(int, int)) const { if (Memory::IsValidRange(op.addr, op.sz)) { - InvalidateICache(op.addr, op.sz); // See note at top of file int memoryValue = 0; if (op.sz == 1) @@ -785,8 +779,6 @@ bool CWCheatEngine::TestIf(const CheatOperation &op, bool(*oper)(int, int)) cons bool CWCheatEngine::TestIfAddr(const CheatOperation &op, bool(*oper)(int, int)) const { if (Memory::IsValidRange(op.addr, op.sz) && Memory::IsValidRange(op.ifAddrTypes.compareAddr, op.sz)) { - InvalidateICache(op.addr, op.sz); // See note at top of file - InvalidateICache(op.ifAddrTypes.compareAddr, op.sz); int memoryValue1 = 0; int memoryValue2 = 0; @@ -883,7 +875,6 @@ void CWCheatEngine::ExecuteOp(const CheatOperation &op, const CheatCode &cheat, } case CheatOp::CopyBytesFrom: if (Memory::IsValidRange(op.addr, op.val) && Memory::IsValidRange(op.copyBytesFrom.destAddr, op.val)) { - InvalidateICache(op.addr, op.val); // See note at top of file InvalidateICache(op.copyBytesFrom.destAddr, op.val); Memory::Memcpy(op.copyBytesFrom.destAddr, op.addr, op.val, "CwCheat"); @@ -960,7 +951,6 @@ void CWCheatEngine::ExecuteOp(const CheatOperation &op, const CheatCode &cheat, case CheatOp::Assert: if (Memory::IsValidRange(op.addr, 4)) { - InvalidateICache(op.addr, 4); // See note at top of file if (Memory::ReadUnchecked_U32(op.addr) != op.val) { i = cheat.lines.size(); } @@ -1053,7 +1043,6 @@ void CWCheatEngine::ExecuteOp(const CheatOperation &op, const CheatCode &cheat, if (!Memory::IsValidRange(op.addr + op.pointerCommands.baseOffset, 4)) { break; } - InvalidateICache(op.addr + op.pointerCommands.baseOffset, 4); // See note at top of file u32 base = Memory::ReadUnchecked_U32(op.addr + op.pointerCommands.baseOffset); u32 val = op.val; int type = op.pointerCommands.type; @@ -1062,12 +1051,10 @@ void CWCheatEngine::ExecuteOp(const CheatOperation &op, const CheatCode &cheat, switch (line.part1 >> 28) { case 0x1: // type copy byte if (Memory::IsValidRange(op.addr, 4) && Memory::IsValidRange(op.addr + op.pointerCommands.baseOffset, 4)) { - InvalidateICache(op.addr, 4); // See note at top of file u32 srcAddr = Memory::ReadUnchecked_U32(op.addr) + op.pointerCommands.offset; u32 dstAddr = Memory::ReadUnchecked_U32(op.addr + op.pointerCommands.baseOffset) + (line.part1 & 0x0FFFFFFF); if (Memory::IsValidRange(dstAddr, val) && Memory::IsValidRange(srcAddr, val)) { InvalidateICache(dstAddr, val); - InvalidateICache(srcAddr, val); // See note at top of file Memory::Memcpy(dstAddr, srcAddr, val, "CwCheat"); } // Don't perform any further action. @@ -1094,7 +1081,6 @@ void CWCheatEngine::ExecuteOp(const CheatOperation &op, const CheatCode &cheat, walkOffset = -walkOffset; } if (Memory::IsValidRange(base + walkOffset, 4)) { - InvalidateICache(base + walkOffset, 4); // See note at top of file base = Memory::ReadUnchecked_U32(base + walkOffset); } break; diff --git a/Core/Debugger/DisassemblyManager.cpp b/Core/Debugger/DisassemblyManager.cpp index f73c35552d..3b861d722a 100644 --- a/Core/Debugger/DisassemblyManager.cpp +++ b/Core/Debugger/DisassemblyManager.cpp @@ -986,7 +986,7 @@ bool GetDisasmAddressText(u32 address, char *dest, size_t bufSize, bool abbrevia } } else { if (showData) { - const u32 encoding = Memory::Read_Instruction(address, true).encoding; + const u32 encoding = Memory::Read_Instruction(address).encoding; snprintf(dest, bufSize, "%08X %08X", address, encoding); } else { snprintf(dest, bufSize, "%08X", address); diff --git a/Core/Debugger/WebSocket/MemorySubscriber.cpp b/Core/Debugger/WebSocket/MemorySubscriber.cpp index 37e75816ed..7c7cd7f67f 100644 --- a/Core/Debugger/WebSocket/MemorySubscriber.cpp +++ b/Core/Debugger/WebSocket/MemorySubscriber.cpp @@ -46,56 +46,6 @@ DebuggerSubscriber *WebSocketMemoryInit(DebuggerEventHandlerMap &map) { return nullptr; } -struct AutoDisabledReplacements { - AutoDisabledReplacements() {} - AutoDisabledReplacements(AutoDisabledReplacements &&other); - AutoDisabledReplacements(const AutoDisabledReplacements &) = delete; - AutoDisabledReplacements &operator =(const AutoDisabledReplacements &) = delete; - ~AutoDisabledReplacements(); - - std::map replacements; - std::vector emuhacks; - bool saved = false; -}; - -// Call this from within a Core_RunOnCPUThread() callback - see Core_RunOnCPUThread() in Core.h. -// No longer needs to pause a running CPU to do this safely: we're already running on the CPU thread -// by the time this is called, so nothing else can be concurrently executing MIPS code or touching the -// JIT's emuhack ops on this thread while we hold onto them below. -// -// Deliberately does NOT take a CoreShutdownLock: memory teardown only ever happens on the -// CPU thread too, so there's nothing to guard against, and taking it here deadlocked against the -// Win32 debugger's paint handlers. See the lock ordering section in AGENTS.md. -// -// Important: Only use keepReplacements=false when reading, not writing. -static AutoDisabledReplacements LockMemory(bool keepReplacements) { - AutoDisabledReplacements result; - if (!keepReplacements) { - result.saved = true; - // Okay, save so we can restore later. - result.replacements = SaveAndClearReplacements(); - if (MIPSComp::jit) { - result.emuhacks = MIPSComp::jit->SaveAndClearEmuHackOps(); - } - } - return result; -} - -AutoDisabledReplacements::AutoDisabledReplacements(AutoDisabledReplacements &&other) { - replacements = std::move(other.replacements); - emuhacks = std::move(other.emuhacks); - saved = other.saved; - other.saved = false; -} - -AutoDisabledReplacements::~AutoDisabledReplacements() { - if (saved) { - if (MIPSComp::jit) - MIPSComp::jit->RestoreSavedEmuHackOps(emuhacks); - RestoreSavedReplacements(replacements); - } -} - // Read a byte from memory (memory.read_u8) // // Parameters: @@ -121,7 +71,6 @@ void WebSocketMemoryReadU8(DebuggerRequest &req) { // Route the actual memory read to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); JsonWriter &json = req.Respond(); json.writeUint("value", Memory::ReadUnchecked_U8(addr)); // Alias: cpu.getReg and cpu.getAllRegs call this uintValue. Same number, both names. @@ -154,7 +103,6 @@ void WebSocketMemoryReadU16(DebuggerRequest &req) { // Route the actual memory read to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); JsonWriter &json = req.Respond(); json.writeUint("value", Memory::ReadUnchecked_U16(addr)); json.writeUint("uintValue", Memory::ReadUnchecked_U16(addr)); @@ -186,7 +134,6 @@ void WebSocketMemoryReadU32(DebuggerRequest &req) { // Route the actual memory read to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); JsonWriter &json = req.Respond(); json.writeUint("value", Memory::ReadUnchecked_U32(addr)); json.writeUint("uintValue", Memory::ReadUnchecked_U32(addr)); @@ -198,7 +145,6 @@ void WebSocketMemoryReadU32(DebuggerRequest &req) { // Parameters: // - address: unsigned integer address for the start of the memory range. // - size: unsigned integer specifying size of memory range. -// - replacements: optional, false to ignore PPSSPP replacements in MIPS code. // // Response (same event name): // - base64: base64 encode of binary data. @@ -209,10 +155,6 @@ void WebSocketMemoryRead(DebuggerRequest &req) { uint32_t size; if (!req.ParamU32("size", &size)) return; - bool replacements = true; - if (!req.ParamBool("replacements", &replacements, DebuggerParamType::OPTIONAL)) - return; - if (!currentDebugMIPS->isAlive() || !Memory::IsActive()) return req.Fail("CPU not started"); // This only depends on addr/size, not on anything CPU-thread-owned, so fail fast here rather @@ -223,13 +165,12 @@ void WebSocketMemoryRead(DebuggerRequest &req) { return req.Fail("Invalid size"); // Route the actual memory read to the CPU thread instead of poking at it directly from this - // WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Only the raw copy (which - // needs replacements/emuhacks disabled) happens on the CPU thread - the base64 encoding itself + // WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Only the raw copy happens + // on the CPU thread - the base64 encoding itself // happens back on this WebSocket thread afterward, so it doesn't block the CPU thread's frame // pump for a large 'size'. std::vector raw(size); Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(replacements); if (size != 0) memcpy(raw.data(), Memory::GetPointerUnchecked(addr), size); }); @@ -287,7 +228,6 @@ void WebSocketMemoryReadString(DebuggerRequest &req) { // the CPU thread - the base64 encoding itself happens back on this WebSocket thread afterward. std::string raw; Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); // Let's try to avoid crashing and get a safe length. const uint8_t *p = Memory::GetPointerUnchecked(addr); size_t longest = Memory::ClampValidSizeAt(addr, Memory::g_MemorySize); @@ -333,7 +273,6 @@ void WebSocketMemoryWriteU8(DebuggerRequest &req) { // Route the actual memory write to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); Memory::WriteUnchecked_U8(val, addr); currentMIPS->InvalidateICacheRangeDeferred(addr, 1); Reporting::NotifyDebugger(); @@ -372,7 +311,6 @@ void WebSocketMemoryWriteU16(DebuggerRequest &req) { // Route the actual memory write to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); Memory::WriteUnchecked_U16(val, addr); currentMIPS->InvalidateICacheRangeDeferred(addr, 2); Reporting::NotifyDebugger(); @@ -410,7 +348,6 @@ void WebSocketMemoryWriteU32(DebuggerRequest &req) { // Route the actual memory write to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); Memory::WriteUnchecked_U32(val, addr); currentMIPS->InvalidateICacheRangeDeferred(addr, 4); Reporting::NotifyDebugger(); @@ -452,7 +389,6 @@ void WebSocketMemoryWrite(DebuggerRequest &req) { // Route the actual memory write to the CPU thread instead of poking at it directly // from this WebSocket handler thread - see Core_RunOnCPUThread() in Core.h. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); currentMIPS->InvalidateICacheRangeDeferred(addr, size); if (size != 0) { Memory::MemcpyUnchecked(addr, &value[0], size); @@ -570,7 +506,6 @@ void WebSocketMemorySearch(DebuggerRequest &req) { // CPU thread's own frame pump for its duration - at least the parameter validation above no // longer costs a round trip through the queue first. Core_RunOnCPUThread([&] { - AutoDisabledReplacements memLock = LockMemory(true); const uint8_t *base = Memory::GetPointerUnchecked(addr); std::vector matches; diff --git a/Core/ELF/ElfReader.cpp b/Core/ELF/ElfReader.cpp index 1bf565e11b..2fbf5be361 100644 --- a/Core/ELF/ElfReader.cpp +++ b/Core/ELF/ElfReader.cpp @@ -462,7 +462,7 @@ void ElfReader::LoadRelocations2(int rel_seg) ERROR_LOG_REPORT(Log::Loader, "Rel2: invalid lo16 type! %x", flag); } - op = Memory::Read_Instruction(rel_offset, true).encoding; + op = Memory::Read_Instruction(rel_offset).encoding; const u32 origOp = op; VERBOSE_LOG(Log::Loader, "Rel2: %5d: CMD=0x%04X flag=%x type=%d off_seg=%d offset=%08x addr_seg=%d op=%08x", rcount, cmd, flag, type, off_seg, rel_base, addr_seg, op); diff --git a/Core/HLE/ReplaceTables.cpp b/Core/HLE/ReplaceTables.cpp index 31f44f7cc6..217a07b9d6 100644 --- a/Core/HLE/ReplaceTables.cpp +++ b/Core/HLE/ReplaceTables.cpp @@ -30,8 +30,10 @@ #include "Core/Debugger/SymbolMap.h" #include "Core/MemMap.h" #include "Core/MIPS/JitCommon/JitCommon.h" +#include "Core/MIPS/MIPS.h" #include "Core/MIPS/MIPSCodeUtils.h" #include "Core/MIPS/MIPSAnalyst.h" +#include "Core/MIPS/MIPSTables.h" #include "Core/HLE/ReplaceTables.h" #include "Core/HLE/FunctionWrappers.h" #include "Core/HLE/sceDisplay.h" @@ -40,6 +42,7 @@ #include "GPU/GPU.h" #include "GPU/GPUCommon.h" #include "Common/Math/SIMDHeaders.h" +#include "ext/xxhash.h" enum class GPUReplacementSkip { MEMSET = 1, @@ -125,8 +128,6 @@ static int Replace_memcpy() { return 10; } - // Some games use memcpy on executable code. We need to flush emuhack ops. - currentMIPS->InvalidateICacheRangeImmediate(srcPtr, bytes); if ((skipGPUReplacements & (int)GPUReplacementSkip::MEMCPY) == 0) { if (Memory::IsVRAMAddress(destPtr) || Memory::IsVRAMAddress(srcPtr)) { skip = gpu->PerformMemoryCopy(destPtr, srcPtr, bytes); @@ -188,7 +189,6 @@ static int Replace_memcpy_jak() { bool sliced = false; static constexpr uint32_t SLICE_SIZE = 32768; - currentMIPS->InvalidateICacheRangeImmediate(srcPtr, bytes); if ((skipGPUReplacements & (int)GPUReplacementSkip::MEMCPY) == 0) { if (Memory::IsVRAMAddress(destPtr) || Memory::IsVRAMAddress(srcPtr)) { skip = gpu->PerformMemoryCopy(destPtr, srcPtr, bytes); @@ -258,9 +258,6 @@ static int Replace_memcpy16() { u32 bytes = PARAM(2) * 16; bool skip = false; - // Some games use memcpy on executable code. We need to flush emuhack ops. - if (bytes != 0) - currentMIPS->InvalidateICacheRangeImmediate(srcPtr, bytes); if ((skipGPUReplacements & (int)GPUReplacementSkip::MEMCPY) == 0 && bytes != 0) { if (Memory::IsVRAMAddress(destPtr) || Memory::IsVRAMAddress(srcPtr)) { skip = gpu->PerformMemoryCopy(destPtr, srcPtr, bytes); @@ -328,9 +325,7 @@ static int Replace_memmove() { u32 bytes = PARAM(2); bool skip = false; - // Some games use memcpy on executable code. We need to flush emuhack ops. if ((skipGPUReplacements & (int)GPUReplacementSkip::MEMMOVE) == 0 && bytes != 0) { - currentMIPS->InvalidateICacheRangeImmediate(srcPtr, bytes); if (Memory::IsVRAMAddress(destPtr) || Memory::IsVRAMAddress(srcPtr)) { skip = gpu->PerformMemoryCopy(destPtr, srcPtr, bytes); } @@ -704,11 +699,11 @@ static int Replace_dl_write_matrix() { } static bool GetMIPSStaticAddress(u32 &addr, s32 lui_offset, s32 lw_offset) { - const MIPSOpcode upper = Memory::Read_Instruction(currentMIPS->pc + lui_offset, true); + const MIPSOpcode upper = Memory::Read_Instruction(currentMIPS->pc + lui_offset); if (upper != MIPS_MAKE_LUI(MIPS_GET_RT(upper), upper & 0xffff)) { return false; } - const MIPSOpcode lower = Memory::Read_Instruction(currentMIPS->pc + lw_offset, true); + const MIPSOpcode lower = Memory::Read_Instruction(currentMIPS->pc + lw_offset); if (lower != MIPS_MAKE_LW(MIPS_GET_RT(lower), MIPS_GET_RS(lower), lower & 0xffff)) { if (lower != MIPS_MAKE_ORI(MIPS_GET_RT(lower), MIPS_GET_RS(lower), lower & 0xffff)) { return false; @@ -719,7 +714,7 @@ static bool GetMIPSStaticAddress(u32 &addr, s32 lui_offset, s32 lw_offset) { } static bool GetMIPSGPAddress(u32 &addr, s32 offset) { - const MIPSOpcode loadOp = Memory::Read_Instruction(currentMIPS->pc + offset, true); + const MIPSOpcode loadOp = Memory::Read_Instruction(currentMIPS->pc + offset); if (MIPS_GET_RS(loadOp) == MIPS_REG_GP) { s16 gpoff = (s16)(u16)(loadOp & 0x0000FFFF); addr = currentMIPS->r[MIPS_REG_GP] + gpoff; @@ -844,7 +839,7 @@ static int Hook_brandish_download_frame() { return 0; } const u32 fb_info = Memory::ReadUnchecked_U32(fb_infoaddr); - const MIPSOpcode fb_index_load = Memory::Read_Instruction(currentMIPS->pc + 0x38, true); + const MIPSOpcode fb_index_load = Memory::Read_Instruction(currentMIPS->pc + 0x38); if (fb_index_load != MIPS_MAKE_LW(MIPS_GET_RT(fb_index_load), MIPS_GET_RS(fb_index_load), fb_index_load & 0xffff)) { return 0; } @@ -974,7 +969,7 @@ static int Hook_soranokiseki_sc_download_frame() { } const u32 fb_info = Memory::ReadUnchecked_U32(fb_infoaddr); - const MIPSOpcode fb_index_load = Memory::Read_Instruction(currentMIPS->pc + 0x34, true); + const MIPSOpcode fb_index_load = Memory::Read_Instruction(currentMIPS->pc + 0x34); if (fb_index_load != MIPS_MAKE_LW(MIPS_GET_RT(fb_index_load), MIPS_GET_RS(fb_index_load), fb_index_load & 0xffff)) { return 0; } @@ -1035,7 +1030,7 @@ static int Hook_danganronpa1_1_download_frame() { } static int Hook_danganronpa1_2_download_frame() { - const MIPSOpcode instruction = Memory::Read_Instruction(currentMIPS->pc + 0x8, true); + const MIPSOpcode instruction = Memory::Read_Instruction(currentMIPS->pc + 0x8); const int reg_num = instruction >> 11 & 31; const u32 fb_base = currentMIPS->r[reg_num]; const u32 fb_offset = currentMIPS->r[MIPS_REG_V0]; @@ -1205,7 +1200,7 @@ static int Hook_youkosohitsujimura_download_frame() { } static int Hook_zettai_hero_update_minimap_tex() { - const MIPSOpcode storeOffset = Memory::Read_Instruction(currentMIPS->pc + 4, true); + const MIPSOpcode storeOffset = Memory::Read_Instruction(currentMIPS->pc + 4); const uint32_t texAddr = currentMIPS->r[MIPS_REG_A0] + SignExtend16ToS32(storeOffset); const uint32_t texSize = 64 * 64 * 1; const uint32_t writeAddr = currentMIPS->r[MIPS_REG_V1] + SignExtend16ToS32(storeOffset); @@ -1378,11 +1373,11 @@ static int Hook_kingdomhearts_download_frame() { return 0; } - const MIPSOpcode fb_offset_table_lui = Memory::Read_Instruction(get_fb_offset + 0x08, true); // 0x08821EB8 + const MIPSOpcode fb_offset_table_lui = Memory::Read_Instruction(get_fb_offset + 0x08); // 0x08821EB8 if (fb_offset_table_lui != MIPS_MAKE_LUI(MIPS_REG_A1, fb_offset_table_lui & 0xFFFF)) { return 0; } - const MIPSOpcode fb_offset_table_addiu = Memory::Read_Instruction(get_fb_offset + 0x10, true); // 0x08821EC0 + const MIPSOpcode fb_offset_table_addiu = Memory::Read_Instruction(get_fb_offset + 0x10); // 0x08821EC0 if (fb_offset_table_addiu != MIPS_MAKE_ADDIU(MIPS_REG_A1, MIPS_REG_A1, fb_offset_table_addiu & 0xFFFF)) { return 0; } @@ -1747,9 +1742,69 @@ static const ReplacementTableEntry entries[] = { }; -static std::map replacedInstructions; +struct InstalledReplacement { + int index; + // Unmasked, for invalidation. The native block cache doesn't fold mirrors when filtering ranges. + u32 address; + // What was in memory when it was installed. If the game overwrites it, the hook is dropped. + u32 origOp; + // The function the hook was found in. A HOOKEXIT hook sits on a jr ra, which new code loaded + // over the function is likely to have too, so we also drop the hook when the function's + // hash changes - checked whenever the game invalidates the icache over it. + u32 funcStart; + u32 funcSize; + u64 funcHash; +}; + +// Keyed by address & BLOCK_SHADOW_MASK, so mirrors share a hook the way they share memory. +static std::map installedReplacements; static std::unordered_map> replacementNameLookup; +// What the block shadow holds at a hooked instruction without a compiled block. The JIT's value +// sends the dispatcher on to the compiler, which handles the hook. +static u32 shadowHookValue = Memory::BLOCK_SHADOW_HOOK_NO_JIT; + +// An entry holding a block is left alone: invalidating the block destroys it, which writes the +// hook value then (see Memory::ClearBlockShadow). +static void MarkHook(u32 address) { + if (Memory::ReadBlockShadow(address) == 0) { + Memory::WriteBlockShadow(address, shadowHookValue); + } +} + +// Call after erasing the hook from the map, so destroying a block there clears its entry to 0. +static void UnmarkHook(u32 address) { + Memory::ClearBlockShadow(address, shadowHookValue); +} + +// Deferred, since hooks change in HLE calls (module loading) that can run inside the block. +static void InvalidateReplacedAddress(u32 address) { + currentMIPS->InvalidateICacheRangeDeferred(address, 4); +} + +// Immediates are masked out like in MIPSAnalyst::HashFunctions, which matched the function, so a +// relocation doesn't drop the hook. +static u64 HashReplacedFunc(u32 start, u32 size) { + if (!Memory::IsValidRange(start, size)) { + return 0; + } + std::vector buffer; + buffer.reserve(size / 4); + for (u32 addr = start; addr + 4 <= start + size; addr += 4) { + const MIPSOpcode op = Memory::ReadUnchecked_Instruction(addr); + const MIPSInfo flags = MIPSGetInfo(op); + u32 validbits = 0xFFFFFFFF; + if (flags & IN_IMM16) { + validbits &= ~0xFFFF; + } + if (flags & IN_IMM26) { + validbits &= ~0x03FFFFFF; + } + buffer.push_back(op.encoding & validbits); + } + return XXH3_64bits(buffer.data(), buffer.size() * sizeof(u32)); +} + void Replacement_Init() { for (int i = 0; i < (int)ARRAY_SIZE(entries); i++) { const auto entry = &entries[i]; @@ -1762,10 +1817,29 @@ void Replacement_Init() { } void Replacement_Shutdown() { - replacedInstructions.clear(); + RestoreReplacedInstructions(0, 0xFFFFFFFF); replacementNameLookup.clear(); } +void Replacement_SetBlockShadowHook(u32 value) { + const u32 oldValue = shadowHookValue; + shadowHookValue = value; + for (const auto &[key, repl] : installedReplacements) { + const u32 entry = Memory::ReadBlockShadow(key); + if (entry == 0 || entry == oldValue) { + Memory::WriteBlockShadow(key, value); + } + } +} + +u32 Replacement_GetBlockShadowHook() { + return shadowHookValue; +} + +bool Replacement_IsHooked(u32 address) { + return installedReplacements.count(address & Memory::BLOCK_SHADOW_MASK) != 0; +} + int GetNumReplacementFuncs() { return ARRAY_SIZE(entries); } @@ -1791,47 +1865,47 @@ const ReplacementTableEntry *GetReplacementFunc(size_t i) { return &entries[i]; } -static bool WriteReplaceInstruction(u32 address, int index) { - u32 prevInstr = Memory::Read_Instruction(address, false).encoding; - if (MIPS_IS_REPLACEMENT(prevInstr)) { - int prevIndex = prevInstr & MIPS_EMUHACK_VALUE_MASK; - if (prevIndex == index) { +static bool WriteReplaceInstruction(u32 address, int index, u32 funcStart, u32 funcSize, u64 funcHash) { + if (!Memory::IsValid4AlignedAddress(address)) { + return false; + } + const u32 key = address & Memory::BLOCK_SHADOW_MASK; + const u32 memOp = Memory::ReadUnchecked_U32(address); + auto iter = installedReplacements.find(key); + if (iter != installedReplacements.end() && iter->second.origOp == memOp) { + if (iter->second.index == index) { return false; } - WARN_LOG(Log::HLE, "Replacement func changed at %08x (%d -> %d)", address, prevIndex, index); - // Make sure we don't save the old replacement. - prevInstr = replacedInstructions[address]; + WARN_LOG(Log::HLE, "Replacement func changed at %08x (%d -> %d)", address, iter->second.index, index); } - - if (MIPS_IS_RUNBLOCK(Memory::ReadUnchecked_U32(address))) { - WARN_LOG(Log::HLE, "Replacing jitted func address %08x", address); - } - replacedInstructions[address] = prevInstr; - Memory::WriteUnchecked_U32(MIPS_EMUHACK_CALL_REPLACEMENT | index, address); + installedReplacements[key] = { index, address, memOp, funcStart, funcSize, funcHash }; + InvalidateReplacedAddress(address); + MarkHook(address); return true; } void WriteReplaceInstructions(u32 address, u64 hash, int size) { std::vector indexes = GetReplacementFuncIndexes(hash, size); + const u64 funcHash = indexes.empty() ? 0 : HashReplacedFunc(address, size); for (int index : indexes) { bool didReplace = false; const ReplacementTableEntry *entry = GetReplacementFunc(index); if (entry->flags & REPFLAG_HOOKEXIT) { // When hooking func exit, we search for jr ra, and replace those. for (u32 offset = 0; offset < (u32)size; offset += 4) { - const u32 op = Memory::Read_Instruction(address + offset, false).encoding; + const u32 op = Memory::Read_Instruction(address + offset).encoding; if (op == MIPS_MAKE_JR_RA()) { - if (WriteReplaceInstruction(address + offset, index)) { + if (WriteReplaceInstruction(address + offset, index, address, size, funcHash)) { didReplace = true; } } } } else if (entry->flags & REPFLAG_HOOKENTER) { - if (WriteReplaceInstruction(address + entry->hookOffset, index)) { + if (WriteReplaceInstruction(address + entry->hookOffset, index, address, size, funcHash)) { didReplace = true; } } else { - if (WriteReplaceInstruction(address, index)) { + if (WriteReplaceInstruction(address, index, address, size, funcHash)) { didReplace = true; } } @@ -1842,79 +1916,80 @@ void WriteReplaceInstructions(u32 address, u64 hash, int size) { } } -// address is valid here. void RestoreReplacedInstruction(u32 address) { - const u32 curInstr = Memory::ReadUnchecked_U32(address); - if (MIPS_IS_REPLACEMENT(curInstr)) { - Memory::WriteUnchecked_U32(replacedInstructions[address], address); + const u32 key = address & Memory::BLOCK_SHADOW_MASK; + auto iter = installedReplacements.find(key); + if (iter != installedReplacements.end()) { + const u32 hookAddress = iter->second.address; + installedReplacements.erase(iter); + InvalidateReplacedAddress(hookAddress); + UnmarkHook(hookAddress); NOTICE_LOG(Log::HLE, "Restored replaced func at %08x", address); - } else { - NOTICE_LOG(Log::HLE, "Replaced func changed at %08x", address); } - replacedInstructions.erase(address); } -// startaddr and endaddr are valid here. void RestoreReplacedInstructions(u32 startAddr, u32 endAddr) { if (endAddr == startAddr) return; // Need to be in order, or we'll hang. if (endAddr < startAddr) std::swap(endAddr, startAddr); - const auto start = replacedInstructions.lower_bound(startAddr); - const auto end = replacedInstructions.upper_bound(endAddr); - int restored = 0; + const u32 maskedStart = startAddr & Memory::BLOCK_SHADOW_MASK; + u32 maskedEnd = endAddr & Memory::BLOCK_SHADOW_MASK; + if (maskedEnd < maskedStart || endAddr - startAddr >= Memory::BLOCK_SHADOW_MASK) { + // Wraps under the mask. + maskedEnd = Memory::BLOCK_SHADOW_MASK; + } + const auto start = installedReplacements.lower_bound(maskedStart); + const auto end = installedReplacements.upper_bound(maskedEnd); + std::vector restored; for (auto it = start; it != end; ++it) { - const u32 addr = it->first; - const u32 curInstr = Memory::ReadUnchecked_U32(addr); - if (MIPS_IS_REPLACEMENT(curInstr)) { - Memory::WriteUnchecked_U32(it->second, addr); - ++restored; - } + restored.push_back(it->second.address); } - INFO_LOG(Log::HLE, "Restored %d replaced funcs between %08x-%08x", restored, startAddr, endAddr); - replacedInstructions.erase(start, end); + // Erase first, so destroying blocks there doesn't put the hook back. + installedReplacements.erase(start, end); + for (u32 hookAddress : restored) { + InvalidateReplacedAddress(hookAddress); + UnmarkHook(hookAddress); + } + INFO_LOG(Log::HLE, "Restored %d replaced funcs between %08x-%08x", (int)restored.size(), startAddr, endAddr); } -std::map SaveAndClearReplacements() { - std::map saved; - for (const auto &[addr, instr] : replacedInstructions) { - if (!Memory::IsValid4AlignedAddress(addr)) { - continue; - } - // This will not retain jit blocks. - const u32 curInstr = Memory::Read_Opcode_JIT(addr).encoding; - if (MIPS_IS_REPLACEMENT(curInstr)) { - saved[addr] = curInstr; - Memory::WriteUnchecked_U32(instr, addr); - } - } - - // TODO: Should we not also call replacedInstructions.clear(); ? - return saved; -} - -void RestoreSavedReplacements(const std::map &saved) { - for (const auto &[addr, instr] : saved) { - // Just put the replacements back. - if (Memory::IsValid4AlignedAddress(addr)) { - Memory::WriteUnchecked_U32(instr, addr); +void Replacement_CheckRange(u32 address, u32 length) { + const u32 start = address & Memory::BLOCK_SHADOW_MASK; + const u32 end = length >= Memory::BLOCK_SHADOW_MASK ? Memory::BLOCK_SHADOW_MASK : start + length; + std::vector dropped; + for (auto it = installedReplacements.begin(); it != installedReplacements.end(); ) { + const InstalledReplacement &repl = it->second; + const u32 funcStart = repl.funcStart & Memory::BLOCK_SHADOW_MASK; + if (funcStart < end && funcStart + repl.funcSize > start && HashReplacedFunc(repl.funcStart, repl.funcSize) != repl.funcHash) { + NOTICE_LOG(Log::HLE, "Replaced func changed at %08x, dropping its hook at %08x", repl.funcStart, repl.address); + dropped.push_back(repl.address); + it = installedReplacements.erase(it); } else { - ERROR_LOG(Log::HLE, "RestoreSavedReplacements: Invalid address %08x", addr); + ++it; } } + // The hook can be outside the range (a HOOKEXIT on a later jr ra), so its blocks need invalidating too. + for (u32 hookAddress : dropped) { + InvalidateReplacedAddress(hookAddress); + UnmarkHook(hookAddress); + } } -bool GetReplacedOpAt(u32 address, u32 *op) { - u32 instr = Memory::Read_Opcode_JIT(address).encoding; - if (MIPS_IS_REPLACEMENT(instr)) { - auto iter = replacedInstructions.find(address); - if (iter != replacedInstructions.end()) { - *op = iter->second; - return true; - } else { - return false; - } +MIPSOpcode GetReplacementOpAt(u32 address) { + const u32 key = address & Memory::BLOCK_SHADOW_MASK; + const u32 memOp = Memory::ReadUnchecked_U32(address); + auto iter = installedReplacements.find(key); + if (iter == installedReplacements.end()) { + return MIPSOpcode(memOp); } - return false; + if (iter->second.origOp != memOp) { + // The game wrote over the hooked instruction, so the hook goes with it. + NOTICE_LOG(Log::HLE, "Replaced func changed at %08x", address); + installedReplacements.erase(iter); + UnmarkHook(address); + return MIPSOpcode(memOp); + } + return MIPSOpcode(MIPS_EMUHACK_CALL_REPLACEMENT | iter->second.index); } diff --git a/Core/HLE/ReplaceTables.h b/Core/HLE/ReplaceTables.h index 33b7cc0ed0..7979391457 100644 --- a/Core/HLE/ReplaceTables.h +++ b/Core/HLE/ReplaceTables.h @@ -41,6 +41,7 @@ #include #include "Common/CommonTypes.h" +#include "Core/MemMap.h" #include "Core/MIPS/JitCommon/JitCommon.h" typedef int (* ReplaceFunc)(); @@ -72,11 +73,31 @@ int GetNumReplacementFuncs(); std::vector GetReplacementFuncIndexes(u64 hash, int funcSize); const ReplacementTableEntry *GetReplacementFunc(size_t index); +// Installed replacements never touch PSP memory. A hooked instruction gets a nonzero block shadow +// entry: the JIT's block there, or the hook value if there's none. The CPU cores look up every +// instruction with a nonzero entry when they run or compile it, and substitute the CallRepl +// pseudo-op. void WriteReplaceInstructions(u32 address, u64 hash, int size); void RestoreReplacedInstruction(u32 address); void RestoreReplacedInstructions(u32 startAddr, u32 endAddr); -bool GetReplacedOpAt(u32 address, u32 *op); +// Drops hooks in functions the game has changed. Called when the icache is invalidated over them. +void Replacement_CheckRange(u32 address, u32 length); + +// The JIT's hook value, which its dispatcher sends to the compiler. Rewrites the hooks' entries. +void Replacement_SetBlockShadowHook(u32 value); +u32 Replacement_GetBlockShadowHook(); +bool Replacement_IsHooked(u32 address); + +// Only call for a nonzero block shadow entry. Returns the CallRepl pseudo-op, or the instruction in +// memory if there's no hook there or the game has overwritten the hooked instruction (which also +// drops the hook). Address must be valid. +MIPSOpcode GetReplacementOpAt(u32 address); + +// What the CPU runs at a valid address: the CallRepl pseudo-op if hooked, otherwise memory. +inline MIPSOpcode ReadExecutedOp(u32 address) { + if (*Memory::GetBlockShadowEntry(address) != 0) { + return GetReplacementOpAt(address); + } + return Memory::ReadUnchecked_Instruction(address); +} -// For savestates. If you call SaveAndClearReplacements(), you must call RestoreSavedReplacements(). -std::map SaveAndClearReplacements(); -void RestoreSavedReplacements(const std::map &saved); diff --git a/Core/HLE/sceDmac.cpp b/Core/HLE/sceDmac.cpp index c311cf0bc4..4a5f591856 100644 --- a/Core/HLE/sceDmac.cpp +++ b/Core/HLE/sceDmac.cpp @@ -53,7 +53,6 @@ static int __DmacMemcpy(MIPSState *mips, u32 dst, u32 src, u32 size) { } gpu->NotifyVideoCopy(dst, src, size); if (!skip && size != 0) { - mips->InvalidateICacheRangeDeferred(src, size); if (Memory::IsValidRange(dst, size) && Memory::IsValidRange(src, size)) { memcpy(Memory::GetPointerWriteUnchecked(dst), Memory::GetPointerUnchecked(src), size); } diff --git a/Core/HLE/sceKernelInterrupt.cpp b/Core/HLE/sceKernelInterrupt.cpp index 00be5940c3..f1a6b6492a 100644 --- a/Core/HLE/sceKernelInterrupt.cpp +++ b/Core/HLE/sceKernelInterrupt.cpp @@ -715,11 +715,6 @@ static u32 sceKernelMemset(u32 addr, u32 fillc, u32 n) { } static u32 sceKernelMemcpy(u32 dst, u32 src, u32 size) { - // Some games copy from executable code. We need to flush emuhack ops. - if (size != 0) { - currentMIPS->InvalidateICacheRangeDeferred(src, size); - } - bool skip = false; if (Memory::IsVRAMAddress(src) || Memory::IsVRAMAddress(dst)) { skip = gpu->PerformMemoryCopy(dst, src, size); diff --git a/Core/HLE/sceKernelModule.cpp b/Core/HLE/sceKernelModule.cpp index 403311b718..a39cd17150 100644 --- a/Core/HLE/sceKernelModule.cpp +++ b/Core/HLE/sceKernelModule.cpp @@ -581,7 +581,7 @@ struct WriteVarSymbolState { }; static void WriteVarSymbol(WriteVarSymbolState &state, u32 exportAddress, u32 relocAddress, u8 type, bool reverse = false) { - u32 relocData = Memory::Read_Instruction(relocAddress, true).encoding; + u32 relocData = Memory::Read_Instruction(relocAddress).encoding; switch (type) { case R_MIPS_NONE: @@ -625,7 +625,7 @@ static void WriteVarSymbol(WriteVarSymbolState &state, u32 exportAddress, u32 re // The R_MIPS_LO16 and R_MIPS_HI16 will often be *different* relocAddress values. HI16RelocInfo reloc; reloc.addr = relocAddress; - reloc.data = Memory::Read_Instruction(relocAddress, true).encoding; + reloc.data = Memory::Read_Instruction(relocAddress).encoding; state.lastHI16Relocs.push_back(reloc); state.lastHI16Processed = false; break; diff --git a/Core/MIPS/ARM/ArmAsm.cpp b/Core/MIPS/ARM/ArmAsm.cpp index e5a8f8adf1..a91efc6b26 100644 --- a/Core/MIPS/ARM/ArmAsm.cpp +++ b/Core/MIPS/ARM/ArmAsm.cpp @@ -200,17 +200,15 @@ void ArmJit::GenerateFixedCode() { } LDR(R0, CTXREG, offsetof(MIPSState, pc)); - // TODO: In practice, do we ever run code from uncached space (| 0x40000000)? If not, we can remove this BIC. - BIC(R0, R0, Operand2(0xC0, 4)); // &= 0x3FFFFFFF + static_assert(Memory::BLOCK_SHADOW_ENTRY_MASK == 0x0FFFFFFC, "BICs below assume the 32-bit shadow mask"); + BIC(R0, R0, Operand2(0xF0, 4)); // &= 0x0FFFFFFF. Rotation is to the right, in 2-bit increments. + BIC(R0, R0, Operand2(3)); + MOVI2R(R1, (u32)(uintptr_t)Memory::blockShadow); dispatcherFetch = GetCodePtr(); - LDR(R0, MEMBASEREG, R0); - AND(R1, R0, Operand2(0xFF, 4)); // rotation is to the right, in 2-bit increments. - BIC(R0, R0, Operand2(0xFF, 4)); - CMP(R1, Operand2(MIPS_EMUHACK_OPCODE >> 24, 4)); - SetCC(CC_EQ); - // IDEA - we have 26 bits, why not just use offsets from base of code? - // Another idea: Shift the bloc number left by two in the op, this would let us do - // LDR(R0, R9, R0); here, replacing the next instructions. + LDR(R0, R1, R0); + // The entry is the block's offset into the code space, or 0 for none. + CMP(R0, Operand2(0)); + SetCC(CC_NEQ); #if PPSSPP_PLATFORM(IOS) // On iOS, R9 (JITBASEREG) is volatile. We have to reload it. MOVI2R(JITBASEREG, (u32)(uintptr_t)GetBasePtr()); @@ -220,6 +218,9 @@ void ArmJit::GenerateFixedCode() { SetCC(CC_AL); // No block found, let's jit + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + blockShadowHook = GetCodePtr(); SaveDowncount(); RestoreRoundingMode(true); QuickCallFunctionR(R2, (void *)&MIPSComp::JitAt, CTXREG); diff --git a/Core/MIPS/ARM/ArmJit.cpp b/Core/MIPS/ARM/ArmJit.cpp index 8af7087b28..9d7f6db98f 100644 --- a/Core/MIPS/ARM/ArmJit.cpp +++ b/Core/MIPS/ARM/ArmJit.cpp @@ -90,7 +90,7 @@ static u32 JitMemCheck(u32 pc) { return 0; // Note: pc may be the delay slot. - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); s32 offset = SignExtend16ToS32(op & 0xFFFF); if (MIPSGetInfo(op) & IS_VFPU) offset &= 0xFFFC; @@ -196,10 +196,12 @@ void ArmJit::ClearCache() blocks.Clear(); ClearCodeSpace(0); GenerateFixedCode(); + // The compile path may have moved, and replacement hooks' block shadow entries point to it. + Replacement_SetBlockShadowHook(GetBlockShadowHookValue()); } void ArmJit::InvalidateCacheAt(u32 em_address, int length) { - if (blocks.RangeMayHaveEmuHacks(em_address, em_address + length)) { + if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { blocks.InvalidateICache(em_address, length); } } @@ -301,7 +303,8 @@ u32 ArmJit::GetCompilerPC() { } MIPSOpcode ArmJit::GetOffsetInstruction(int offset) { - return Memory::Read_Instruction(GetCompilerPC() + 4 * offset); + const u32 addr = GetCompilerPC() + 4 * offset; + return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); } void ArmJit::DoJit(u32 em_address, JitBlock *b) { @@ -349,7 +352,7 @@ void ArmJit::DoJit(u32 em_address, JitBlock *b) { // Jit breakpoints are quite fast, so let's do them in release too. CheckJitBreakpoint(GetCompilerPC(), 0); - MIPSOpcode inst = Memory::Read_Opcode_JIT(GetCompilerPC()); + MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); //MIPSInfo info = MIPSGetInfo(inst); //if (info & IS_VFPU) { // logBlocks = 1; @@ -391,7 +394,7 @@ void ArmJit::DoJit(u32 em_address, JitBlock *b) { if (logBlocks > 0 && dontLogBlocks == 0) { INFO_LOG(Log::JIT, "=============== mips ==============="); for (u32 cpc = em_address; cpc != GetCompilerPC() + 4; cpc += 4) { - MIPSDisAsm(Memory::Read_Opcode_JIT(cpc), cpc, temp, sizeof(temp), true); + MIPSDisAsm(Memory::ReadUnchecked_Instruction(cpc), cpc, temp, sizeof(temp), true); INFO_LOG(Log::JIT, "M: %08x %s", cpc, temp); } } @@ -419,12 +422,6 @@ bool ArmJit::DescribeCodePtr(const u8 *ptr, std::string &name) return false; } -void ArmJit::Comp_RunBlock(MIPSOpcode op) -{ - // This shouldn't be necessary, the dispatcher should catch us before we get here. - ERROR_LOG(Log::JIT, "Comp_RunBlock should never be reached!"); -} - void ArmJit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) { if (PlatformIsWXExclusive()) { ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE); @@ -476,6 +473,12 @@ void ArmJit::Comp_ReplacementFunc(MIPSOpcode op) // Inlined function calls (caught in jal) are handled differently. + // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. + if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { + Comp_Generic(op); + return; + } + int index = op.encoding & MIPS_EMUHACK_VALUE_MASK; const ReplacementTableEntry *entry = GetReplacementFunc(index); @@ -496,14 +499,14 @@ void ArmJit::Comp_ReplacementFunc(MIPSOpcode op) } if (disabled) { - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else if (entry->jitReplaceFunc) { MIPSReplaceFunc repl = entry->jitReplaceFunc; int cycles = (this->*repl)(); if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { // Compile the original instruction at this address. We ignore cycles for hooks. - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { FlushAll(); // Flushed, so R1 is safe. @@ -532,7 +535,7 @@ void ArmJit::Comp_ReplacementFunc(MIPSOpcode op) // Compile the original instruction at this address. We ignore cycles for hooks. ApplyRoundingMode(); RestoreDowncount(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { ApplyRoundingMode(); RestoreDowncount(); @@ -772,16 +775,6 @@ bool ArmJit::CheckMemoryBreakpoint(int instructionOffset) { void ArmJit::Comp_DoNothing(MIPSOpcode op) { } -MIPSOpcode ArmJit::GetOriginalOp(MIPSOpcode op) { - JitBlockCache *bc = GetBlockCache(); - int block_num = bc->GetBlockNumberFromEmuHackOp(op, true); - if (block_num >= 0) { - return bc->GetOriginalFirstOp(block_num); - } else { - return op; - } -} - } // namespace #endif // PPSSPP_ARCH(ARM) diff --git a/Core/MIPS/ARM/ArmJit.h b/Core/MIPS/ARM/ArmJit.h index f01b22922a..02a357dd05 100644 --- a/Core/MIPS/ARM/ArmJit.h +++ b/Core/MIPS/ARM/ArmJit.h @@ -54,9 +54,7 @@ public: const u8 *GetCrashHandler() const override { return crashHandler; } bool CodeInRange(const u8 *ptr) const override { return IsInSpace(ptr); } bool DescribeCodePtr(const u8 *ptr, std::string &name) override; - MIPSOpcode GetOriginalOp(MIPSOpcode op) override; - void Comp_RunBlock(MIPSOpcode op) override; void Comp_ReplacementFunc(MIPSOpcode op) override; // Ops @@ -133,9 +131,6 @@ public: JitBlockCache *GetBlockCache() override { return &blocks; } JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - std::vector SaveAndClearEmuHackOps() override { return blocks.SaveAndClearEmuHackOps(); } - void RestoreSavedEmuHackOps(std::vector saved) override { blocks.RestoreSavedEmuHackOps(saved); } - void ClearCache() override; void InvalidateCacheAt(u32 em_address, int length = 4) override; void UpdateFCR31() override; @@ -145,6 +140,9 @@ public: const u8 *GetDispatcher() const override { return dispatcher; } + u32 GetBlockShadowHookValue() const override { + return (u32)(blockShadowHook - GetBasePtr()); + } bool IsAtDispatchFetch(const u8 *ptr) const override { return ptr == dispatcherFetch; } @@ -282,6 +280,7 @@ public: const u8 *applyRoundingMode; const u8 *crashHandler; + const u8 *blockShadowHook; }; } // namespace MIPSComp diff --git a/Core/MIPS/ARM64/Arm64Asm.cpp b/Core/MIPS/ARM64/Arm64Asm.cpp index 1fa073b6ce..12bc8948d1 100644 --- a/Core/MIPS/ARM64/Arm64Asm.cpp +++ b/Core/MIPS/ARM64/Arm64Asm.cpp @@ -247,21 +247,21 @@ void Arm64Jit::GenerateFixedCode(const JitOptions &jo) { } LDR(INDEX_UNSIGNED, SCRATCH1, CTXREG, offsetof(MIPSState, pc)); -#ifdef MASKED_PSP_MEMORY - ANDI2R(SCRATCH1, SCRATCH1, 0x3FFFFFFF); -#endif + ANDI2R(SCRATCH1, SCRATCH1, Memory::BLOCK_SHADOW_ENTRY_MASK); + MOVP2R(SCRATCH2_64, Memory::blockShadow); dispatcherFetch = GetCodePtr(); - LDR(SCRATCH1, MEMBASEREG, SCRATCH1_64); - LSR(SCRATCH2, SCRATCH1, 24); // or UBFX(SCRATCH2, SCRATCH1, 24, 8) - ANDI2R(SCRATCH1, SCRATCH1, 0x00FFFFFF); - CMP(SCRATCH2, MIPS_EMUHACK_OPCODE >> 24); - FixupBranch skipJump = B(CC_NEQ); + LDR(SCRATCH1, SCRATCH2_64, SCRATCH1_64); + // The entry is the block's offset into the code space, or 0 for none. + FixupBranch skipJump = CBZ(SCRATCH1); ADD(SCRATCH1_64, JITBASEREG, SCRATCH1_64); BR(SCRATCH1_64); SetJumpTarget(skipJump); // No block found, let's jit. I don't think we actually need to save static regs that are in callee-save regs here but whatever. // Also, rounding mode gotta be irrelevant here.. + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + blockShadowHook = GetCodePtr(); SaveStaticRegisters(); RestoreRoundingMode(true); QuickCallFunctionR(SCRATCH1_64, (void *)&MIPSComp::JitAt, CTXREG); diff --git a/Core/MIPS/ARM64/Arm64IRAsm.cpp b/Core/MIPS/ARM64/Arm64IRAsm.cpp index 893488cd07..07296dbe11 100644 --- a/Core/MIPS/ARM64/Arm64IRAsm.cpp +++ b/Core/MIPS/ARM64/Arm64IRAsm.cpp @@ -156,8 +156,7 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) { // Fixed registers, these are always kept when in Jit context. MOVP2R(MEMBASEREG, Memory::base); MOVP2R(CTXREG, mipsState); - // Pre-subtract this to save time later. - MOVI2R(JITBASEREG, (intptr_t)GetBasePtr() - MIPS_EMUHACK_OPCODE); + MOVP2R(JITBASEREG, GetBasePtr()); LoadStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); @@ -204,20 +203,20 @@ void Arm64JitBackend::GenerateFixedCode(MIPSState *mipsState) { MovFromPC(SCRATCH1); WriteDebugPC(SCRATCH1); -#ifdef MASKED_PSP_MEMORY - ANDI2R(SCRATCH1, SCRATCH1, Memory::MEMVIEW32_MASK); -#endif + ANDI2R(SCRATCH1, SCRATCH1, Memory::BLOCK_SHADOW_ENTRY_MASK); + MOVP2R(SCRATCH2_64, Memory::blockShadow); hooks_.dispatchFetch = GetCodePtr(); - LDR(SCRATCH1, MEMBASEREG, SCRATCH1_64); - LSR(SCRATCH2, SCRATCH1, 24); // or UBFX(SCRATCH2, SCRATCH1, 24, 8) - // We don't mask SCRATCH1 as that's already baked into JITBASEREG. - CMP(SCRATCH2, MIPS_EMUHACK_OPCODE >> 24); - FixupBranch skipJump = B(CC_NEQ); + LDR(SCRATCH1, SCRATCH2_64, SCRATCH1_64); + // The entry is the block's offset into the code space, or 0 for none. + FixupBranch skipJump = CBZ(SCRATCH1); ADD(SCRATCH1_64, JITBASEREG, SCRATCH1_64); BR(SCRATCH1_64); SetJumpTarget(skipJump); // No block found, let's jit. We don't need to save static regs, they're all callee saved. + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + hooks_.blockShadowHook = GetCodePtr(); RestoreRoundingMode(true); WriteDebugProfilerStatus(IRProfilerStatus::COMPILING); QuickCallFunctionR(SCRATCH1_64, &MIPSComp::JitAt, CTXREG); diff --git a/Core/MIPS/ARM64/Arm64IRJit.cpp b/Core/MIPS/ARM64/Arm64IRJit.cpp index 541badc205..56b0adbc93 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.cpp +++ b/Core/MIPS/ARM64/Arm64IRJit.cpp @@ -384,7 +384,7 @@ void Arm64JitBackend::MovToPC(ARM64Reg r) { void Arm64JitBackend::WriteDebugPC(uint32_t pc) { if (hooks_.profilerPC) { int offset = (int)((const u8 *)hooks_.profilerPC - GetBasePtr()); - MOVI2R(SCRATCH2, MIPS_EMUHACK_OPCODE + offset); + MOVI2R(SCRATCH2, offset); MOVI2R(SCRATCH1, pc); STR(SCRATCH1, JITBASEREG, SCRATCH2); } @@ -393,7 +393,7 @@ void Arm64JitBackend::WriteDebugPC(uint32_t pc) { void Arm64JitBackend::WriteDebugPC(ARM64Reg r) { if (hooks_.profilerPC) { int offset = (int)((const u8 *)hooks_.profilerPC - GetBasePtr()); - MOVI2R(SCRATCH2, MIPS_EMUHACK_OPCODE + offset); + MOVI2R(SCRATCH2, offset); STR(r, JITBASEREG, SCRATCH2); } } @@ -401,7 +401,7 @@ void Arm64JitBackend::WriteDebugPC(ARM64Reg r) { void Arm64JitBackend::WriteDebugProfilerStatus(IRProfilerStatus status) { if (hooks_.profilerPC) { int offset = (int)((const u8 *)hooks_.profilerStatus - GetBasePtr()); - MOVI2R(SCRATCH2, MIPS_EMUHACK_OPCODE + offset); + MOVI2R(SCRATCH2, offset); MOVI2R(SCRATCH1, (int)status); STR(SCRATCH1, JITBASEREG, SCRATCH2); } diff --git a/Core/MIPS/ARM64/Arm64Jit.cpp b/Core/MIPS/ARM64/Arm64Jit.cpp index 6cc3764e25..d53b8567be 100644 --- a/Core/MIPS/ARM64/Arm64Jit.cpp +++ b/Core/MIPS/ARM64/Arm64Jit.cpp @@ -81,7 +81,7 @@ static u32 JitMemCheck(u32 pc) { return 0; // Note: pc may be the delay slot. - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); s32 offset = SignExtend16ToS32(op & 0xFFFF); if (MIPSGetInfo(op) & IS_VFPU) offset &= 0xFFFC; @@ -199,7 +199,7 @@ void Arm64Jit::ClearCache() { } void Arm64Jit::InvalidateCacheAt(u32 em_address, int length) { - if (blocks.RangeMayHaveEmuHacks(em_address, em_address + length)) { + if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { blocks.InvalidateICache(em_address, length); } } @@ -309,7 +309,8 @@ u32 Arm64Jit::GetCompilerPC() { } MIPSOpcode Arm64Jit::GetOffsetInstruction(int offset) { - return Memory::Read_Instruction(GetCompilerPC() + 4 * offset); + const u32 addr = GetCompilerPC() + 4 * offset; + return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); } void Arm64Jit::DoJit(u32 em_address, JitBlock *b) { @@ -354,7 +355,7 @@ void Arm64Jit::DoJit(u32 em_address, JitBlock *b) { // Jit breakpoints are quite fast, so let's do them in release too. CheckJitBreakpoint(GetCompilerPC(), 0); - MIPSOpcode inst = Memory::Read_Opcode_JIT(GetCompilerPC()); + MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); js.downcountAmount += MIPSGetInstructionCycleEstimate(inst); MIPSCompileOp(inst, this); @@ -388,7 +389,7 @@ void Arm64Jit::DoJit(u32 em_address, JitBlock *b) { if (logBlocks > 0 && dontLogBlocks == 0) { INFO_LOG(Log::JIT, "=============== mips %d ===============", blocks.GetNumBlocks()); for (u32 cpc = b->originalAddress; cpc != GetCompilerPC() + 4; cpc += 4) { - MIPSDisAsm(Memory::Read_Opcode_JIT(cpc), cpc, temp, sizeof(temp), true); + MIPSDisAsm(Memory::ReadUnchecked_Instruction(cpc), cpc, temp, sizeof(temp), true); INFO_LOG(Log::JIT, "M: %08x %s", cpc, temp); } } @@ -450,11 +451,6 @@ bool Arm64Jit::DescribeCodePtr(const u8 *ptr, std::string &name) { return true; } -void Arm64Jit::Comp_RunBlock(MIPSOpcode op) { - // This shouldn't be necessary, the dispatcher should catch us before we get here. - ERROR_LOG(Log::JIT, "Comp_RunBlock should never be reached!"); -} - void Arm64Jit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) { if (PlatformIsWXExclusive()) { ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE); @@ -494,6 +490,12 @@ void Arm64Jit::Comp_ReplacementFunc(MIPSOpcode op) // Inlined function calls (caught in jal) are handled differently. + // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. + if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { + Comp_Generic(op); + return; + } + int index = op.encoding & MIPS_EMUHACK_VALUE_MASK; const ReplacementTableEntry *entry = GetReplacementFunc(index); @@ -515,14 +517,14 @@ void Arm64Jit::Comp_ReplacementFunc(MIPSOpcode op) } if (disabled) { - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else if (entry->jitReplaceFunc) { MIPSReplaceFunc repl = entry->jitReplaceFunc; int cycles = (this->*repl)(); if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { // Compile the original instruction at this address. We ignore cycles for hooks. - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { FlushAll(); // Flushed, so R1 is safe. @@ -546,7 +548,7 @@ void Arm64Jit::Comp_ReplacementFunc(MIPSOpcode op) // Compile the original instruction at this address. We ignore cycles for hooks. ApplyRoundingMode(); LoadStaticRegisters(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { ApplyRoundingMode(); LoadStaticRegisters(); @@ -767,16 +769,6 @@ bool Arm64Jit::CheckMemoryBreakpoint(int instructionOffset) { void Arm64Jit::Comp_DoNothing(MIPSOpcode op) { } -MIPSOpcode Arm64Jit::GetOriginalOp(MIPSOpcode op) { - JitBlockCache *bc = GetBlockCache(); - int block_num = bc->GetBlockNumberFromEmuHackOp(op, true); - if (block_num >= 0) { - return bc->GetOriginalFirstOp(block_num); - } else { - return op; - } -} - } // namespace #endif // PPSSPP_ARCH(ARM64) diff --git a/Core/MIPS/ARM64/Arm64Jit.h b/Core/MIPS/ARM64/Arm64Jit.h index a9a8d50c11..ba7e7a19cd 100644 --- a/Core/MIPS/ARM64/Arm64Jit.h +++ b/Core/MIPS/ARM64/Arm64Jit.h @@ -55,9 +55,7 @@ public: const u8 *GetCrashHandler() const override { return crashHandler; } bool CodeInRange(const u8 *ptr) const override { return IsInSpace(ptr); } bool DescribeCodePtr(const u8 *ptr, std::string &name) override; - MIPSOpcode GetOriginalOp(MIPSOpcode op) override; - void Comp_RunBlock(MIPSOpcode op) override; void Comp_ReplacementFunc(MIPSOpcode op) override; // Ops @@ -174,9 +172,6 @@ public: JitBlockCache *GetBlockCache() override { return &blocks; } JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - std::vector SaveAndClearEmuHackOps() override { return blocks.SaveAndClearEmuHackOps(); } - void RestoreSavedEmuHackOps(std::vector saved) override { blocks.RestoreSavedEmuHackOps(saved); } - void ClearCache() override; void InvalidateCacheAt(u32 em_address, int length = 4) override; void UpdateFCR31() override; @@ -186,6 +181,9 @@ public: const u8 *GetDispatcher() const override { return dispatcher; } + u32 GetBlockShadowHookValue() const override { + return (u32)(blockShadowHook - GetBasePtr()); + } bool IsAtDispatchFetch(const u8 *ptr) const override { return ptr == dispatcherFetch; } @@ -292,6 +290,7 @@ public: const u8 *updateRoundingMode; const u8 *crashHandler; + const u8 *blockShadowHook; int jitStartOffset; diff --git a/Core/MIPS/IR/IRFrontend.cpp b/Core/MIPS/IR/IRFrontend.cpp index d98b49dbc5..8443fc4afb 100644 --- a/Core/MIPS/IR/IRFrontend.cpp +++ b/Core/MIPS/IR/IRFrontend.cpp @@ -142,6 +142,15 @@ bool IRFrontend::CheckRounding(u32 blockAddress) { } void IRFrontend::Comp_ReplacementFunc(MIPSOpcode op) { + // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. + if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { + // The interpreter checks memory at PC, which Comp_Generic doesn't set here. + FlushAll(); + ir.Write(IROp::SetPCConst, 0, 0, 0, GetCompilerPC()); + Comp_Generic(op); + return; + } + int index = op.encoding & MIPS_EMUHACK_VALUE_MASK; const ReplacementTableEntry *entry = GetReplacementFunc(index); @@ -162,7 +171,7 @@ void IRFrontend::Comp_ReplacementFunc(MIPSOpcode op) { } if (disabled) { - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else if (entry->replaceFunc) { FlushAll(); RestoreRoundingMode(); @@ -172,7 +181,7 @@ void IRFrontend::Comp_ReplacementFunc(MIPSOpcode op) { if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { // Compile the original instruction at this address. We ignore cycles for hooks. ApplyRoundingMode(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { ApplyRoundingMode(); // If IRTEMP_0 was set to 1, it means the replacement needs to run again (sliced.) @@ -238,7 +247,8 @@ u32 IRFrontend::GetCompilerPC() { } MIPSOpcode IRFrontend::GetOffsetInstruction(int offset) { - return Memory::Read_Instruction(GetCompilerPC() + 4 * offset); + const u32 addr = GetCompilerPC() + 4 * offset; + return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); } void IRFrontend::DoJit(u32 em_address, std::vector &instructions, u32 &mipsBytes) { @@ -264,7 +274,7 @@ void IRFrontend::DoJit(u32 em_address, std::vector &instructions, u32 &m // Jit breakpoints are quite fast, so let's do them in release too. CheckBreakpoint(GetCompilerPC()); - MIPSOpcode inst = Memory::Read_Opcode_JIT(GetCompilerPC()); + MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); js.downcountAmount += MIPSGetInstructionCycleEstimate(inst); MIPSCompileOp(inst, this); js.compilerPC += 4; @@ -323,7 +333,7 @@ void IRFrontend::DoJit(u32 em_address, std::vector &instructions, u32 &m NOTICE_LOG(Log::JIT, "=============== mips %08x ===============", em_address); for (u32 cpc = em_address; cpc != GetCompilerPC(); cpc += 4) { temp2[0] = 0; - MIPSDisAsm(Memory::Read_Opcode_JIT(cpc), cpc, temp2, sizeof(temp2), true); + MIPSDisAsm(Memory::ReadUnchecked_Instruction(cpc), cpc, temp2, sizeof(temp2), true); NOTICE_LOG(Log::JIT, "M: %08x %s", cpc, temp2); } } @@ -354,11 +364,6 @@ void IRFrontend::DoJit(u32 em_address, std::vector &instructions, u32 &m dontLogBlocks--; } -void IRFrontend::Comp_RunBlock(MIPSOpcode op) { - // This shouldn't be necessary, the dispatcher should catch us before we get here. - ERROR_LOG(Log::JIT, "Comp_RunBlock should never be reached!"); -} - void IRFrontend::CheckBreakpoint(u32 addr) { if (g_breakpoints.NeedsBreakCheckAt(addr)) { FlushAll(); @@ -370,8 +375,8 @@ void IRFrontend::CheckBreakpoint(u32 addr) { // At this point, downcount HAS the delay slot, but not the instruction itself. int downcountOffset = 0; if (js.inDelaySlot) { - MIPSOpcode branchOp = Memory::Read_Opcode_JIT(GetCompilerPC()); - MIPSOpcode delayOp = Memory::Read_Opcode_JIT(addr); + MIPSOpcode branchOp = Memory::ReadUnchecked_Instruction(GetCompilerPC()); + MIPSOpcode delayOp = Memory::ReadUnchecked_Instruction(addr); downcountOffset = -MIPSGetInstructionCycleEstimate(delayOp); if ((MIPSGetInfo(branchOp) & LIKELY) != 0) { // Okay, we're in a likely branch. Also negate the branch cycles. @@ -403,8 +408,8 @@ void IRFrontend::CheckMemoryBreakpoint(int rs, int offset) { int downcountOffset = 0; if (js.inDelaySlot) { // We assume delay slot in compilerPC + 4. - MIPSOpcode branchOp = Memory::Read_Opcode_JIT(GetCompilerPC()); - MIPSOpcode delayOp = Memory::Read_Opcode_JIT(GetCompilerPC() + 4); + MIPSOpcode branchOp = Memory::ReadUnchecked_Instruction(GetCompilerPC()); + MIPSOpcode delayOp = Memory::ReadUnchecked_Instruction(GetCompilerPC() + 4); downcountOffset = -MIPSGetInstructionCycleEstimate(delayOp); if ((MIPSGetInfo(branchOp) & LIKELY) != 0) { // Okay, we're in a likely branch. Also negate the branch cycles. diff --git a/Core/MIPS/IR/IRFrontend.h b/Core/MIPS/IR/IRFrontend.h index 58262831b4..d2057d5055 100644 --- a/Core/MIPS/IR/IRFrontend.h +++ b/Core/MIPS/IR/IRFrontend.h @@ -13,7 +13,6 @@ public: IRFrontend(bool startDefaultPrefix); void Comp_Generic(MIPSOpcode op) override; - void Comp_RunBlock(MIPSOpcode op) override; void Comp_ReplacementFunc(MIPSOpcode op) override; // Ops diff --git a/Core/MIPS/IR/IRJit.cpp b/Core/MIPS/IR/IRJit.cpp index 8cb9f51222..295cd217e2 100644 --- a/Core/MIPS/IR/IRJit.cpp +++ b/Core/MIPS/IR/IRJit.cpp @@ -142,7 +142,7 @@ bool IRJit::CompileBlock(u32 em_address, std::vector &instructions, u32 _dbg_assert_(!instructions.empty()); int block_num = blocks_.AllocateBlock(em_address, mipsBytes, instructions); - if ((block_num & ~MIPS_EMUHACK_VALUE_MASK) != 0) { + if (block_num < 0) { WARN_LOG(Log::JIT, "Failed to allocate block for %08x (%d instructions)", em_address, (int)instructions.size()); // Out of block numbers. Caller will handle. return false; @@ -162,7 +162,7 @@ bool IRJit::CompileBlock(u32 em_address, std::vector &instructions, u32 mipsTracer.prepare_block(b, blocks_); } - // Updates stats, also patches the first MIPS instruction into an emuhack if 'preload == false' + // Updates stats, also enters the block in the block shadow. blocks_.FinalizeBlock(block_num); FinalizeNativeBlock(&blocks_, block_num); return true; @@ -187,10 +187,11 @@ void IRJit::RunLoopUntil(u64 globalticks) { compilerEnabled_ = false; #endif while (mips->downcount >= 0) { - u32 inst = Memory::ReadUnchecked_U32(mips->pc); - u32 opcode = inst & 0xFF000000; - if (opcode == MIPS_EMUHACK_OPCODE) { - u32 offset = inst & 0x00FFFFFF; // Alternatively, inst - opcode + // The block's offset into the IR arena, or 0 for none, or a replacement hook (handled by + // compiling a block). + const u32 offset = *Memory::GetBlockShadowEntry(mips->pc); + static_assert(Memory::BLOCK_SHADOW_HOOK_NO_JIT == 1, "IR arena reserves offsets 0 and 1"); + if (offset > Memory::BLOCK_SHADOW_HOOK_NO_JIT) { const IRInst *instPtr = blocks_.GetArenaPtr() + offset; // First op is always, except when using breakpoints, downcount, to save one dispatch inside IRInterpret. // This branch is very cpu-branch-predictor-friendly so this still beats the dispatch. @@ -262,8 +263,12 @@ void IRBlockCache::Clear() { IRBlockCache::IRBlockCache(bool compileToNative) : compileToNative_(compileToNative) {} int IRBlockCache::AllocateBlock(int emAddr, u32 origSize, const std::vector &insts) { - // We have 24 bits to represent offsets with. const u32 MAX_ARENA_SIZE = 0x1000000 - 1; + if (arena_.empty()) { + // Offset 0 would mean "no block" in the block shadow, and 1 a replacement hook. + arena_.push_back(IRInst{ IROp::Bad }); + arena_.push_back(IRInst{ IROp::Bad }); + } int offset = (int)arena_.size(); if (offset >= MAX_ARENA_SIZE) { WARN_LOG(Log::JIT, "Filled JIT arena, restarting"); @@ -423,39 +428,6 @@ int IRBlockCache::FindByCookie(int cookie) { return -1; } -std::vector IRBlockCache::SaveAndClearEmuHackOps() { - std::vector result; - result.resize(blocks_.size()); - - for (int number = 0; number < (int)blocks_.size(); ++number) { - IRBlock &b = blocks_[number]; - int cookie = compileToNative_ ? b.GetNativeOffset() : b.GetIRArenaOffset(); - if (b.IsValid() && b.RestoreOriginalFirstOp(cookie)) { - result[number] = number; - } else { - result[number] = 0; - } - } - - return result; -} - -void IRBlockCache::RestoreSavedEmuHackOps(const std::vector &saved) { - if ((int)blocks_.size() != (int)saved.size()) { - ERROR_LOG(Log::JIT, "RestoreSavedEmuHackOps: Wrong saved block size."); - return; - } - - for (int number = 0; number < (int)blocks_.size(); ++number) { - IRBlock &b = blocks_[number]; - // Only if we restored it, write it back. - if (b.IsValid() && saved[number] != 0 && b.HasOriginalFirstOp()) { - int cookie = compileToNative_ ? b.GetNativeOffset() : b.GetIRArenaOffset(); - b.Finalize(cookie); - } - } -} - JitBlockDebugInfo IRBlockCache::GetBlockDebugInfo(int blockNum) const { const IRBlock &ir = blocks_[blockNum]; JitBlockDebugInfo debugInfo{}; @@ -533,26 +505,13 @@ int IRBlockCache::GetBlockNumberFromStartAddress(u32 em_address) const { return best; } -bool IRBlock::HasOriginalFirstOp() const { - return Memory::ReadUnchecked_U32(origAddr_) == origFirstOpcode_.encoding; -} - -bool IRBlock::RestoreOriginalFirstOp(int cookie) { - const u32 emuhack = MIPS_EMUHACK_OPCODE | cookie; - if (Memory::ReadUnchecked_U32(origAddr_) == emuhack) { - Memory::Write_Opcode_JIT(origAddr_, origFirstOpcode_); - return true; - } - return false; -} - void IRBlock::Finalize(int cookie) { // Check it wasn't invalidated, in case this is after preload. // TODO: Allow reusing blocks when the code matches hash_ again, instead. if (origAddr_ && Memory::IsValid4AlignedAddress(origAddr_)) { - origFirstOpcode_ = Memory::Read_Opcode_JIT(origAddr_); - MIPSOpcode opcode = MIPSOpcode(MIPS_EMUHACK_OPCODE | cookie); - Memory::Write_Opcode_JIT(origAddr_, opcode); + origFirstOpcode_ = Memory::ReadUnchecked_Instruction(origAddr_); + _dbg_assert_(cookie != 0); + Memory::WriteBlockShadow(origAddr_, cookie); } else { WARN_LOG(Log::JIT, "Finalizing invalid block (cookie: %d)", cookie); } @@ -560,14 +519,8 @@ void IRBlock::Finalize(int cookie) { void IRBlock::Destroy(int cookie) { if (origAddr_ && Memory::IsValid4AlignedAddress(origAddr_)) { - MIPSOpcode opcode = MIPSOpcode(MIPS_EMUHACK_OPCODE | cookie); - u32 memOp = Memory::ReadUnchecked_U32(origAddr_); - if (memOp == opcode.encoding) { - Memory::Write_Opcode_JIT(origAddr_, origFirstOpcode_); - } else { - // NOTE: This is not an error. Just interesting to log. - DEBUG_LOG(Log::JIT, "IRBlock::Destroy: Note: Block at %08x was overwritten - checked for %08x, got %08x when restoring the MIPS op to %08x", origAddr_, opcode.encoding, memOp, origFirstOpcode_.encoding); - } + // A block at a mirror may have taken over the entry, leave that alone. + Memory::ClearBlockShadow(origAddr_, cookie); // TODO: Also wipe the block in the IR opcode arena. // Let's mark this invalid so we don't try to clear it again. origAddr_ = 0; @@ -575,18 +528,8 @@ void IRBlock::Destroy(int cookie) { } u64 IRBlock::CalculateHash() const { - if (origAddr_) { - // This is unfortunate. In case there are emuhacks, we have to make a copy. - // If we could hash while reading we could avoid this. - std::vector buffer; - buffer.resize(origSize_ / 4); - size_t pos = 0; - for (u32 off = 0; off < origSize_; off += 4) { - // Let's actually hash the replacement, if any. - MIPSOpcode instr = Memory::ReadUnchecked_Instruction(origAddr_ + off, false); - buffer[pos++] = instr.encoding; - } - return XXH3_64bits(&buffer[0], origSize_); + if (origAddr_ && Memory::IsValidRange(origAddr_, origSize_)) { + return XXH3_64bits(Memory::GetPointerUnchecked(origAddr_), origSize_); } return 0; } @@ -597,12 +540,4 @@ bool IRBlock::OverlapsRange(u32 addr, u32 size) const { return addr + size > origAddr && addr < origAddr + origSize_; } -MIPSOpcode IRJit::GetOriginalOp(MIPSOpcode op) { - IRBlock *b = blocks_.GetBlock(blocks_.FindByCookie(op.encoding & 0xFFFFFF)); - if (b) { - return b->GetOriginalFirstOp(); - } - return op; -} - } // namespace MIPSComp diff --git a/Core/MIPS/IR/IRJit.h b/Core/MIPS/IR/IRJit.h index 5c89819d69..6ccdba5bd5 100644 --- a/Core/MIPS/IR/IRJit.h +++ b/Core/MIPS/IR/IRJit.h @@ -68,8 +68,6 @@ public: u32 GetIRArenaOffset() const { return arenaOffset_; } int GetNumIRInstructions() const { return numIRInstructions_; } MIPSOpcode GetOriginalFirstOp() const { return origFirstOpcode_; } - bool HasOriginalFirstOp() const; - bool RestoreOriginalFirstOp(int number); bool IsValid() const { return origAddr_ != 0 && origFirstOpcode_.encoding != 0x68FFFFFF; } void SetNativeOffset(int offset) { nativeOffset_ = offset; @@ -163,12 +161,10 @@ public: int FindPreloadBlock(u32 em_address); - // "Cookie" means the 24 bits we inject into the first instruction of each block. + // "Cookie" means the value the block shadow holds for a block: its offset into the + // IR arena, or into the native code space when compiling to native. int FindByCookie(int cookie); - std::vector SaveAndClearEmuHackOps(); - void RestoreSavedEmuHackOps(const std::vector &saved); - JitBlockDebugInfo GetBlockDebugInfo(int blockNum) const override; JitBlockMeta GetBlockMeta(int blockNum) const override { JitBlockMeta meta{}; @@ -221,10 +217,6 @@ public: // Not using a regular block cache. JitBlockCache *GetBlockCache() override { return nullptr; } JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks_; } - MIPSOpcode GetOriginalOp(MIPSOpcode op) override; - - std::vector SaveAndClearEmuHackOps() override { return blocks_.SaveAndClearEmuHackOps(); } - void RestoreSavedEmuHackOps(std::vector saved) override { blocks_.RestoreSavedEmuHackOps(saved); } void ClearCache() override; void InvalidateCacheAt(u32 em_address, int length = 4) override; diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index a2b52208dc..e845e6e0f8 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -609,6 +609,10 @@ const u8 *IRNativeJit::GetCodeBase() const { return backend_->CodeBlock().GetBasePtr(); } +u32 IRNativeJit::GetBlockShadowHookValue() const { + return (u32)backend_->OffsetFromCodePtr(backend_->GetNativeHooks().blockShadowHook); +} + bool IRNativeJit::IsAtDispatchFetch(const u8 *ptr) const { return ptr == backend_->GetNativeHooks().dispatchFetch; } diff --git a/Core/MIPS/IR/IRNativeCommon.h b/Core/MIPS/IR/IRNativeCommon.h index 6c90e89fbe..89d83ddb50 100644 --- a/Core/MIPS/IR/IRNativeCommon.h +++ b/Core/MIPS/IR/IRNativeCommon.h @@ -44,6 +44,8 @@ struct IRNativeHooks { const uint8_t *dispatcher = nullptr; const uint8_t *dispatchFetch = nullptr; const uint8_t *crashHandler = nullptr; + // The dispatcher's compile path, where a replacement hook's block shadow entry goes. + const uint8_t *blockShadowHook = nullptr; uint32_t *profilerPC = nullptr; IRProfilerStatus *profilerStatus = nullptr; @@ -189,6 +191,7 @@ public: bool DescribeCodePtr(const u8 *ptr, std::string &name) override; bool CodeInRange(const u8 *ptr) const override; bool IsAtDispatchFetch(const u8 *ptr) const override; + u32 GetBlockShadowHookValue() const override; const u8 *GetDispatcher() const override; const u8 *GetCrashHandler() const override; diff --git a/Core/MIPS/Interpreter.cpp b/Core/MIPS/Interpreter.cpp index 723b566a64..464874cede 100644 --- a/Core/MIPS/Interpreter.cpp +++ b/Core/MIPS/Interpreter.cpp @@ -67,7 +67,7 @@ static inline void DelayBranchTo(MIPSState *mips, u32 where) { } static inline void SkipLikely(MIPSState *mips) { - MIPSInfo delaySlot = MIPSGetInfo(Memory::Read_Instruction(PC + 4, true)); + MIPSInfo delaySlot = MIPSGetInfo(Memory::Read_Instruction(PC + 4)); // Don't actually skip if it is a jump (seen in Brooktown High.) if (delaySlot & IS_JUMP) { PC += 4; @@ -82,7 +82,7 @@ int MIPS_InterpretSingleStep(MIPSState *mips) { Core_ExecException(mips->pc, mips-> pc, ExecExceptionType::JUMP); return 0; } - MIPSOpcode op = Memory::Read_Opcode_JIT(mips->pc); // now unchecked + MIPSOpcode op = ReadExecutedOp(mips->pc); // Same reason as the run loop in MIPSInterpret_RunUntil - see ApplyHostRoundingMode. ApplyHostRoundingMode(mips); if (mips->inDelaySlot) { @@ -189,13 +189,15 @@ namespace MIPSInt { switch (func) { // Icache case 8: + { // Invalidate the instruction cache at this address. // We assume the CPU won't be reset during this, so no locking. + // Let's over invalidate to be super safe. + uint32_t alignedAddr = addr & ~0x3F; + int size = 0x40 + (addr & 0x3F); + // Through MIPSState, which also drops replacement hooks in functions the game changed. + mips->InvalidateICacheRangeImmediate(alignedAddr, size); if (MIPSComp::jit) { - // Let's over invalidate to be super safe. - uint32_t alignedAddr = addr & ~0x3F; - int size = 0x40 + (addr & 0x3F); - MIPSComp::jit->InvalidateCacheAt(alignedAddr, size); // Using a bool to avoid locking/etc. in case it's slow. if (!loggedAlignment && (addr & 0x3F) != 0) { // These are seen exclusively in Lego games, and are really no big deal. Reporting removed. @@ -208,6 +210,7 @@ namespace MIPSInt { } } break; + } // Dcache case 24: @@ -1287,12 +1290,15 @@ namespace MIPSInt { } void Int_Emuhack(MIPSState *mips, MIPSOpcode op) { - if (((op >> 24) & 3) != EMUOP_CALL_REPLACEMENT) { - _dbg_assert_msg_(false, "Trying to interpret emuhack instruction that can't be interpreted"); - } - _assert_((PC & 3) == 0); + // The CPU cores substitute these for replaced instructions, memory never holds them. + // If it does, the game is executing data. + if (Memory::Read_Instruction(PC).encoding == op.encoding) { + Core_ExecException(PC, PC, ExecExceptionType::ILLEGAL); + return; + } + // It's a replacement func! int index = op.encoding & 0xFFFFFF; const ReplacementTableEntry *entry = GetReplacementFunc(index); @@ -1304,7 +1310,7 @@ namespace MIPSInt { if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { // Interpret the original instruction under the hook. - MIPSInterpret(mips, Memory::Read_Instruction(PC, true)); + MIPSInterpret(mips, Memory::Read_Instruction(PC)); } else if (cycles < 0) { // Leave PC unchanged, call the replacement again (assumes args are modified.) mips->downcount += cycles; @@ -1317,7 +1323,7 @@ namespace MIPSInt { ERROR_LOG(Log::CPU, "Bad replacement function index %i", index); } // Interpret the original instruction under it. - MIPSInterpret(mips, Memory::Read_Instruction(PC, true)); + MIPSInterpret(mips, Memory::Read_Instruction(PC)); } } } diff --git a/Core/MIPS/InterpreterDispatch.cpp b/Core/MIPS/InterpreterDispatch.cpp index 9539e6d0ea..3fb657ef7b 100644 --- a/Core/MIPS/InterpreterDispatch.cpp +++ b/Core/MIPS/InterpreterDispatch.cpp @@ -343,12 +343,7 @@ int ExecInstruction(MIPSState *mips, MIPSOpcode op) { case 26: { switch ((op.encoding >> 24) & 0x3) { - // RUNBLOCK - case 0: - MIPSInt::Int_Emuhack(mips, op); - return 2; - // RetKrnl, CallRepl - case 1: + // CallRepl case 2: MIPSInt::Int_Emuhack(mips, op); return 1; diff --git a/Core/MIPS/JitCommon/JitBlockCache.cpp b/Core/MIPS/JitCommon/JitBlockCache.cpp index ac010e627b..c6a61e6a44 100644 --- a/Core/MIPS/JitCommon/JitBlockCache.cpp +++ b/Core/MIPS/JitCommon/JitBlockCache.cpp @@ -46,7 +46,6 @@ constexpr u32 SENTINEL_VAL = 0xc0ffeefe; static uint64_t HashJitBlock(const JitBlock &b) { PROFILE_THIS_SCOPE("jithash"); if (JIT_USE_COMPILEDHASH) { - // Includes the emuhack (or emuhacks) in memory. if (Memory::IsValidRange(b.originalAddress, b.originalSize * 4)) { return XXH3_64bits(Memory::GetPointerUnchecked(b.originalAddress), b.originalSize * 4); } else { @@ -172,11 +171,8 @@ static void ExpandRange(std::pair &range, u32 newStart, u32 newEnd) { void JitBlockCache::FinalizeBlock(int block_num, bool block_link) { JitBlock &b = blocks_[block_num]; - b.originalFirstOpcode = Memory::Read_Opcode_JIT(b.originalAddress); - MIPSOpcode opcode = GetEmuHackOpForBlock(block_num); - Memory::Write_Opcode_JIT(b.originalAddress, opcode); - - // Note that this hashes the emuhack too, which is intentional. + b.originalFirstOpcode = Memory::ReadUnchecked_Instruction(b.originalAddress); + Memory::WriteBlockShadow(b.originalAddress, GetShadowValueForBlock(block_num)); b.compiledHash = HashJitBlock(b); AddBlockMap(block_num); @@ -207,7 +203,7 @@ void JitBlockCache::FinalizeBlock(int block_num, bool block_link) { } } -bool JitBlockCache::RangeMayHaveEmuHacks(u32 start, u32 end) const { +bool JitBlockCache::RangeMayHaveBlocks(u32 start, u32 end) const { for (int i = 0; i < JITBLOCK_RANGE_COUNT; ++i) { if (end >= blockMemRanges_[i].first && start <= blockMemRanges_[i].second) { return true; @@ -230,16 +226,13 @@ static int binary_search(const JitBlock blocks_[], const u8 *baseoff, int imin, return -1; } -int JitBlockCache::GetBlockNumberFromEmuHackOp(MIPSOpcode inst, bool ignoreBad) const { - if (!num_blocks_ || !MIPS_IS_EMUHACK(inst)) // definitely not a JIT block +int JitBlockCache::GetBlockNumberFromShadowValue(u32 value) const { + if (!num_blocks_ || value == 0) return -1; - int off = (inst & MIPS_EMUHACK_VALUE_MASK); - const u8 *baseoff = codeBlock_->GetBasePtr() + off; - if (baseoff < codeBlock_->GetBasePtr() || baseoff >= codeBlock_->GetCodePtr()) { - if (!ignoreBad) { - ERROR_LOG(Log::JIT, "JitBlockCache: Invalid Emuhack Op %08x", inst.encoding); - } + const u8 *baseoff = codeBlock_->GetBasePtr() + value; + if (baseoff >= codeBlock_->GetCodePtr()) { + ERROR_LOG(Log::JIT, "JitBlockCache: Invalid block shadow value %08x", value); return -1; } @@ -251,17 +244,19 @@ int JitBlockCache::GetBlockNumberFromEmuHackOp(MIPSOpcode inst, bool ignoreBad) } } -MIPSOpcode JitBlockCache::GetEmuHackOpForBlock(int blockNum) const { - int off = (int)(blocks_[blockNum].normalEntry - codeBlock_->GetBasePtr()); - return MIPSOpcode(MIPS_EMUHACK_OPCODE | off); +u32 JitBlockCache::GetShadowValueForBlock(int blockNum) const { + // Never 0, the dispatcher is at the start of the code space. + const u32 off = (u32)(blocks_[blockNum].normalEntry - codeBlock_->GetBasePtr()); + _dbg_assert_(off != 0); + return off; } int JitBlockCache::GetBlockNumberFromStartAddress(u32 addr) const { if (!blocks_ || !Memory::IsValid4AlignedAddress(addr)) return -1; - MIPSOpcode inst = MIPSOpcode(Memory::ReadUnchecked_U32(addr)); - int bl = GetBlockNumberFromEmuHackOp(inst); + // A replacement hook's value is the dispatcher's compile path, which isn't any block's entry. + int bl = GetBlockNumberFromShadowValue(Memory::ReadBlockShadow(addr)); if (bl < 0) { return -1; } @@ -365,45 +360,6 @@ void JitBlockCache::UnlinkBlock(int i) { } } -std::vector JitBlockCache::SaveAndClearEmuHackOps() { - std::vector result; - result.resize(num_blocks_); - - for (int block_num = 0; block_num < num_blocks_; ++block_num) { - JitBlock &b = blocks_[block_num]; - if (b.invalid) - continue; - - const u32 emuhack = GetEmuHackOpForBlock(block_num).encoding; - if (Memory::ReadUnchecked_U32(b.originalAddress) == emuhack) - { - result[block_num] = emuhack; - Memory::Write_Opcode_JIT(b.originalAddress, b.originalFirstOpcode); - } - else - result[block_num] = 0; - } - - return result; -} - -void JitBlockCache::RestoreSavedEmuHackOps(const std::vector &saved) { - if (num_blocks_ != (int)saved.size()) { - ERROR_LOG(Log::JIT, "RestoreSavedEmuHackOps: Wrong saved block size."); - return; - } - - for (int block_num = 0; block_num < num_blocks_; ++block_num) { - const JitBlock &b = blocks_[block_num]; - if (b.invalid || saved[block_num] == 0) - continue; - - // Only if we restored it, write it back. - if (Memory::ReadUnchecked_U32(b.originalAddress) == b.originalFirstOpcode.encoding) - Memory::Write_Opcode_JIT(b.originalAddress, MIPSOpcode(saved[block_num])); - } -} - void JitBlockCache::DestroyBlock(int block_num, DestroyType type) { if (block_num < 0 || block_num >= num_blocks_) { ERROR_LOG_REPORT(Log::JIT, "DestroyBlock: Invalid block number %d", block_num); @@ -424,13 +380,12 @@ void JitBlockCache::DestroyBlock(int block_num, DestroyType type) { b->invalid = true; - // Only restoring the original opcode needs a valid address. The unlinking below has to + // Only clearing the shadow entry needs a valid address. The unlinking below has to // happen either way - otherwise other blocks go on jumping straight into this one, which // we've just marked invalid. if (Memory::IsValid4AlignedAddress(b->originalAddress)) { - if (Memory::ReadUnchecked_U32(b->originalAddress) == GetEmuHackOpForBlock(block_num).encoding) { - Memory::Write_Opcode_JIT(b->originalAddress, b->originalFirstOpcode); - } + // A mirror may have replaced the entry with its own block, leave that alone. + Memory::ClearBlockShadow(b->originalAddress, GetShadowValueForBlock(block_num)); } else { _dbg_assert_msg_(false, "Destroying block with invalid original address: %08x (block num: %d)", b->originalAddress, block_num); } @@ -500,8 +455,7 @@ void JitBlockCache::InvalidateChangedBlocks() { if (JIT_USE_COMPILEDHASH) { changed = b.compiledHash != HashJitBlock(b); } else { - const u32 emuhack = GetEmuHackOpForBlock(block_num).encoding; - changed = Memory::ReadUnchecked_U32(b.originalAddress) != emuhack; + changed = Memory::ReadUnchecked_U32(b.originalAddress) != b.originalFirstOpcode.encoding; } if (changed) { diff --git a/Core/MIPS/JitCommon/JitBlockCache.h b/Core/MIPS/JitCommon/JitBlockCache.h index e76b7a9cbf..a39f37187b 100644 --- a/Core/MIPS/JitCommon/JitBlockCache.h +++ b/Core/MIPS/JitCommon/JitBlockCache.h @@ -140,24 +140,19 @@ public: void GetBlockNumbersFromAddress(u32 em_address, std::vector *block_numbers) const; // Similar to above, but only the first matching address. int GetBlockNumberFromAddress(u32 em_address) const; - int GetBlockNumberFromEmuHackOp(MIPSOpcode inst, bool ignoreBad = false) const; + int GetBlockNumberFromShadowValue(u32 value) const; u32 GetAddressFromBlockPtr(const u8 *ptr) const; MIPSOpcode GetOriginalFirstOp(int block_num) const; - bool RangeMayHaveEmuHacks(u32 start, u32 end) const; + bool RangeMayHaveBlocks(u32 start, u32 end) const; // DOES NOT WORK CORRECTLY WITH JIT INLINING void InvalidateICache(u32 address, const u32 length); void InvalidateChangedBlocks(); void DestroyBlock(int block_num, DestroyType type); - // No jit operations may be run between these calls. - // Meant to be used to make memory safe for savestates, memcpy, etc. - std::vector SaveAndClearEmuHackOps(); - void RestoreSavedEmuHackOps(const std::vector &saved); - int GetNumBlocks() const override { return num_blocks_; } bool IsValidBlock(int blockNum) const override { return blockNum >= 0 && blockNum < num_blocks_ && !blocks_[blockNum].invalid; } JitBlockMeta GetBlockMeta(int blockNum) const override { @@ -189,7 +184,8 @@ private: void AddBlockMap(int block_num); void RemoveBlockMap(int block_num); - MIPSOpcode GetEmuHackOpForBlock(int block_num) const; + // What the block shadow holds for a block: its normal entry's offset into the code space. + u32 GetShadowValueForBlock(int block_num) const; int num_blocks_ = 0; CodeBlockCommon *codeBlock_; diff --git a/Core/MIPS/JitCommon/JitCommon.h b/Core/MIPS/JitCommon/JitCommon.h index 13bed7b74e..3a9b69d8f6 100644 --- a/Core/MIPS/JitCommon/JitCommon.h +++ b/Core/MIPS/JitCommon/JitCommon.h @@ -21,6 +21,7 @@ #include #include "Common/CommonTypes.h" +#include "Core/MemMap.h" #include "Core/MIPS/MIPS.h" // TODO: Find a better place for these. @@ -47,7 +48,6 @@ namespace MIPSComp { virtual void EatPrefix() = 0; virtual void Comp_Generic(MIPSOpcode op) = 0; - virtual void Comp_RunBlock(MIPSOpcode op) = 0; virtual void Comp_ReplacementFunc(MIPSOpcode op) = 0; virtual void Comp_ITypeMem(MIPSOpcode op) = 0; virtual void Comp_StoreSync(MIPSOpcode op) = 0; @@ -135,12 +135,9 @@ namespace MIPSComp { virtual void Compile(u32 em_address) = 0; virtual void ClearCache() = 0; virtual void UpdateFCR31() = 0; - virtual MIPSOpcode GetOriginalOp(MIPSOpcode op) = 0; - - // No jit operations may be run between these calls. - // Meant to be used to make memory safe for savestates, memcpy, etc. - virtual std::vector SaveAndClearEmuHackOps() = 0; - virtual void RestoreSavedEmuHackOps(std::vector saved) = 0; + // What the block shadow holds for a replacement hook without a compiled block. The + // dispatcher has to send this to the compiler. + virtual u32 GetBlockShadowHookValue() const { return Memory::BLOCK_SHADOW_HOOK_NO_JIT; } // Block linking. This may need to work differently for whole-function JITs and stuff // like that. diff --git a/Core/MIPS/LoongArch64/LoongArch64Asm.cpp b/Core/MIPS/LoongArch64/LoongArch64Asm.cpp index 23d4e2b734..be09398a74 100644 --- a/Core/MIPS/LoongArch64/LoongArch64Asm.cpp +++ b/Core/MIPS/LoongArch64/LoongArch64Asm.cpp @@ -121,7 +121,7 @@ void LoongArch64JitBackend::GenerateFixedCode(MIPSState *mipsState) { // Fixed registers, these are always kept when in Jit context. LI(MEMBASEREG, Memory::base); LI(CTXREG, mipsState); - LI(JITBASEREG, GetBasePtr() - MIPS_EMUHACK_OPCODE); + LI(JITBASEREG, GetBasePtr()); LoadStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); @@ -167,23 +167,22 @@ void LoongArch64JitBackend::GenerateFixedCode(MIPSState *mipsState) { LD_WU(SCRATCH1, CTXREG, offsetof(MIPSState, pc)); WriteDebugPC(SCRATCH1); -#ifdef MASKED_PSP_MEMORY - LI(SCRATCH2, 0x3FFFFFFF); + LI(SCRATCH2, (u32)Memory::BLOCK_SHADOW_ENTRY_MASK); AND(SCRATCH1, SCRATCH1, SCRATCH2); -#endif - ADD_D(SCRATCH1, SCRATCH1, MEMBASEREG); + LI(SCRATCH2, Memory::blockShadow); + ADD_D(SCRATCH1, SCRATCH1, SCRATCH2); hooks_.dispatchFetch = GetCodePtr(); LD_WU(SCRATCH1, SCRATCH1, 0); - SRLI_D(SCRATCH2, SCRATCH1, 24); - // We're in other words comparing to the top 8 bits of MIPS_EMUHACK_OPCODE by subtracting. - ADDI_D(SCRATCH2, SCRATCH2, -(MIPS_EMUHACK_OPCODE >> 24)); - FixupBranch needsCompile = BNEZ(SCRATCH2); - // No need to mask, JITBASEREG has already accounted for the upper bits. + // The entry is the block's offset into the code space, or 0 for none. + FixupBranch needsCompile = BEQZ(SCRATCH1); ADD_D(SCRATCH1, JITBASEREG, SCRATCH1); JR(SCRATCH1); SetJumpTarget(needsCompile); // No block found, let's jit. We don't need to save static regs, they're all callee saved. + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + hooks_.blockShadowHook = GetCodePtr(); RestoreRoundingMode(true); WriteDebugProfilerStatus(IRProfilerStatus::COMPILING); QuickCallFunctionR(&MIPSComp::JitAt, CTXREG, R20); diff --git a/Core/MIPS/MIPS.cpp b/Core/MIPS/MIPS.cpp index 828d52170d..fde01de30f 100644 --- a/Core/MIPS/MIPS.cpp +++ b/Core/MIPS/MIPS.cpp @@ -43,6 +43,7 @@ #include "Core/System.h" #include "Core/MIPS/JitCommon/JitCommon.h" #include "Core/CoreTiming.h" +#include "Core/HLE/ReplaceTables.h" MIPSState mipsr4k; MIPSState *currentMIPS = &mipsr4k; @@ -252,12 +253,18 @@ MIPSState::MIPSState() { MIPSState::~MIPSState() { } +// The block shadow's replacement hook entries have to match the JIT's dispatcher. +static void UpdateBlockShadowHook() { + Replacement_SetBlockShadowHook(MIPSComp::jit ? MIPSComp::jit->GetBlockShadowHookValue() : Memory::BLOCK_SHADOW_HOOK_NO_JIT); +} + void MIPSState::Shutdown() { MIPSComp::JitInterface *oldjit = MIPSComp::jit; if (oldjit) { MIPSComp::jit = nullptr; delete oldjit; } + UpdateBlockShadowHook(); pendingInvalidates_.clear(); invalidateAll_ = false; } @@ -309,6 +316,7 @@ void MIPSState::Init() { } else { MIPSComp::jit = nullptr; } + UpdateBlockShadowHook(); } bool MIPSState::HasDefaultPrefix() const { @@ -354,6 +362,7 @@ void MIPSState::UpdateCore(CPUCore desired) { } MIPSComp::jit = newjit; + UpdateBlockShadowHook(); } void MIPSState::DoState(PointerWrap &p) { @@ -433,12 +442,14 @@ int MIPSState::RunLoopUntil(u64 globalTicks) { break; case CPUCore::INTERPRETER: + ProcessPendingInvalidates(); return MIPSInterpret_RunUntil(this, globalTicks); } return 1; } void MIPSState::InvalidateICacheRangeImmediate(u32 address, u32 length) { + Replacement_CheckRange(address, length); if (!MIPSComp::jit) { // Nothing to do. return; @@ -448,45 +459,40 @@ void MIPSState::InvalidateICacheRangeImmediate(u32 address, u32 length) { } } +// The replacement hook checks happen here rather than when queued, since callers may queue the +// invalidation before writing. Dropping a hook queues its address, hence the loop. void MIPSState::ProcessPendingInvalidates() { - if (!MIPSComp::jit) { - // Nothing to do. Clear any old state from CPU core switching. - pendingInvalidates_.clear(); - return; - } - - if (invalidateAll_) { - MIPSComp::jit->ClearCache(); - pendingInvalidates_.clear(); + while (invalidateAll_ || !pendingInvalidates_.empty()) { + const bool all = invalidateAll_; invalidateAll_ = false; - return; - } - - if (!pendingInvalidates_.empty()) { - for (const auto &p : pendingInvalidates_) { - MIPSComp::jit->InvalidateCacheAt(p.addr, p.size); + std::vector pending; + pending.swap(pendingInvalidates_); + for (const auto &p : pending) { + Replacement_CheckRange(p.addr, p.size); + if (MIPSComp::jit && !all) { + MIPSComp::jit->InvalidateCacheAt(p.addr, p.size); + } + } + if (MIPSComp::jit && all) { + MIPSComp::jit->ClearCache(); } - pendingInvalidates_.clear(); } } void MIPSState::InvalidateICacheRangeDeferred(u32 address, u32 length) { - if (!MIPSComp::jit) { - // Nothing to do. - return; - } - // TODO: We can try to merge the invalidate with the previous one. pendingInvalidates_.push_back(PendingCacheOperation{address, length}); - Core_ReenterDispatcher(); + if (MIPSComp::jit) { + Core_ReenterDispatcher(); + } } +// Queued ranges still get their replacement hook checks. void MIPSState::ClearJitCacheDeferred() { if (!MIPSComp::jit) { return; } - pendingInvalidates_.clear(); invalidateAll_ = true; Core_ReenterDispatcher(); } diff --git a/Core/MIPS/MIPS.h b/Core/MIPS/MIPS.h index a2c60da099..9f1f8d1b75 100644 --- a/Core/MIPS/MIPS.h +++ b/Core/MIPS/MIPS.h @@ -30,21 +30,15 @@ class PointerWrap; typedef Memory::Opcode MIPSOpcode; -// Unlike on the PPC, opcode 0 is not unused and thus we have to choose another fake -// opcode to represent JIT blocks and other emu hacks. -// I've chosen 0x68000000. +// A fake opcode, 0x68000000, that the CPU cores run in place of an instruction with a replacement +// function (see ReplaceTables.h). It's never written to PSP memory. #define MIPS_EMUHACK_OPCODE 0x68000000 -#define MIPS_EMUHACK_MASK 0xFC000000 -#define MIPS_JITBLOCK_MASK 0xFF000000 #define MIPS_EMUHACK_VALUE_MASK 0x00FFFFFF -// There are 2 bits available for sub-opcodes, 0x03000000. -#define EMUOP_RUNBLOCK 0 // Runs a JIT block -#define EMUOP_RETKERNEL 1 // Returns to the simulated PSP kernel from a thread +// There are 2 bits available for sub-opcodes, 0x03000000. 0 and 1 are no longer used. #define EMUOP_CALL_REPLACEMENT 2 #define MIPS_IS_EMUHACK(op) (((op) & 0xFC000000) == MIPS_EMUHACK_OPCODE) // masks away the subop -#define MIPS_IS_RUNBLOCK(op) (((op) & 0xFF000000) == MIPS_EMUHACK_OPCODE) // masks away the subop #define MIPS_IS_REPLACEMENT(op) (((op) & 0xFF000000) == (MIPS_EMUHACK_OPCODE | (EMUOP_CALL_REPLACEMENT << 24))) // masks away the subop #define MIPS_EMUHACK_CALL_REPLACEMENT (MIPS_EMUHACK_OPCODE | (EMUOP_CALL_REPLACEMENT << 24)) diff --git a/Core/MIPS/MIPSAnalyst.cpp b/Core/MIPS/MIPSAnalyst.cpp index c3d55859f9..b35210eea1 100644 --- a/Core/MIPS/MIPSAnalyst.cpp +++ b/Core/MIPS/MIPSAnalyst.cpp @@ -618,18 +618,18 @@ namespace MIPSAnalyst { } int OpMemoryAccessSize(u32 pc) { - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); return MIPSGetMemoryAccessSize(op); } bool IsOpMemoryWrite(u32 pc) { - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); MIPSInfo info = MIPSGetInfo(op); return (info & OUT_MEM) != 0; } bool OpHasDelaySlot(u32 pc) { - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); MIPSInfo info = MIPSGetInfo(op); return (info & DELAYSLOT) != 0; } @@ -639,7 +639,7 @@ namespace MIPSAnalyst { return false; } - const auto op = Memory::Read_Instruction(pc, true); + const auto op = Memory::Read_Instruction(pc); // TODO: Trap sc/ll, svl.q, svr.q? @@ -708,7 +708,7 @@ namespace MIPSAnalyst { } for (u32 addr = address, endAddr = address + MAX_ANALYZE; addr <= endAddr; addr += 4) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); MIPSInfo info = MIPSGetInfo(op); MIPSGPReg rs = MIPS_GET_RS(op); @@ -784,7 +784,7 @@ namespace MIPSAnalyst { u32 end = addr + instrs * sizeof(u32); bool canClobber = true; while (addr < end) { - const MIPSOpcode op = Memory::Read_Instruction(addr, true); + const MIPSOpcode op = Memory::Read_Instruction(addr); const MIPSInfo info = MIPSGetInfo(op); // Yes, used. @@ -833,7 +833,7 @@ namespace MIPSAnalyst { u32 end = addr + instrs * sizeof(u32); bool canClobber = true; while (addr < end) { - const MIPSOpcode op = Memory::Read_Instruction(addr, true); + const MIPSOpcode op = Memory::Read_Instruction(addr); const MIPSInfo info = MIPSGetInfo(op); // Yes, used. @@ -890,17 +890,12 @@ namespace MIPSAnalyst { continue; } - // This is unfortunate. In case of emuhacks or relocs, we have to make a copy. + // Immediates are masked out, so relocations don't change the hash. buffer.resize((f.end - f.start + 4) / 4); size_t pos = 0; for (u32 addr = f.start; addr <= f.end; addr += 4) { u32 validbits = 0xFFFFFFFF; - MIPSOpcode instr = Memory::ReadUnchecked_Instruction(addr, true); - if (MIPS_IS_EMUHACK(instr)) { - f.hasHash = false; - goto skip; - } - + MIPSOpcode instr = Memory::ReadUnchecked_Instruction(addr); MIPSInfo flags = MIPSGetInfo(instr); if (flags & IN_IMM16) validbits &= ~0xFFFF; @@ -911,8 +906,6 @@ namespace MIPSAnalyst { f.hash = CityHash64((const char *) &buffer[0], buffer.size() * sizeof(u32)); f.hasHash = true; -skip: - ; } } @@ -964,7 +957,7 @@ skip: const u32 scanEnd = fromAddr + Memory::ClampValidSizeAt(fromAddr, MAX_AHEAD_SCAN); for (u32 ahead = fromAddr; ahead < scanEnd; ahead += 4) { - MIPSOpcode aheadOp = Memory::Read_Instruction(ahead, true); + MIPSOpcode aheadOp = Memory::Read_Instruction(ahead); u32 target = GetBranchTargetNoRA(ahead, aheadOp); if (target == INVALIDTARGET && ((aheadOp & 0xFC000000) == 0x08000000)) { target = GetJumpTarget(ahead); @@ -988,7 +981,7 @@ skip: if (closestJumpbackAddr != INVALIDTARGET && furthestJumpbackAddr == INVALIDTARGET) { for (u32 behind = closestJumpbackTarget; behind < fromAddr; behind += 4) { - MIPSOpcode behindOp = Memory::Read_Instruction(behind, true); + MIPSOpcode behindOp = Memory::Read_Instruction(behind); u32 target = GetBranchTargetNoRA(behind, behindOp); if (target == INVALIDTARGET && ((behindOp & 0xFC000000) == 0x08000000)) { target = GetJumpTarget(behind); @@ -1024,7 +1017,7 @@ skip: u32 addr; for (addr = startAddr; addr < endAddr; addr += 4) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); u32 target = GetBranchTargetNoRA(addr, op); if (target != INVALIDTARGET) { isStraightLeaf = false; @@ -1047,7 +1040,7 @@ skip: // If it's a nearby forward jump, and not a stackless leaf, assume not a tail call. if (sureTarget <= addr + MAX_JUMP_FORWARD && decreasedSp) { // But let's check the delay slot. - MIPSOpcode op = Memory::Read_Instruction(addr + 4, true); + MIPSOpcode op = Memory::Read_Instruction(addr + 4); // addiu sp, sp, +X if ((op & 0xFFFF8000) != 0x27BD0000) { furthestBranch = sureTarget; diff --git a/Core/MIPS/MIPSCodeUtils.cpp b/Core/MIPS/MIPSCodeUtils.cpp index 7bd2847a24..470ff2b661 100644 --- a/Core/MIPS/MIPSCodeUtils.cpp +++ b/Core/MIPS/MIPSCodeUtils.cpp @@ -32,7 +32,7 @@ namespace MIPSCodeUtils #define TARGET26 (_IMM26 << 2) u32 GetJumpTarget(u32 addr) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); if (op != 0) { MIPSInfo info = MIPSGetInfo(op); if ((info & IS_JUMP) && (info & IN_IMM26)) @@ -45,7 +45,7 @@ namespace MIPSCodeUtils } u32 GetBranchTarget(u32 addr) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); if (op != 0) { MIPSInfo info = MIPSGetInfo(op); if (info & IS_CONDBRANCH) @@ -58,7 +58,7 @@ namespace MIPSCodeUtils } u32 GetBranchTargetNoRA(u32 addr) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); return GetBranchTargetNoRA(addr, op); } @@ -76,7 +76,7 @@ namespace MIPSCodeUtils // As long as addr is aligned, this will only return aligned addresses. u32 GetSureBranchTarget(u32 addr) { - MIPSOpcode op = Memory::Read_Instruction(addr, true); + MIPSOpcode op = Memory::Read_Instruction(addr); if (op != 0) { MIPSInfo info = MIPSGetInfo(op); if ((info & IS_CONDBRANCH) && !(info & (IN_FPUFLAG | IS_VFPU))) { diff --git a/Core/MIPS/MIPSDebugInterface.cpp b/Core/MIPS/MIPSDebugInterface.cpp index b690a9781a..5dd2ab20a3 100644 --- a/Core/MIPS/MIPSDebugInterface.cpp +++ b/Core/MIPS/MIPSDebugInterface.cpp @@ -308,7 +308,7 @@ bool parseExpression(const DebugInterface *debug, PostfixExpression& exp, u32& d void DisAsm(u32 pc, char *out, size_t outSize) { if (Memory::IsValid4AlignedAddress(pc)) - MIPSDisAsm(Memory::Read_Opcode_JIT(pc), pc, out, outSize); + MIPSDisAsm(Memory::ReadUnchecked_Instruction(pc), pc, out, outSize); else truncate_cpy(out, outSize, "-"); } diff --git a/Core/MIPS/MIPSDis.cpp b/Core/MIPS/MIPSDis.cpp index 3133dfbc5f..d43dbc6e52 100644 --- a/Core/MIPS/MIPSDis.cpp +++ b/Core/MIPS/MIPSDis.cpp @@ -334,7 +334,7 @@ namespace MIPSDis } void Dis_Emuhack(MIPSOpcode op, uint32_t pc, char *out, size_t outSize) { - auto resolved = Memory::Read_Instruction(pc, true); + auto resolved = Memory::Read_Instruction(pc); char disasm[256]; if (MIPS_IS_EMUHACK(resolved)) { truncate_cpy(disasm, sizeof(disasm), "(invalid emuhack)"); @@ -342,10 +342,8 @@ namespace MIPSDis MIPSDisAsm(resolved, pc, disasm, sizeof(disasm), true); } + // Memory doesn't hold these, but the CPU cores substitute CallRepl for a replaced instruction. switch (op.encoding >> 24) { - case 0x68: - snprintf(out, outSize, "* jitblock: %s", disasm); - break; case 0x6a: snprintf(out, outSize, "* replacement: %s", disasm); break; diff --git a/Core/MIPS/MIPSStackWalk.cpp b/Core/MIPS/MIPSStackWalk.cpp index 3b312b441b..8e35f63351 100644 --- a/Core/MIPS/MIPSStackWalk.cpp +++ b/Core/MIPS/MIPSStackWalk.cpp @@ -68,7 +68,7 @@ namespace MIPSStackWalk { // It ought to be pretty close. u32 stop = pc - 32 * 4; for (; Memory::IsValidAddress(pc) && pc >= stop; pc -= 4) { - MIPSOpcode op = Memory::Read_Instruction(pc, true); + MIPSOpcode op = Memory::Read_Instruction(pc); // We're looking for a "mov fp, sp" close by a "addiu sp, sp, -N". if (IsMovRegsInstr(op) && _RD == MIPS_REG_FP && (_RS == MIPS_REG_SP || _RT == MIPS_REG_SP)) { @@ -114,7 +114,7 @@ namespace MIPSStackWalk { } for (u32 pc = start; Memory::IsValidAddress(pc) && pc >= stop; pc -= 4) { _dbg_assert_(Memory::IsValidAddress(pc)); - MIPSOpcode op = Memory::Read_Instruction(pc, true); + MIPSOpcode op = Memory::Read_Instruction(pc); // Here's where they store the ra address. if (IsSWInstr(op) && _RT == MIPS_REG_RA && _RS == MIPS_REG_SP) { diff --git a/Core/MIPS/MIPSTables.cpp b/Core/MIPS/MIPSTables.cpp index 6ace0f2712..f7cf744793 100644 --- a/Core/MIPS/MIPSTables.cpp +++ b/Core/MIPS/MIPSTables.cpp @@ -21,6 +21,7 @@ #include "Core/Core.h" #include "Core/MemMap.h" +#include "Core/HLE/ReplaceTables.h" #include "Core/MIPS/MIPS.h" #include "Core/MIPS/MIPSDis.h" #include "Core/MIPS/MIPSDisVFPU.h" @@ -804,8 +805,8 @@ static const MIPSInstruction tableALLEGREX0[32] = // 011111 ..... ..... ..... x }; static const MIPSInstruction tableEMU[4] = { - INSTR("RUNBLOCK", JITFUNC(Comp_RunBlock), Dis_Emuhack, Int_Emuhack, 0xFFFFFFFF), - INSTR("RetKrnl", 0, Dis_Emuhack, Int_Emuhack, 0), + INVALID, + INVALID, INSTR("CallRepl", JITFUNC(Comp_ReplacementFunc), Dis_Emuhack, Int_Emuhack, 0), INVALID, }; @@ -1099,7 +1100,7 @@ static void RunUntilDowncountZeroFast(MIPSState *mips) { Core_ExecException(pc, pc, ExecExceptionType::JUMP); return; } - const MIPSOpcode op = MIPSOpcode(Memory::ReadUnchecked_U32(pc)); + const MIPSOpcode op = ReadExecutedOp(pc); const bool wasInDelaySlot = mips->inDelaySlot; const int cycles = ExecInstruction(mips, op); @@ -1230,8 +1231,8 @@ static void RunUntilDowncountZeroWithChecks(MIPSState *mips, u64 globalTicks) { Core_ExecException(mips->pc, mips->pc, ExecExceptionType::JUMP); return; } - const MIPSOpcode op = MIPSOpcode(Memory::ReadUnchecked_U32(mips->pc)); - // Replacements and similar are processed here, intentionally. + const MIPSOpcode op = ReadExecutedOp(mips->pc); + // Replacements are processed here, intentionally. const MIPSInstruction *instr = MIPSGetInstruction(op); // Check for breakpoint. Route through ExecBreakPoint() (also used by the JIT diff --git a/Core/MIPS/RiscV/RiscVAsm.cpp b/Core/MIPS/RiscV/RiscVAsm.cpp index bf4b003cc7..d2ffe918ad 100644 --- a/Core/MIPS/RiscV/RiscVAsm.cpp +++ b/Core/MIPS/RiscV/RiscVAsm.cpp @@ -133,7 +133,7 @@ void RiscVJitBackend::GenerateFixedCode(MIPSState *mipsState) { // Fixed registers, these are always kept when in Jit context. LI(MEMBASEREG, Memory::base, SCRATCH1); LI(CTXREG, mipsState, SCRATCH1); - LI(JITBASEREG, GetBasePtr() - MIPS_EMUHACK_OPCODE, SCRATCH1); + LI(JITBASEREG, GetBasePtr(), SCRATCH1); LoadStaticRegisters(); WriteDebugProfilerStatus(IRProfilerStatus::IN_JIT); @@ -179,23 +179,22 @@ void RiscVJitBackend::GenerateFixedCode(MIPSState *mipsState) { LWU(SCRATCH1, CTXREG, offsetof(MIPSState, pc)); WriteDebugPC(SCRATCH1); -#ifdef MASKED_PSP_MEMORY - LI(SCRATCH2, 0x3FFFFFFF); + LI(SCRATCH2, (u32)Memory::BLOCK_SHADOW_ENTRY_MASK); AND(SCRATCH1, SCRATCH1, SCRATCH2); -#endif - ADD(SCRATCH1, SCRATCH1, MEMBASEREG); + LI(SCRATCH2, Memory::blockShadow, X7); + ADD(SCRATCH1, SCRATCH1, SCRATCH2); hooks_.dispatchFetch = GetCodePtr(); LWU(SCRATCH1, SCRATCH1, 0); - SRLI(SCRATCH2, SCRATCH1, 24); - // We're in other words comparing to the top 8 bits of MIPS_EMUHACK_OPCODE by subtracting. - ADDI(SCRATCH2, SCRATCH2, -(MIPS_EMUHACK_OPCODE >> 24)); - FixupBranch needsCompile = BNE(SCRATCH2, R_ZERO); - // No need to mask, JITBASEREG has already accounted for the upper bits. + // The entry is the block's offset into the code space, or 0 for none. + FixupBranch needsCompile = BEQ(SCRATCH1, R_ZERO); ADD(SCRATCH1, JITBASEREG, SCRATCH1); JR(SCRATCH1); SetJumpTarget(needsCompile); // No block found, let's jit. We don't need to save static regs, they're all callee saved. + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + hooks_.blockShadowHook = GetCodePtr(); RestoreRoundingMode(true); WriteDebugProfilerStatus(IRProfilerStatus::COMPILING); QuickCallFunctionR(&MIPSComp::JitAt, CTXREG, X7); diff --git a/Core/MIPS/fake/FakeJit.cpp b/Core/MIPS/fake/FakeJit.cpp index b37b09065d..2c1724874c 100644 --- a/Core/MIPS/fake/FakeJit.cpp +++ b/Core/MIPS/fake/FakeJit.cpp @@ -101,7 +101,7 @@ void FakeJit::ClearCache() } void FakeJit::InvalidateCacheAt(u32 em_address, int length) { - if (blocks.RangeMayHaveEmuHacks(em_address, em_address + length)) { + if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { blocks.InvalidateICache(em_address, length); } } @@ -142,12 +142,6 @@ bool FakeJit::DescribeCodePtr(const u8 *ptr, std::string &name) return false; } -void FakeJit::Comp_RunBlock(MIPSOpcode op) -{ - // This shouldn't be necessary, the dispatcher should catch us before we get here. - ERROR_LOG(Log::JIT, "Comp_RunBlock should never be reached!"); -} - void FakeJit::Comp_ReplacementFunc(MIPSOpcode op) { } diff --git a/Core/MIPS/fake/FakeJit.h b/Core/MIPS/fake/FakeJit.h index bab59c000f..8f325cccb8 100644 --- a/Core/MIPS/fake/FakeJit.h +++ b/Core/MIPS/fake/FakeJit.h @@ -54,7 +54,6 @@ public: void CompileDelaySlot(int flags); void EatInstruction(MIPSOpcode op); - void Comp_RunBlock(MIPSOpcode op) override; void Comp_ReplacementFunc(MIPSOpcode op) override; // Ops @@ -131,11 +130,6 @@ public: JitBlockCache *GetBlockCache() override { return &blocks; } JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - MIPSOpcode GetOriginalOp(MIPSOpcode op) override { return op; } - - std::vector SaveAndClearEmuHackOps() override { return blocks.SaveAndClearEmuHackOps(); } - void RestoreSavedEmuHackOps(std::vector saved) override { blocks.RestoreSavedEmuHackOps(saved); } - void ClearCache() override; void InvalidateCacheAt(u32 em_address, int length = 4) override; void UpdateFCR31() override {} diff --git a/Core/MIPS/x86/Asm.cpp b/Core/MIPS/x86/Asm.cpp index fc450ec226..4101768221 100644 --- a/Core/MIPS/x86/Asm.cpp +++ b/Core/MIPS/x86/Asm.cpp @@ -159,26 +159,23 @@ void Jit::GenerateFixedCode(JitOptions &jo) { MOV(32, R(EAX), MIPSSTATE_VAR(pc)); dispatcherInEAXNoCheck = GetCodePtr(); -#ifdef MASKED_PSP_MEMORY - AND(32, R(EAX), Imm32(Memory::MEMVIEW32_MASK)); -#endif - dispatcherFetch = GetCodePtr(); + // Look the block up in the shadow. The address stays in EAX for MemFault if this faults. + AND(32, R(EAX), Imm32(Memory::BLOCK_SHADOW_ENTRY_MASK)); #if PPSSPP_ARCH(X86) - _assert_msg_( Memory::base != 0, "Memory base bogus"); - MOV(32, R(EAX), MDisp(EAX, (u32)Memory::base)); + _assert_msg_(Memory::blockShadow != 0, "Block shadow bogus"); + dispatcherFetch = GetCodePtr(); + MOV(32, R(EAX), MDisp(EAX, (u32)Memory::blockShadow)); #elif PPSSPP_ARCH(AMD64) - MOV(32, R(EAX), MComplex(MEMBASEREG, RAX, SCALE_1, 0)); + MOV(64, R(RDX), ImmPtr(Memory::blockShadow)); + dispatcherFetch = GetCodePtr(); + MOV(32, R(EAX), MComplex(RDX, RAX, SCALE_1, 0)); #endif - MOV(32, R(EDX), R(EAX)); - _assert_msg_(MIPS_JITBLOCK_MASK == 0xFF000000, "Hardcoded assumption of emuhack mask"); - SHR(32, R(EDX), Imm8(24)); - CMP(32, R(EDX), Imm8(MIPS_EMUHACK_OPCODE >> 24)); - FixupBranch notfound = J_CC(CC_NE); + TEST(32, R(EAX), R(EAX)); + FixupBranch notfound = J_CC(CC_Z); if (enableDebug) { ADD(32, MIPSSTATE_VAR(debugCount), Imm8(1)); } - //grab from list and jump to it - AND(32, R(EAX), Imm32(MIPS_EMUHACK_VALUE_MASK)); + // The entry is the block's offset into the code space. #if PPSSPP_ARCH(X86) ADD(32, R(EAX), ImmPtr(GetBasePtr())); #elif PPSSPP_ARCH(AMD64) @@ -193,6 +190,9 @@ void Jit::GenerateFixedCode(JitOptions &jo) { SetJumpTarget(notfound); //Ok, no block, let's jit + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + blockShadowHook = GetCodePtr(); RestoreRoundingMode(true); LEA(PTRBITS, ECX, MDisp(CTXREG, -(s32)offsetof(MIPSState, f[0]))); // Adjust to get the real pointer. ABI_CallFunctionR(reinterpret_cast(&MIPSComp::JitAt), ECX); diff --git a/Core/MIPS/x86/Jit.cpp b/Core/MIPS/x86/Jit.cpp index 9313a21ba0..3aef46f92a 100644 --- a/Core/MIPS/x86/Jit.cpp +++ b/Core/MIPS/x86/Jit.cpp @@ -242,6 +242,8 @@ void Jit::ClearCache() blocks.Clear(); ClearCodeSpace(0); GenerateFixedCode(jo); + // The compile path may have moved, and replacement hooks' block shadow entries point to it. + Replacement_SetBlockShadowHook(GetBlockShadowHookValue()); } void Jit::SaveFlags() { @@ -360,7 +362,8 @@ u32 Jit::GetCompilerPC() { } MIPSOpcode Jit::GetOffsetInstruction(int offset) { - return Memory::Read_Instruction(GetCompilerPC() + 4 * offset); + const u32 addr = GetCompilerPC() + 4 * offset; + return Memory::IsValid4AlignedAddress(addr) ? ReadExecutedOp(addr) : Memory::Read_Instruction(addr); } void Jit::DoJit(u32 em_address, JitBlock *b) { @@ -384,7 +387,7 @@ void Jit::DoJit(u32 em_address, JitBlock *b) { // Jit breakpoints are quite fast, so let's do them in release too. CheckJitBreakpoint(GetCompilerPC(), 0); - MIPSOpcode inst = Memory::Read_Opcode_JIT(GetCompilerPC()); + MIPSOpcode inst = ReadExecutedOp(GetCompilerPC()); js.downcountAmount += MIPSGetInstructionCycleEstimate(inst); MIPSCompileOp(inst, this); @@ -486,11 +489,6 @@ bool Jit::DescribeCodePtr(const u8 *ptr, std::string &name) { return true; } -void Jit::Comp_RunBlock(MIPSOpcode op) { - // This shouldn't be necessary, the dispatcher should catch us before we get here. - ERROR_LOG(Log::JIT, "Comp_RunBlock"); -} - void Jit::LinkBlock(u8 *exitPoint, const u8 *checkedEntry) { if (PlatformIsWXExclusive()) { ProtectMemoryPages(exitPoint, 32, MEM_PROT_READ | MEM_PROT_WRITE); @@ -533,6 +531,12 @@ void Jit::Comp_ReplacementFunc(MIPSOpcode op) { // Inlined function calls (caught in jal) are handled differently. + // Memory never holds these: if it does, the game is executing data. Let the interpreter raise it. + if (Memory::Read_Instruction(GetCompilerPC()).encoding == op.encoding) { + Comp_Generic(op); + return; + } + int index = op.encoding & MIPS_EMUHACK_VALUE_MASK; const ReplacementTableEntry *entry = GetReplacementFunc(index); @@ -552,14 +556,7 @@ void Jit::Comp_ReplacementFunc(MIPSOpcode op) { } } - // Hack for old savestates: Avoid stack overflow (MIPSCompileOp/CompReplacementFunc) - // Not sure about the cause. - Memory::Opcode origInstruction = Memory::Read_Instruction(GetCompilerPC(), true); - if (origInstruction.encoding == op.encoding) { - ERROR_LOG(Log::HLE, "Replacement broken (savestate problem?): %08x at %08x", op.encoding, GetCompilerPC()); - return; - } - + Memory::Opcode origInstruction = Memory::Read_Instruction(GetCompilerPC()); if (disabled) { MIPSCompileOp(origInstruction, this); } else if (entry->jitReplaceFunc) { @@ -588,7 +585,7 @@ void Jit::Comp_ReplacementFunc(MIPSOpcode op) { if (entry->flags & (REPFLAG_HOOKENTER | REPFLAG_HOOKEXIT)) { // Compile the original instruction at this address. We ignore cycles for hooks. ApplyRoundingMode(); - MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC(), true), this); + MIPSCompileOp(Memory::Read_Instruction(GetCompilerPC()), this); } else { CMP(32, R(EAX), Imm32(0)); FixupBranch positive = J_CC(CC_GE); @@ -952,16 +949,6 @@ void Jit::CallProtectedLeaf(const void *func) { void Jit::Comp_DoNothing(MIPSOpcode op) { } -MIPSOpcode Jit::GetOriginalOp(MIPSOpcode op) { - JitBlockCache *bc = GetBlockCache(); - int block_num = bc->GetBlockNumberFromEmuHackOp(op, true); - if (block_num >= 0) { - return bc->GetOriginalFirstOp(block_num); - } else { - return op; - } -} - } // namespace #endif // PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64) diff --git a/Core/MIPS/x86/Jit.h b/Core/MIPS/x86/Jit.h index 0d2070ddfa..4f255b4d50 100644 --- a/Core/MIPS/x86/Jit.h +++ b/Core/MIPS/x86/Jit.h @@ -64,7 +64,6 @@ public: bool CodeInRange(const u8 *ptr) const override { return IsInSpace(ptr); } bool DescribeCodePtr(const u8 *ptr, std::string &name) override; - void Comp_RunBlock(MIPSOpcode op) override; void Comp_ReplacementFunc(MIPSOpcode op) override; // Ops @@ -160,14 +159,9 @@ public: JitBlockCache *GetBlockCache() override { return &blocks; } JitBlockCacheDebugInterface *GetBlockCacheDebugInterface() override { return &blocks; } - MIPSOpcode GetOriginalOp(MIPSOpcode op) override; - - std::vector SaveAndClearEmuHackOps() override { return blocks.SaveAndClearEmuHackOps(); } - void RestoreSavedEmuHackOps(std::vector saved) override { blocks.RestoreSavedEmuHackOps(saved); } - void ClearCache() override; void InvalidateCacheAt(u32 em_address, int length = 4) override { - if (blocks.RangeMayHaveEmuHacks(em_address, em_address + length)) { + if (blocks.RangeMayHaveBlocks(em_address, em_address + length)) { blocks.InvalidateICache(em_address, length); } } @@ -274,6 +268,9 @@ private: } bool PredictTakeBranch(u32 targetAddr, bool likely); + u32 GetBlockShadowHookValue() const override { + return (u32)(blockShadowHook - GetBasePtr()); + } bool IsAtDispatchFetch(const u8 *codePtr) const override { return codePtr == dispatcherFetch; } @@ -309,6 +306,7 @@ private: const u8 *endOfPregeneratedCode; const u8 *crashHandler; + const u8 *blockShadowHook; friend class JitSafeMem; friend class JitSafeMemFuncs; diff --git a/Core/MIPS/x86/X64IRAsm.cpp b/Core/MIPS/x86/X64IRAsm.cpp index 92a3dbecf2..0638cfc1c0 100644 --- a/Core/MIPS/x86/X64IRAsm.cpp +++ b/Core/MIPS/x86/X64IRAsm.cpp @@ -69,8 +69,7 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) { #if PPSSPP_ARCH(AMD64) bool jitbaseInR15 = false; int jitbaseCtxDisp = 0; - // We pre-bake the MIPS_EMUHACK_OPCODE subtraction into our jitbase value. - intptr_t jitbase = (intptr_t)GetBasePtr() - MIPS_EMUHACK_OPCODE; + intptr_t jitbase = (intptr_t)GetBasePtr(); if ((jitbase < -0x80000000LL || jitbase > 0x7FFFFFFFLL) && !Accessible((const u8 *)&mipsState->f[0], (const u8 *)jitbase)) { jo.reserveR15ForAsm = true; jitbaseInR15 = true; @@ -226,29 +225,21 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) { MovFromPC(SCRATCH1); WriteDebugPC(SCRATCH1); -#ifdef MASKED_PSP_MEMORY - AND(32, R(SCRATCH1), Imm32(Memory::MEMVIEW32_MASK)); -#endif + // Look the block up in the shadow. Its entry is the offset into the code space, or 0 for none. + AND(32, R(SCRATCH1), Imm32(Memory::BLOCK_SHADOW_ENTRY_MASK)); +#if PPSSPP_ARCH(X86) + _assert_msg_(Memory::blockShadow != 0, "Block shadow bogus"); hooks_.dispatchFetch = GetCodePtr(); -#if PPSSPP_ARCH(X86) - _assert_msg_( Memory::base != 0, "Memory base bogus"); - MOV(32, R(SCRATCH1), MDisp(SCRATCH1, (u32)Memory::base)); + MOV(32, R(SCRATCH1), MDisp(SCRATCH1, (u32)Memory::blockShadow)); #elif PPSSPP_ARCH(AMD64) - MOV(32, R(SCRATCH1), MComplex(MEMBASEREG, SCRATCH1, SCALE_1, 0)); + MOV(64, R(RDX), ImmPtr(Memory::blockShadow)); + hooks_.dispatchFetch = GetCodePtr(); + MOV(32, R(SCRATCH1), MComplex(RDX, SCRATCH1, SCALE_1, 0)); #endif - _assert_msg_(MIPS_JITBLOCK_MASK == 0xFF000000, "Hardcoded assumption of emuhack mask"); - if (cpu_info.bBMI2) { - RORX(32, EDX, R(SCRATCH1), 24); - CMP(8, R(EDX), Imm8(MIPS_EMUHACK_OPCODE >> 24)); - } else { - MOV(32, R(EDX), R(SCRATCH1)); - SHR(32, R(EDX), Imm8(24)); - CMP(32, R(EDX), Imm8(MIPS_EMUHACK_OPCODE >> 24)); - } - FixupBranch needsCompile = J_CC(CC_NE); - // We don't mask here - that's baked into jitbase. + TEST(32, R(SCRATCH1), R(SCRATCH1)); + FixupBranch needsCompile = J_CC(CC_Z); #if PPSSPP_ARCH(X86) - LEA(32, SCRATCH1, MDisp(SCRATCH1, (u32)GetBasePtr() - MIPS_EMUHACK_OPCODE)); + LEA(32, SCRATCH1, MDisp(SCRATCH1, (u32)GetBasePtr())); #elif PPSSPP_ARCH(AMD64) if (jitbaseInR15) { ADD(64, R(SCRATCH1), R(JITBASEREG)); @@ -263,6 +254,9 @@ void X64JitBackend::GenerateFixedCode(MIPSState *mipsState) { SetJumpTarget(needsCompile); // No block found, let's jit. We don't need to save static regs, they're all callee saved. + // A replacement hook's block shadow entry jumps here too (GetBlockShadowHookValue()). JitAt + // replaces it with the new block's entry, and destroying that block puts the hook value back. + hooks_.blockShadowHook = GetCodePtr(); RestoreRoundingMode(true); WriteDebugProfilerStatus(IRProfilerStatus::COMPILING); LEA(PTRBITS, ECX, MDisp(CTXREG, -(s32)offsetof(MIPSState, f[0]))); diff --git a/Core/MemFault.cpp b/Core/MemFault.cpp index 43b82fef7b..841e14916d 100644 --- a/Core/MemFault.cpp +++ b/Core/MemFault.cpp @@ -141,6 +141,12 @@ bool HandleFault(uintptr_t hostAddress, void *ctx) { const uintptr_t addressSpaceSize = 0x100000000ULL; #endif + // The dispatcher looks a jump target up in the block shadow, which faults where there's no PSP memory. + const uintptr_t shadowAddress = (uintptr_t)blockShadow; + if (shadowAddress && hostAddress >= shadowAddress && hostAddress <= shadowAddress + BLOCK_SHADOW_MASK) { + baseAddress = shadowAddress; + } + // Check whether hostAddress is within the PSP memory space, which (likely) means it was a guest executable that did the bad access. bool invalidHostAddress = hostAddress == (uintptr_t)0xFFFFFFFFFFFFFFFFULL; if (hostAddress < baseAddress || hostAddress >= baseAddress + addressSpaceSize) { diff --git a/Core/MemMap.cpp b/Core/MemMap.cpp index bad98c8486..78a1c9b7fb 100644 --- a/Core/MemMap.cpp +++ b/Core/MemMap.cpp @@ -26,6 +26,7 @@ #include "Common/CommonTypes.h" #include "Common/MemArena.h" +#include "Common/MemoryUtil.h" #include "Common/Serialize/Serializer.h" #include "Common/Serialize/SerializeFuncs.h" @@ -46,6 +47,9 @@ namespace Memory { // The base pointer to the auto-mirrored arena. u8* base = nullptr; +u8 *blockShadow = nullptr; +static const size_t BLOCK_SHADOW_SIZE = (size_t)BLOCK_SHADOW_MASK + 1; + // The MemArena class MemArena g_arena; // ============== @@ -328,6 +332,27 @@ void MemoryMap_Shutdown() { #endif } +// Commits the shadow under every view, so exactly the addresses that are valid PSP memory have entries. +// Mirrors fold onto the same range under the mask, and committing it again is harmless. +static bool BlockShadow_Setup(MemMapSetupFlags flags) { + blockShadow = (u8 *)ReserveMemoryPages(BLOCK_SHADOW_SIZE); + if (!blockShadow) { + return false; + } + for (size_t i = 0; i < ARRAY_SIZE(views); i++) { + if (views[i].size == 0 || SkipView(flags, views[i].flags)) + continue; + const u32 start = views[i].virtual_address & BLOCK_SHADOW_MASK; + _dbg_assert_((size_t)start + views[i].size <= BLOCK_SHADOW_SIZE); + if (!CommitMemoryPages(blockShadow + start, views[i].size)) { + FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE); + blockShadow = nullptr; + return false; + } + } + return true; +} + // On some 32 bit platforms (like old Android, old iOS, etc.), there are/were restrictions on memory map sizes. // This particular size I can't find any sources for though. static const int MAX_MMAP_SIZE = 31 * 1024 * 1024; @@ -348,7 +373,15 @@ bool Init(MemMapSetupFlags flags) { views[i].size = std::min(std::max((int)g_MemorySize - MAX_MMAP_SIZE * 2, 0), MAX_MMAP_SIZE); } + // The shadow first, so the base search can't put it in a gap of the PSP's 4GB window, where a bad + // guest access would hit it instead of faulting. + if (!BlockShadow_Setup(flags)) { + return false; + } + if (!MemoryMap_Setup(flags)) { + FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE); + blockShadow = nullptr; return false; } @@ -468,6 +501,8 @@ void Shutdown() { u32 flags = 0; MemoryMap_Shutdown(); base = nullptr; + FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE); + blockShadow = nullptr; DEBUG_LOG(Log::MemMap, "Memory system shut down."); } @@ -475,78 +510,41 @@ bool IsActive() { return base != nullptr; } -static Opcode Read_Instruction(u32 address, bool resolveReplacements, Opcode inst) { - if (!MIPS_IS_EMUHACK(inst.encoding)) { - return inst; - } - - // No mutex on jit access here, but we assume the caller has locked, if necessary. - if (MIPS_IS_RUNBLOCK(inst.encoding) && MIPSComp::jit) { - inst = MIPSComp::jit->GetOriginalOp(inst); - if (resolveReplacements && MIPS_IS_REPLACEMENT(inst)) { - u32 op; - if (GetReplacedOpAt(address, &op)) { - if (MIPS_IS_EMUHACK(op)) { - ERROR_LOG(Log::MemMap, "WTF 1"); - return Opcode(op); - } else { - return Opcode(op); - } - } else { - ERROR_LOG(Log::MemMap, "Replacement, but no replacement op? %08x", inst.encoding); - } - } - return inst; - } else if (resolveReplacements && MIPS_IS_REPLACEMENT(inst.encoding)) { - u32 op; - if (GetReplacedOpAt(address, &op)) { - if (MIPS_IS_EMUHACK(op)) { - ERROR_LOG(Log::MemMap, "WTF 2"); - return Opcode(op); - } else { - return Opcode(op); - } - } else { - return inst; - } - } else { - return inst; - } -} - -Opcode Read_Instruction(u32 address, bool resolveReplacements) { +Opcode Read_Instruction(u32 address) { if (!IsValid4AlignedAddress(address)) { // BAD! _dbg_assert_(false); return Opcode(0); } - - Opcode inst = Opcode(ReadUnchecked_U32(address)); - return Read_Instruction(address, resolveReplacements, inst); + return Opcode(ReadUnchecked_U32(address)); } -Opcode ReadUnchecked_Instruction(u32 address, bool resolveReplacements) { +Opcode ReadUnchecked_Instruction(u32 address) { _dbg_assert_((address & 3) == 0); - Opcode inst = Opcode(ReadUnchecked_U32(address)); - return Read_Instruction(address, resolveReplacements, inst); + return Opcode(ReadUnchecked_U32(address)); } -// WARNING! Caller checks that address is valid! -Opcode Read_Opcode_JIT(u32 address) { - _dbg_assert_(Memory::IsValid4AlignedAddress(address)); - Opcode inst = Opcode(ReadUnchecked_U32(address)); - // No mutex around jit access here, but we assume caller has if necessary. - if (MIPS_IS_RUNBLOCK(inst.encoding) && MIPSComp::jit) { - return MIPSComp::jit->GetOriginalOp(inst); - } else { - return inst; - } +u32 ReadBlockShadow(u32 address) { + if (!blockShadow || !IsValid4AlignedAddress(address)) + return 0; + return *GetBlockShadowEntry(address); } -// WARNING! No checks! -void Write_Opcode_JIT(const u32 address, const Opcode& _Value) { - _dbg_assert_((address & 3) == 0); - Memory::WriteUnchecked_U32(_Value.encoding, address); +void WriteBlockShadow(u32 address, u32 value) { + if (!blockShadow || !IsValid4AlignedAddress(address)) + return; + *GetBlockShadowEntry(address) = value; +} + +bool ClearBlockShadow(u32 address, u32 value) { + if (!blockShadow || !IsValid4AlignedAddress(address)) + return false; + u32 *entry = GetBlockShadowEntry(address); + if (*entry != value) + return false; + // A replacement hook keeps the entry nonzero, see ReplaceTables.h. + *entry = Replacement_IsHooked(address) ? Replacement_GetBlockShadowHook() : 0; + return true; } void Memset(const u32 addr, const u8 value, const u32 size, const char *tag) { diff --git a/Core/MemMap.h b/Core/MemMap.h index 9bda44338d..0ac3684f13 100644 --- a/Core/MemMap.h +++ b/Core/MemMap.h @@ -108,8 +108,45 @@ enum { // This wraparound should work for PSP too. MEMVIEW32_MASK = 0x3FFFFFFF, #endif + + // Index mask for the block shadow (see blockShadow below). Dropping bits 30-31 folds the + // kernel and uncached mirrors onto the same entry, the way they share physical memory. + // Where address space is short, bits 28-29 are dropped too, so garbage PCs with those bits + // set alias real code instead of faulting. +#if PPSSPP_ARCH(64BIT) && !PPSSPP_PLATFORM(IOS) && !PPSSPP_PLATFORM(SWITCH) + BLOCK_SHADOW_MASK = 0x3FFFFFFF, +#else + BLOCK_SHADOW_MASK = 0x0FFFFFFF, +#endif + // For looking up an entry from a PC. A misaligned PC would otherwise read across two entries, and + // jump to a garbage offset. + BLOCK_SHADOW_ENTRY_MASK = BLOCK_SHADOW_MASK & ~3, }; +enum : u32 { + // The block shadow value for a replacement hook when no JIT is running, and for the IR + // interpreter, which keeps this offset in its arena empty. + BLOCK_SHADOW_HOOK_NO_JIT = 1, +}; + +// One u32 per guest instruction, at the same byte offset as the instruction (after masking with +// BLOCK_SHADOW_MASK). Holds the JIT's handle for the block starting there, 0 for none, or the +// replacement hook value (see ReplaceTables.h). +// Only the ranges backed by PSP memory are committed, so it's never safe to touch an entry for +// an address that isn't valid. Reallocated by Init(), along with base. +extern u8 *blockShadow; + +inline u32 *GetBlockShadowEntry(u32 address) { + return (u32 *)(blockShadow + (address & BLOCK_SHADOW_ENTRY_MASK)); +} + +// These check the address, and are safe to call when there's no shadow (after Shutdown()). +u32 ReadBlockShadow(u32 address); +void WriteBlockShadow(u32 address, u32 value); +// Clears the entry only if it still holds value, back to the hook value if the address is hooked. +// Returns true if it did. +bool ClearBlockShadow(u32 address, u32 value); + enum { MV_MIRROR_PREVIOUS = 1, MV_IS_PRIMARY_RAM = 0x100, @@ -140,14 +177,9 @@ void DoState(PointerWrap &p); // False when shutdown has already been called. bool IsActive(); -// used by JIT to read instructions. Does not resolve replacements. -Opcode Read_Opcode_JIT(const u32 _Address); -// used by JIT. Reads in the "Locked cache" mode -void Write_Opcode_JIT(const u32 _Address, const Opcode& _Value); - -// Should be used by analyzers, disassemblers etc. Does resolve replacements. -Opcode Read_Instruction(const u32 _Address, bool resolveReplacements = false); -Opcode ReadUnchecked_Instruction(const u32 _Address, bool resolveReplacements = false); +Opcode Read_Instruction(const u32 _Address); +// Caller checks that the address is valid. +Opcode ReadUnchecked_Instruction(const u32 _Address); u8 ReadOrException_U8(const u32 _Address); u16 ReadOrException_U16(const u32 _Address); diff --git a/Core/SaveState.cpp b/Core/SaveState.cpp index ed96079923..67f1086568 100644 --- a/Core/SaveState.cpp +++ b/Core/SaveState.cpp @@ -177,32 +177,16 @@ int g_screenshotFailures; CoreTiming::DoState(p); } - // Memory is a bit tricky when jit is enabled, since there's emuhacks in it. - // These must be saved before copying out memory and restored after. - auto savedReplacements = SaveAndClearReplacements(); - if (MIPSComp::jit && (p.mode == p.MODE_WRITE || p.mode == p.MODE_VERIFY)) { - if (MIPSComp::jit) { - std::vector savedBlocks; - savedBlocks = MIPSComp::jit->SaveAndClearEmuHackOps(); - Memory::DoState(p); - MIPSComp::jit->RestoreSavedEmuHackOps(savedBlocks); - } else { - Memory::DoState(p); - } - } else { - Memory::DoState(p); + // The state may hold different code. KernelModuleDoState hooks the functions again. + if (p.mode == p.MODE_READ) { + RestoreReplacedInstructions(0, 0xFFFFFFFF); } + Memory::DoState(p); if (s >= 3) { CoreTiming::DoState(p); } - // Don't bother restoring if reading, we'll deal with that in KernelModuleDoState. - // In theory, different functions might have been runtime loaded in the state. - if (p.mode != p.MODE_READ) { - RestoreSavedReplacements(savedReplacements); - } - MemoryStick_DoState(p); currentMIPS->DoState(p); HLEDoState(p); diff --git a/Windows/Debugger/DumpMemoryWindow.cpp b/Windows/Debugger/DumpMemoryWindow.cpp index 495e837108..977072da3d 100644 --- a/Windows/Debugger/DumpMemoryWindow.cpp +++ b/Windows/Debugger/DumpMemoryWindow.cpp @@ -80,7 +80,6 @@ INT_PTR CALLBACK DumpMemoryWindow::dlgFunc(HWND hwnd, UINT iMsg, WPARAM wParam, break; case IDOK: if (bp->fetchDialogData(hwnd)) { - bool includeReplacements = SendMessage(GetDlgItem(hwnd, IDC_DUMP_INCLUDEHACKS), BM_GETCHECK, 0, 0) != 0; // Route the actual memory dump to the CPU thread instead of forcing the emulator // to pause and poking at it directly from this GUI thread - see @@ -98,19 +97,7 @@ INT_PTR CALLBACK DumpMemoryWindow::dlgFunc(HWND hwnd, UINT iMsg, WPARAM wParam, return; } - if (includeReplacements) { - fwrite(Memory::GetPointerOrException(bp->start), 1, bp->size, output); - } else { - auto savedReplacements = SaveAndClearReplacements(); - if (MIPSComp::jit) { - auto savedBlocks = MIPSComp::jit->SaveAndClearEmuHackOps(); - fwrite(Memory::GetPointerOrException(bp->start), 1, bp->size, output); - MIPSComp::jit->RestoreSavedEmuHackOps(savedBlocks); - } else { - fwrite(Memory::GetPointerOrException(bp->start), 1, bp->size, output); - } - RestoreSavedReplacements(savedReplacements); - } + fwrite(Memory::GetPointerOrException(bp->start), 1, bp->size, output); fclose(output); outcome = Outcome::Success; diff --git a/Windows/ppsspp.rc b/Windows/ppsspp.rc index f28cc1cbaf..11607ab915 100644 --- a/Windows/ppsspp.rc +++ b/Windows/ppsspp.rc @@ -402,7 +402,6 @@ BEGIN CONTROL "VRAM",IDC_DUMP_VRAM,"Button",BS_AUTORADIOBUTTON,152,22,35,10 CONTROL "Scratchpad",IDC_DUMP_SCRATCHPAD,"Button",BS_AUTORADIOBUTTON,152,34,52,10 CONTROL "Custom range",IDC_DUMP_CUSTOMRANGE,"Button",BS_AUTORADIOBUTTON,152,46,61,10 - CONTROL "Include jit checks and replacements",IDC_DUMP_INCLUDEHACKS,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,41,58,120,14 DEFPUSHBUTTON "OK",IDOK,117,79,50,14 PUSHBUTTON "Cancel",IDCANCEL,173,79,50,14 END diff --git a/Windows/resource.h b/Windows/resource.h index 523e59b92f..feafb68960 100644 --- a/Windows/resource.h +++ b/Windows/resource.h @@ -216,7 +216,6 @@ #define ID_DISASM_ASSEMBLE 40091 #define ID_DISASM_ADDNEWBREAKPOINT 40092 #define ID_DISASM_EDITBREAKPOINT 40093 -#define IDC_DUMP_INCLUDEHACKS 40094 #define ID_EMULATION_CHEATS 40096 #define ID_HELP_CHINESE_FORUM 40097 #define ID_OPTIONS_MORE_SETTINGS 40098 diff --git a/unittest/JitHarness.cpp b/unittest/JitHarness.cpp index 5d1f59fb6f..cf277b8240 100644 --- a/unittest/JitHarness.cpp +++ b/unittest/JitHarness.cpp @@ -16,6 +16,8 @@ // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. #include +#include +#include #include "ppsspp_config.h" @@ -37,6 +39,8 @@ #include "Core/CoreTiming.h" #include "Core/Config.h" #include "Core/HLE/HLE.h" +#include "Core/HLE/ReplaceTables.h" +#include "Core/MIPS/MIPSAnalyst.h" #include "unittest/UnitTest.h" void UnitTestTerminator() { @@ -155,6 +159,22 @@ bool TestJit() { *p++ = MIPS_MAKE_BREAK(1); *p++ = MIPS_MAKE_JR_RA(); + // The cores keep their block lookup and hooks out of PSP memory, so the code has to read back + // as assembled while its blocks are live. + const u8 *code = Memory::GetPointerOrException(currentMIPS->pc); + const std::vector assembled(code, (const u8 *)p); + bool memoryClean = true; + auto checkMemoryClean = [&](const char *core) { + if (memcmp(code, assembled.data(), assembled.size()) != 0) { + printf("%s modified the code in PSP memory\n", core); + memoryClean = false; + } + if (MIPSComp::jit && Memory::ReadBlockShadow(PSP_GetUserMemoryBase()) == 0) { + printf("%s didn't enter its block in the block shadow\n", core); + memoryClean = false; + } + }; + // Dogfood. addr = currentMIPS->pc; for (size_t j = 0; j < ARRAY_SIZE(lines); ++j) { @@ -169,13 +189,19 @@ bool TestJit() { double jit_speed = 0.0, jit_ir_speed = 0.0, ir_speed = 0.0, interp_speed = 0.0; if (compileSuccess) { interp_speed = ExecCPUTest(); + checkMemoryClean("Interpreter"); mipsr4k.UpdateCore(CPUCore::IR_INTERPRETER); - ir_speed = ExecCPUTest(); + ir_speed = ExecCPUTest(false); + checkMemoryClean("IR interpreter"); + MIPSComp::jit->ClearCache(); mipsr4k.UpdateCore(CPUCore::JIT); - jit_speed = ExecCPUTest(); + jit_speed = ExecCPUTest(false); + checkMemoryClean("JIT"); + MIPSComp::jit->ClearCache(); #if !PPSSPP_PLATFORM(MAC) mipsr4k.UpdateCore(CPUCore::JIT_IR); jit_ir_speed = ExecCPUTest(false); // not clearing, so the below can do things. + checkMemoryClean("JIT IR"); #endif // Disassemble @@ -198,5 +224,139 @@ bool TestJit() { DestroyJitHarness(); - return jit_speed >= interp_speed; + return memoryClean && jit_speed >= interp_speed; +} + +// Follows a replacement hook's block shadow entry through compiling, invalidating, clearing and +// switching cores. The hook is installed the normal way, by naming a function memcpy. +static bool CheckReplacementHooks(u32 func, u32 size, const std::vector &code) { + auto compile = [&]() { + currentMIPS->pc = func; + MIPSComp::JitAt(currentMIPS); + }; + auto codeIntact = [&]() { + return memcmp(Memory::GetPointerOrException(func), code.data(), code.size()) == 0; + }; + + EXPECT_TRUE(Replacement_IsHooked(func)); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), (u32)Memory::BLOCK_SHADOW_HOOK_NO_JIT); + EXPECT_TRUE(codeIntact()); + + static const CPUCore cores[] = { + CPUCore::JIT, + CPUCore::IR_INTERPRETER, +#if !PPSSPP_PLATFORM(MAC) + CPUCore::JIT_IR, +#endif + }; + for (CPUCore core : cores) { + mipsr4k.UpdateCore(core); + const u32 hook = MIPSComp::jit->GetBlockShadowHookValue(); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), hook); + + // Compiling replaces the hook value with the block's entry. + compile(); + const u32 block = Memory::ReadBlockShadow(func); + EXPECT_TRUE(block != 0 && block != hook); + + // Destroying the block puts it back, whichever way it goes. + MIPSComp::jit->InvalidateCacheAt(func, 4); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), hook); + compile(); + MIPSComp::jit->ClearCache(); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), MIPSComp::jit->GetBlockShadowHookValue()); + + // Leave a block for the core switch to destroy. + compile(); + EXPECT_TRUE(codeIntact()); + } + mipsr4k.UpdateCore(CPUCore::INTERPRETER); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), (u32)Memory::BLOCK_SHADOW_HOOK_NO_JIT); + + // Removing the hook while a block holds the entry clears it once the block is destroyed. + mipsr4k.UpdateCore(CPUCore::JIT); + compile(); + RestoreReplacedInstructions(func, func + size); + EXPECT_FALSE(Replacement_IsHooked(func)); + currentMIPS->ProcessPendingInvalidates(); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), 0); + + // Rewriting the rest of the function drops the hook once the game invalidates the icache over it, + // with the cache instruction (which the JITs leave to the interpreter). + MIPSAnalyst::ReplaceFunctions(); + const u32 bodyOp = Memory::ReadUnchecked_U32(func + 8); + Memory::WriteUnchecked_U32(MIPS_MAKE_NOP(), func + 8); + EXPECT_TRUE(Replacement_IsHooked(func)); + currentMIPS->r[MIPS_REG_A0] = func + 8; + MIPSInterpret(currentMIPS, MIPSOpcode((0x2F << 26) | (MIPS_REG_A0 << 21) | (8 << 16))); // cache 0x08, 0(a0) + EXPECT_FALSE(Replacement_IsHooked(func)); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), 0); + Memory::WriteUnchecked_U32(bodyOp, func + 8); + + // Cheats and the debugger queue the invalidation before writing. The check happens when it's processed. + MIPSAnalyst::ReplaceFunctions(); + currentMIPS->InvalidateICacheRangeDeferred(func + 8, 4); + Memory::WriteUnchecked_U32(MIPS_MAKE_NOP(), func + 8); + EXPECT_TRUE(Replacement_IsHooked(func)); + currentMIPS->ProcessPendingInvalidates(); + EXPECT_FALSE(Replacement_IsHooked(func)); + Memory::WriteUnchecked_U32(bodyOp, func + 8); + + // Changed immediates (relocations) keep the hook, as they don't stop the function from matching. + MIPSAnalyst::ReplaceFunctions(); + Memory::WriteUnchecked_U32(bodyOp + 1, func + 8); + currentMIPS->InvalidateICacheRangeDeferred(func + 8, 4); + currentMIPS->ProcessPendingInvalidates(); + EXPECT_TRUE(Replacement_IsHooked(func)); + Memory::WriteUnchecked_U32(bodyOp, func + 8); + + // The game overwriting the hooked instruction drops the hook. + MIPSAnalyst::ReplaceFunctions(); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), MIPSComp::jit->GetBlockShadowHookValue()); + Memory::WriteUnchecked_U32(MIPS_MAKE_NOP(), func); + EXPECT_EQ_HEX(ReadExecutedOp(func).encoding, MIPS_MAKE_NOP()); + EXPECT_FALSE(Replacement_IsHooked(func)); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(func), 0); + return true; +} + +bool TestJitReplacementHooks() { + SetupJitHarness(); + Replacement_Init(); + + const u32 func = PSP_GetUserMemoryBase() + 0x1000; + static const char *lines[] = { + "addiu v0, a0, 0", + "addiu t0, a1, 0", + "addiu t1, a2, 0", + "addiu t2, zero, 1", + "addiu t3, zero, 2", + "jr ra", + "nop", + }; + u32 addr = func; + for (const char *line : lines) { + std::string error; + if (!MipsAssembleOpcode(line, currentDebugMIPS, addr, &error)) { + printf("ERROR: %s\n", error.c_str()); + return false; + } + addr += 4; + } + const u32 size = addr - func; + const u8 *codePtr = Memory::GetPointerOrException(func); + const std::vector code(codePtr, codePtr + size); + + g_symbolMap->AddFunction("memcpy", func, size); + MIPSAnalyst::RegisterFunction(func, size, "memcpy"); + MIPSAnalyst::UpdateHashMap(); + MIPSAnalyst::ReplaceFunctions(); + + const bool success = CheckReplacementHooks(func, size, code); + + RestoreReplacedInstructions(func, func + size); + Replacement_Shutdown(); + MIPSAnalyst::Reset(); + DestroyJitHarness(); + return success; } diff --git a/unittest/JitHarness.h b/unittest/JitHarness.h index 4d716112ac..2e6a53928d 100644 --- a/unittest/JitHarness.h +++ b/unittest/JitHarness.h @@ -18,3 +18,4 @@ #pragma once bool TestJit(); +bool TestJitReplacementHooks(); diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 5ace7db54b..3a55f8191a 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -2255,6 +2255,45 @@ static bool TestMemMap() { return true; } +static bool CheckBlockShadow() { + // Every valid address has an entry, starting out empty. The ends of each view would fault otherwise. + static const u32 validAddresses[] = { + 0x00010000, 0x00013FFC, 0x40010000, // Scratchpad, and its uncached mirror. + 0x04000000, 0x047FFFFC, // VRAM and its mirrors. + 0x08000000, 0x09F00000, 0x0BE00000, 0x0BFFFFFC, // RAM, across the views it's mapped in. + 0x48000000, 0x88000000, 0xC8000000, // RAM mirrors. + }; + for (u32 addr : validAddresses) { + EXPECT_EQ_HEX(Memory::ReadBlockShadow(addr), 0); + } + + // The kernel and uncached mirrors share an entry, like they share memory. + Memory::WriteBlockShadow(0x08804000, 0x1234); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(0x88804000), 0x1234); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(0x48804000), 0x1234); + + // Invalid addresses read as empty, and ignore writes. + Memory::WriteBlockShadow(0x0C000000, 0x1234); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(0x0C000000), 0); + + // Clearing leaves an entry that holds something else alone. + EXPECT_FALSE(Memory::ClearBlockShadow(0x08804000, 0x5678)); + EXPECT_TRUE(Memory::ClearBlockShadow(0x88804000, 0x1234)); + EXPECT_EQ_HEX(Memory::ReadBlockShadow(0x08804000), 0); + return true; +} + +static bool TestBlockShadow() { + Memory::g_MemorySize = Memory::RAM_DOUBLE_SIZE; + if (!Memory::Init(Memory::MemMapSetupFlags::Default)) { + printf("Memory::Init failed\n"); + return false; + } + const bool success = CheckBlockShadow(); + Memory::Shutdown(); + return success; +} + static bool TestPath() { // Also test the Path class while we're at it. Path path("/asdf/jkl/"); @@ -3248,12 +3287,14 @@ TestItem availableTests[] = { TEST_ITEM(Utf8), TEST_ITEM(IRPassSimplify), TEST_ITEM(Jit), + TEST_ITEM(JitReplacementHooks), TEST_ITEM(VFPUMatrixTranspose), TEST_ITEM(ParseLBN), TEST_ITEM(CachingFileLoader), TEST_ITEM(QuickTexHash), TEST_ITEM(CLZ), TEST_ITEM(MemMap), + TEST_ITEM(BlockShadow), TEST_ITEM(ShaderGenerators), TEST_ITEM(Path), TEST_ITEM(AndroidContentURI),