From 5c2521729dc005eeda6e312d81f307208c2e8a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 10 Nov 2026 19:38:19 +0100 Subject: [PATCH] IR JIT: Compile faster Measured with --jit-bench on a Galaxy S8, recompiling the 11243 blocks of a Tekken 6 state: 363 ms -> 300-310 ms, with the ARM64 pairing fix. - FlushAll, which runs at every exit, no longer looks at all 256 regs. FPRs are found from the native regs, and the rest are skipped quickly when in memory. - IRApplyPasses alternates between two buffers that keep their capacity, instead of allocating a new one for each pass. - The ARM64 backend flushes the icache once per block, for the block's code and every exit linked to and from it, with one set of barriers. Co-Authored-By: Claude Opus 5.5 --- Common/Arm64Emitter.cpp | 38 +++++++++++++++++++++++---------- Common/Arm64Emitter.h | 4 ++++ Core/MIPS/ARM64/Arm64IRJit.cpp | 25 ++++++++++++++++++++-- Core/MIPS/ARM64/Arm64IRJit.h | 9 ++++++++ Core/MIPS/IR/IRNativeCommon.cpp | 1 + Core/MIPS/IR/IRNativeCommon.h | 3 +++ Core/MIPS/IR/IRPassSimplify.cpp | 15 ++++++------- Core/MIPS/IR/IRRegCache.cpp | 29 +++++++++++++++++++++---- 8 files changed, 98 insertions(+), 26 deletions(-) diff --git a/Common/Arm64Emitter.cpp b/Common/Arm64Emitter.cpp index ff0eb84717..a1c70cd5d8 100644 --- a/Common/Arm64Emitter.cpp +++ b/Common/Arm64Emitter.cpp @@ -329,13 +329,25 @@ void ARM64XEmitter::FlushIcache() m_lastCacheFlushEnd = m_code; } -void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end) +void ARM64XEmitter::TakeIcacheFlushRange(const u8 **start, const u8 **end) { + *start = m_lastCacheFlushEnd; + *end = m_code; + m_lastCacheFlushEnd = m_code; +} + +void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end) { + FlushIcacheSections(&start, &end, 1); +} + +void ARM64XEmitter::FlushIcacheSections(const u8 *const *starts, const u8 *const *ends, int count) { #if PPSSPP_PLATFORM(IOS) || PPSSPP_PLATFORM(MAC) // Header file says this is equivalent to: sys_icache_invalidate(start, end - start); - sys_cache_control(kCacheFunctionPrepareForExecution, (void *)start, end - start); + for (int i = 0; i < count; i++) + sys_cache_control(kCacheFunctionPrepareForExecution, (void *)starts[i], ends[i] - starts[i]); #elif PPSSPP_PLATFORM(WINDOWS) - FlushInstructionCache(GetCurrentProcess(), start, end - start); + for (int i = 0; i < count; i++) + FlushInstructionCache(GetCurrentProcess(), starts[i], ends[i] - starts[i]); #elif PPSSPP_ARCH(ARM64) // Code from Dolphin, contributed by the Mono project. @@ -359,16 +371,20 @@ void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end) dcache_line_size = dsize = dcache_line_size < dsize ? dcache_line_size : dsize; } - u64 addr = (u64)start & ~(u64)(dsize - 1); - for (; addr < (u64)end; addr += dsize) - // use "civac" instead of "cvau", as this is the suggested workaround for - // Cortex-A53 errata 819472, 826319, 827319 and 824069. - __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory"); + for (int i = 0; i < count; i++) { + u64 addr = (u64)starts[i] & ~(u64)(dsize - 1); + for (; addr < (u64)ends[i]; addr += dsize) + // use "civac" instead of "cvau", as this is the suggested workaround for + // Cortex-A53 errata 819472, 826319, 827319 and 824069. + __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory"); + } __asm__ volatile("dsb ish" : : : "memory"); - addr = (u64)start & ~(u64)(isize - 1); - for (; addr < (u64)end; addr += isize) - __asm__ volatile("ic ivau, %0" : : "r"(addr) : "memory"); + for (int i = 0; i < count; i++) { + u64 addr = (u64)starts[i] & ~(u64)(isize - 1); + for (; addr < (u64)ends[i]; addr += isize) + __asm__ volatile("ic ivau, %0" : : "r"(addr) : "memory"); + } __asm__ volatile("dsb ish" : : : "memory"); __asm__ volatile("isb" : : : "memory"); diff --git a/Common/Arm64Emitter.h b/Common/Arm64Emitter.h index bd59d6e9e2..c196c7dfbf 100644 --- a/Common/Arm64Emitter.h +++ b/Common/Arm64Emitter.h @@ -421,6 +421,10 @@ public: const u8 *NopAlignCode16(); void FlushIcache(); void FlushIcacheSection(const u8* start, const u8* end); + // Instead of FlushIcache(): the range it would flush, now counted as flushed. For flushing with others. + void TakeIcacheFlushRange(const u8 **start, const u8 **end); + // Like FlushIcacheSection() on each, with one set of barriers. + static void FlushIcacheSections(const u8 *const *starts, const u8 *const *ends, int count); u8* GetWritableCodePtr(); // FixupBranch branching diff --git a/Core/MIPS/ARM64/Arm64IRJit.cpp b/Core/MIPS/ARM64/Arm64IRJit.cpp index c8afe32889..3def68b6d9 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.cpp +++ b/Core/MIPS/ARM64/Arm64IRJit.cpp @@ -213,12 +213,30 @@ bool Arm64JitBackend::CompileBlock(int block_num, u32 startPC, const IRInst *ins } EndWrite(); - FlushIcache(); + // Flushed with the links, after FinalizeBlock. + const u8 *flushStart, *flushEnd; + TakeIcacheFlushRange(&flushStart, &flushEnd); + AddPendingIcacheFlush(flushStart, flushEnd); compilingBlockNum_ = -1; return true; } +void Arm64JitBackend::AddPendingIcacheFlush(const u8 *start, const u8 *end) { + if (numPendingFlushes_ == MAX_PENDING_ICACHE_FLUSHES) + FlushPendingIcache(); + pendingFlushStarts_[numPendingFlushes_] = start; + pendingFlushEnds_[numPendingFlushes_] = end; + numPendingFlushes_++; +} + +void Arm64JitBackend::FlushPendingIcache() { + if (numPendingFlushes_ == 0) + return; + FlushIcacheSections(pendingFlushStarts_, pendingFlushEnds_, numPendingFlushes_); + numPendingFlushes_ = 0; +} + void Arm64JitBackend::WriteConstExit(uint32_t pc) { int block_num = FindBlockByStart(pc); const IRNativeBlock *nativeBlock = GetNativeBlock(block_num); @@ -259,7 +277,8 @@ void Arm64JitBackend::OverwriteExit(int srcOffset, int len, int block_num) { _dbg_assert_(bytesWritten <= MIN_BLOCK_EXIT_LEN); if (bytesWritten < len) emitter.ReserveCodeSpace(len - bytesWritten); - emitter.FlushIcache(); + // Only called from FinalizeBlock, which flushes. + AddPendingIcacheFlush(GetBasePtr() + srcOffset, GetBasePtr() + srcOffset + len); if (PlatformIsWXExclusive()) { ProtectMemoryPages(writable, 16, MEM_PROT_READ | MEM_PROT_EXEC); @@ -353,6 +372,8 @@ bool Arm64JitBackend::DescribeCodePtr(const u8 *ptr, std::string &name) const { void Arm64JitBackend::ClearAllBlocks() { ClearCodeSpace(jitStartOffset_); + // Whatever was pending is in there. + numPendingFlushes_ = 0; FlushIcacheSection(region + jitStartOffset_, region + region_size - jitStartOffset_); EraseAllLinks(-1); } diff --git a/Core/MIPS/ARM64/Arm64IRJit.h b/Core/MIPS/ARM64/Arm64IRJit.h index b02ce0a9ed..b8ff276b72 100644 --- a/Core/MIPS/ARM64/Arm64IRJit.h +++ b/Core/MIPS/ARM64/Arm64IRJit.h @@ -43,6 +43,7 @@ public: bool CompileBlock(int block_num, u32 startPC, const IRInst *instructions, int count) override; void ClearAllBlocks() override; void InvalidateBlock(int block_num) override; + void FlushPendingIcache() override; void UpdateFCR31(MIPSState *mipsState) override; @@ -134,6 +135,14 @@ private: Arm64IRRegCache regs_; Arm64Gen::ARM64FloatEmitter fp_; + // Code written since the last FlushPendingIcache(): the block, and the exits linked to and from it. + // Each flush ends with barriers, which cost more than the cache lines themselves. + void AddPendingIcacheFlush(const u8 *start, const u8 *end); + static constexpr int MAX_PENDING_ICACHE_FLUSHES = 16; + const u8 *pendingFlushStarts_[MAX_PENDING_ICACHE_FLUSHES]{}; + const u8 *pendingFlushEnds_[MAX_PENDING_ICACHE_FLUSHES]{}; + int numPendingFlushes_ = 0; + const u8 *outerLoop_ = nullptr; const u8 *outerLoopPCInSCRATCH1_ = nullptr; const u8 *dispatcherCheckCoreState_ = nullptr; diff --git a/Core/MIPS/IR/IRNativeCommon.cpp b/Core/MIPS/IR/IRNativeCommon.cpp index 7e9199c873..bb6bbcf80d 100644 --- a/Core/MIPS/IR/IRNativeCommon.cpp +++ b/Core/MIPS/IR/IRNativeCommon.cpp @@ -1045,6 +1045,7 @@ void IRNativeBackend::FinalizeBlock(int block_num, const JitOptions &jo) { OverwriteExit(blockExit.offset, blockExit.len, dstBlockNum); } } + FlushPendingIcache(); } int IRNativeBackend::GetBlockCodeStart(int block_num) const { diff --git a/Core/MIPS/IR/IRNativeCommon.h b/Core/MIPS/IR/IRNativeCommon.h index 8159738682..99c7361ca1 100644 --- a/Core/MIPS/IR/IRNativeCommon.h +++ b/Core/MIPS/IR/IRNativeCommon.h @@ -101,6 +101,9 @@ public: virtual void InvalidateBlock(int block_num) = 0; // Links the block's exits, and other blocks' exits to it. void FinalizeBlock(int block_num, const JitOptions &jo); + // For a backend that holds off the icache flushes of CompileBlock and linking, to do them together + // (at the end of FinalizeBlock). + virtual void FlushPendingIcache() {} virtual void UpdateFCR31(MIPSState *mipsState) {} diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp index 529a6c0140..75a08720da 100644 --- a/Core/MIPS/IR/IRPassSimplify.cpp +++ b/Core/MIPS/IR/IRPassSimplify.cpp @@ -97,20 +97,17 @@ bool IRApplyPasses(const IRPassFunc *passes, size_t c, const IRWriter &in, IRWri bool logBlocks = false; + // The passes take turns writing to the two, which keep their capacity. IRWriter temp[2]; const IRWriter *nextIn = ∈ - IRWriter *nextOut = &temp[1]; - temp[1].Reserve(nextIn->GetInstructions().size()); for (size_t i = 0; i < c - 1; ++i) { - if (passes[i](*nextIn, *nextOut, opts)) { + IRWriter &nextOut = temp[i & 1]; + nextOut.Clear(); + nextOut.Reserve(nextIn->GetInstructions().size()); + if (passes[i](*nextIn, nextOut, opts)) { logBlocks = true; } - - temp[0] = std::move(temp[1]); - nextIn = &temp[0]; - - temp[1].Clear(); - temp[1].Reserve(nextIn->GetInstructions().size()); + nextIn = &nextOut; } out.Reserve(nextIn->GetInstructions().size()); diff --git a/Core/MIPS/IR/IRRegCache.cpp b/Core/MIPS/IR/IRRegCache.cpp index 3792c015fc..2248c71fb0 100644 --- a/Core/MIPS/IR/IRRegCache.cpp +++ b/Core/MIPS/IR/IRRegCache.cpp @@ -743,12 +743,12 @@ void IRNativeRegCacheBase::FlushAll(bool gprs, bool fprs) { if (!mr[MIPS_REG_ZERO].isStatic && mr[MIPS_REG_ZERO].nReg != -1) DiscardNativeReg(mr[MIPS_REG_ZERO].nReg); - for (int i = 1; i < TOTAL_MAPPABLE_IRREGS; i++) { - IRReg mipsReg = (IRReg)i; + auto flushReg = [&](IRReg mipsReg) { + const int i = mipsReg; if (!fprs && i >= 32 && IsValidFPR(mipsReg - 32)) - continue; + return; if (!gprs && IsValidGPR(mipsReg)) - continue; + return; if (mr[i].isStatic) { IRNativeReg nreg = mr[i].nReg; @@ -775,6 +775,27 @@ void IRNativeRegCacheBase::FlushAll(bool gprs, bool fprs) { } else if (mr[i].loc != MIPSLoc::MEM) { FlushReg(mipsReg); } + }; + + // This runs at every exit, so it doesn't look at all 256 regs. An FPR (32-191 and 224-239) is only + // out of memory when it's in a native reg, so those are found from nr. The rest can be IMMs. + auto flushIfNeeded = [&](int first, int end) { + for (int i = first; i < end; i++) { + // The common case, nothing to do. + if (mr[i].loc == MIPSLoc::MEM && !mr[i].isStatic) + continue; + flushReg((IRReg)i); + } + }; + flushIfNeeded(1, 32); + flushIfNeeded(192, 224); + flushIfNeeded(240, TOTAL_MAPPABLE_IRREGS); + if (fprs) { + for (int n = 0; n < config_.totalNativeRegs; n++) { + const IRReg mipsReg = nr[n].mipsReg; + if (mipsReg != IRREG_INVALID && mipsReg >= 32 && IsValidFPR(mipsReg - 32)) + flushReg(mipsReg); + } } int count = 0;