IR JIT: Compile faster

Measured with --jit-bench on a Galaxy S8, recompiling the 11243 blocks of a
Tekken 6 state: 363 ms -> 300-310 ms, with the ARM64 pairing fix.

- FlushAll, which runs at every exit, no longer looks at all 256 regs. FPRs are
  found from the native regs, and the rest are skipped quickly when in memory.
- IRApplyPasses alternates between two buffers that keep their capacity,
  instead of allocating a new one for each pass.
- The ARM64 backend flushes the icache once per block, for the block's code and
  every exit linked to and from it, with one set of barriers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Common/Arm64Emitter.cpp

@@ -329,13 +329,25 @@ void ARM64XEmitter::FlushIcache()
 	m_lastCacheFlushEnd = m_code;
 }
 
-void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end)
+void ARM64XEmitter::TakeIcacheFlushRange(const u8 **start, const u8 **end) {
+	*start = m_lastCacheFlushEnd;
+	*end = m_code;
+	m_lastCacheFlushEnd = m_code;
+}
+
+void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end) {
+	FlushIcacheSections(&start, &end, 1);
+}
+
+void ARM64XEmitter::FlushIcacheSections(const u8 *const *starts, const u8 *const *ends, int count)
 {
 #if PPSSPP_PLATFORM(IOS) || PPSSPP_PLATFORM(MAC)
 	// Header file says this is equivalent to: sys_icache_invalidate(start, end - start);
-	sys_cache_control(kCacheFunctionPrepareForExecution, (void *)start, end - start);
+	for (int i = 0; i < count; i++)
+		sys_cache_control(kCacheFunctionPrepareForExecution, (void *)starts[i], ends[i] - starts[i]);
 #elif PPSSPP_PLATFORM(WINDOWS)
-	FlushInstructionCache(GetCurrentProcess(), start, end - start);
+	for (int i = 0; i < count; i++)
+		FlushInstructionCache(GetCurrentProcess(), starts[i], ends[i] - starts[i]);
 #elif PPSSPP_ARCH(ARM64)
 	// Code from Dolphin, contributed by the Mono project.
 
@@ -359,16 +371,20 @@ void ARM64XEmitter::FlushIcacheSection(const u8 *start, const u8 *end)
 		dcache_line_size = dsize = dcache_line_size < dsize ? dcache_line_size : dsize;
 	}
 
-	u64 addr = (u64)start & ~(u64)(dsize - 1);
-	for (; addr < (u64)end; addr += dsize)
-		// use "civac" instead of "cvau", as this is the suggested workaround for
-		// Cortex-A53 errata 819472, 826319, 827319 and 824069.
-		__asm__ volatile("dc civac, %0" : : "r"(addr) : "memory");
+	for (int i = 0; i < count; i++) {
+		u64 addr = (u64)starts[i] & ~(u64)(dsize - 1);
+		for (; addr < (u64)ends[i]; addr += dsize)
+			// use "civac" instead of "cvau", as this is the suggested workaround for
+			// Cortex-A53 errata 819472, 826319, 827319 and 824069.
+			__asm__ volatile("dc civac, %0" : : "r"(addr) : "memory");
+	}
 	__asm__ volatile("dsb ish" : : : "memory");
 
-	addr = (u64)start & ~(u64)(isize - 1);
-	for (; addr < (u64)end; addr += isize)
-		__asm__ volatile("ic ivau, %0" : : "r"(addr) : "memory");
+	for (int i = 0; i < count; i++) {
+		u64 addr = (u64)starts[i] & ~(u64)(isize - 1);
+		for (; addr < (u64)ends[i]; addr += isize)
+			__asm__ volatile("ic ivau, %0" : : "r"(addr) : "memory");
+	}
 
 	__asm__ volatile("dsb ish" : : : "memory");
 	__asm__ volatile("isb" : : : "memory");

Common/Arm64Emitter.h

@@ -421,6 +421,10 @@ public:
 	const u8 *NopAlignCode16();
 	void FlushIcache();
 	void FlushIcacheSection(const u8* start, const u8* end);
+	// Instead of FlushIcache(): the range it would flush, now counted as flushed. For flushing with others.
+	void TakeIcacheFlushRange(const u8 **start, const u8 **end);
+	// Like FlushIcacheSection() on each, with one set of barriers.
+	static void FlushIcacheSections(const u8 *const *starts, const u8 *const *ends, int count);
 	u8* GetWritableCodePtr();
 
 	// FixupBranch branching

Core/MIPS/ARM64/Arm64IRJit.cpp

@@ -213,12 +213,30 @@ bool Arm64JitBackend::CompileBlock(int block_num, u32 startPC, const IRInst *ins
 	}
 
 	EndWrite();
-	FlushIcache();
+	// Flushed with the links, after FinalizeBlock.
+	const u8 *flushStart, *flushEnd;
+	TakeIcacheFlushRange(&flushStart, &flushEnd);
+	AddPendingIcacheFlush(flushStart, flushEnd);
 	compilingBlockNum_ = -1;
 
 	return true;
 }
 
+void Arm64JitBackend::AddPendingIcacheFlush(const u8 *start, const u8 *end) {
+	if (numPendingFlushes_ == MAX_PENDING_ICACHE_FLUSHES)
+		FlushPendingIcache();
+	pendingFlushStarts_[numPendingFlushes_] = start;
+	pendingFlushEnds_[numPendingFlushes_] = end;
+	numPendingFlushes_++;
+}
+
+void Arm64JitBackend::FlushPendingIcache() {
+	if (numPendingFlushes_ == 0)
+		return;
+	FlushIcacheSections(pendingFlushStarts_, pendingFlushEnds_, numPendingFlushes_);
+	numPendingFlushes_ = 0;
+}
+
 void Arm64JitBackend::WriteConstExit(uint32_t pc) {
 	int block_num = FindBlockByStart(pc);
 	const IRNativeBlock *nativeBlock = GetNativeBlock(block_num);
@@ -259,7 +277,8 @@ void Arm64JitBackend::OverwriteExit(int srcOffset, int len, int block_num) {
 		_dbg_assert_(bytesWritten <= MIN_BLOCK_EXIT_LEN);
 		if (bytesWritten < len)
 			emitter.ReserveCodeSpace(len - bytesWritten);
-		emitter.FlushIcache();
+		// Only called from FinalizeBlock, which flushes.
+		AddPendingIcacheFlush(GetBasePtr() + srcOffset, GetBasePtr() + srcOffset + len);
 
 		if (PlatformIsWXExclusive()) {
 			ProtectMemoryPages(writable, 16, MEM_PROT_READ | MEM_PROT_EXEC);
@@ -353,6 +372,8 @@ bool Arm64JitBackend::DescribeCodePtr(const u8 *ptr, std::string &name) const {
 
 void Arm64JitBackend::ClearAllBlocks() {
 	ClearCodeSpace(jitStartOffset_);
+	// Whatever was pending is in there.
+	numPendingFlushes_ = 0;
 	FlushIcacheSection(region + jitStartOffset_, region + region_size - jitStartOffset_);
 	EraseAllLinks(-1);
 }

Core/MIPS/ARM64/Arm64IRJit.h

@@ -43,6 +43,7 @@ public:
 	bool CompileBlock(int block_num, u32 startPC, const IRInst *instructions, int count) override;
 	void ClearAllBlocks() override;
 	void InvalidateBlock(int block_num) override;
+	void FlushPendingIcache() override;
 
 	void UpdateFCR31(MIPSState *mipsState) override;
 
@@ -134,6 +135,14 @@ private:
 	Arm64IRRegCache regs_;
 	Arm64Gen::ARM64FloatEmitter fp_;
 
+	// Code written since the last FlushPendingIcache(): the block, and the exits linked to and from it.
+	// Each flush ends with barriers, which cost more than the cache lines themselves.
+	void AddPendingIcacheFlush(const u8 *start, const u8 *end);
+	static constexpr int MAX_PENDING_ICACHE_FLUSHES = 16;
+	const u8 *pendingFlushStarts_[MAX_PENDING_ICACHE_FLUSHES]{};
+	const u8 *pendingFlushEnds_[MAX_PENDING_ICACHE_FLUSHES]{};
+	int numPendingFlushes_ = 0;
+
 	const u8 *outerLoop_ = nullptr;
 	const u8 *outerLoopPCInSCRATCH1_ = nullptr;
 	const u8 *dispatcherCheckCoreState_ = nullptr;

Core/MIPS/IR/IRNativeCommon.cpp

@@ -1045,6 +1045,7 @@ void IRNativeBackend::FinalizeBlock(int block_num, const JitOptions &jo) {
 				OverwriteExit(blockExit.offset, blockExit.len, dstBlockNum);
 		}
 	}
+	FlushPendingIcache();
 }
 
 int IRNativeBackend::GetBlockCodeStart(int block_num) const {

Core/MIPS/IR/IRNativeCommon.h

@@ -101,6 +101,9 @@ public:
 	virtual void InvalidateBlock(int block_num) = 0;
 	// Links the block's exits, and other blocks' exits to it.
 	void FinalizeBlock(int block_num, const JitOptions &jo);
+	// For a backend that holds off the icache flushes of CompileBlock and linking, to do them together
+	// (at the end of FinalizeBlock).
+	virtual void FlushPendingIcache() {}
 
 	virtual void UpdateFCR31(MIPSState *mipsState) {}
 

Core/MIPS/IR/IRPassSimplify.cpp

@@ -97,20 +97,17 @@ bool IRApplyPasses(const IRPassFunc *passes, size_t c, const IRWriter &in, IRWri
 
 	bool logBlocks = false;
 
+	// The passes take turns writing to the two, which keep their capacity.
 	IRWriter temp[2];
 	const IRWriter *nextIn = ∈
-	IRWriter *nextOut = &temp[1];
-	temp[1].Reserve(nextIn->GetInstructions().size());
 	for (size_t i = 0; i < c - 1; ++i) {
-		if (passes[i](*nextIn, *nextOut, opts)) {
+		IRWriter &nextOut = temp[i & 1];
+		nextOut.Clear();
+		nextOut.Reserve(nextIn->GetInstructions().size());
+		if (passes[i](*nextIn, nextOut, opts)) {
 			logBlocks = true;
 		}
-
-		temp[0] = std::move(temp[1]);
-		nextIn = &temp[0];
-
-		temp[1].Clear();
-		temp[1].Reserve(nextIn->GetInstructions().size());
+		nextIn = &nextOut;
 	}
 
 	out.Reserve(nextIn->GetInstructions().size());

Core/MIPS/IR/IRRegCache.cpp

@@ -743,12 +743,12 @@ void IRNativeRegCacheBase::FlushAll(bool gprs, bool fprs) {
 	if (!mr[MIPS_REG_ZERO].isStatic && mr[MIPS_REG_ZERO].nReg != -1)
 		DiscardNativeReg(mr[MIPS_REG_ZERO].nReg);
 
-	for (int i = 1; i < TOTAL_MAPPABLE_IRREGS; i++) {
-		IRReg mipsReg = (IRReg)i;
+	auto flushReg = [&](IRReg mipsReg) {
+		const int i = mipsReg;
 		if (!fprs && i >= 32 && IsValidFPR(mipsReg - 32))
-			continue;
+			return;
 		if (!gprs && IsValidGPR(mipsReg))
-			continue;
+			return;
 
 		if (mr[i].isStatic) {
 			IRNativeReg nreg = mr[i].nReg;
@@ -775,6 +775,27 @@ void IRNativeRegCacheBase::FlushAll(bool gprs, bool fprs) {
 		} else if (mr[i].loc != MIPSLoc::MEM) {
 			FlushReg(mipsReg);
 		}
+	};
+
+	// This runs at every exit, so it doesn't look at all 256 regs. An FPR (32-191 and 224-239) is only
+	// out of memory when it's in a native reg, so those are found from nr. The rest can be IMMs.
+	auto flushIfNeeded = [&](int first, int end) {
+		for (int i = first; i < end; i++) {
+			// The common case, nothing to do.
+			if (mr[i].loc == MIPSLoc::MEM && !mr[i].isStatic)
+				continue;
+			flushReg((IRReg)i);
+		}
+	};
+	flushIfNeeded(1, 32);
+	flushIfNeeded(192, 224);
+	flushIfNeeded(240, TOTAL_MAPPABLE_IRREGS);
+	if (fprs) {
+		for (int n = 0; n < config_.totalNativeRegs; n++) {
+			const IRReg mipsReg = nr[n].mipsReg;
+			if (mipsReg != IRREG_INVALID && mipsReg >= 32 && IsValidFPR(mipsReg - 32))
+				flushReg(mipsReg);
+		}
 	}
 
 	int count = 0;