diff --git a/Core/MIPS/IR/IRPassSimplify.cpp b/Core/MIPS/IR/IRPassSimplify.cpp
index b05565b457..b74d18722c 100644
--- a/Core/MIPS/IR/IRPassSimplify.cpp
+++ b/Core/MIPS/IR/IRPassSimplify.cpp
@@ -1994,6 +1994,18 @@ bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts) {
 	if (!opts.preferVec4 || opts.optimizeForInterpreter) {
 		DISABLE;
 	}
+	// Everything below starts from a Vec4 (or matrix) operand, which most blocks don't have.
+	bool anyVec4 = false;
+	for (const IRInst &inst : in.GetInstructions()) {
+		const IRMeta *m = GetIRMeta(inst.op);
+		for (int t = 0; t < 3; ++t) {
+			if (m->types[t] == 'V' || m->types[t] == 'M')
+				anyVec4 = true;
+		}
+	}
+	if (!anyVec4) {
+		DISABLE;
+	}
 
 	bool isVec4[256]{};
 	bool isUsed[256]{};
diff --git a/Core/MIPS/IR/IRRegCache.cpp b/Core/MIPS/IR/IRRegCache.cpp
index 9580b2d361..943a74f93c 100644
--- a/Core/MIPS/IR/IRRegCache.cpp
+++ b/Core/MIPS/IR/IRRegCache.cpp
@@ -20,6 +20,7 @@
 #endif
 
 #include <cstring>
+#include "Common/BitSet.h"
 #include "Common/Log.h"
 #include "Common/LogReporting.h"
 #include "Core/MemMap.h"
@@ -30,10 +31,10 @@
 #include "Core/MIPS/JitCommon/JitState.h"
 
 void IRImmRegCache::Flush(IRReg rd) {
-	if (isImm_[rd] && !isWritten_[rd]) {
+	if (TestBit(isImm_, rd) && !TestBit(isWritten_, rd)) {
 		_assert_((rd > 0 && rd < 32) || (rd >= IRTEMP_0 && rd < IRREG_VFPU_CTRL_BASE));
 		ir_->WriteSetConstant(rd, immVal_[rd]);
-		isWritten_[rd] = true;
+		SetBit(isWritten_, rd);
 	}
 }
 
@@ -41,31 +42,29 @@ void IRImmRegCache::Discard(IRReg rd) {
 	if (rd == 0) {
 		return;
 	}
-	isImm_[rd] = false;
+	ClearBit(isImm_, rd);
 }
 
 IRImmRegCache::IRImmRegCache(IRWriter *ir) : ir_(ir) {
 	memset(&isImm_, 0, sizeof(isImm_));
 	memset(&isWritten_, 0, sizeof(isWritten_));
 	memset(&immVal_, 0, sizeof(immVal_));
-	isImm_[0] = true;
-	isWritten_[0] = true;
+	SetBit(isImm_, 0);
+	SetBit(isWritten_, 0);
 	ir_ = ir;
 }
 
 void IRImmRegCache::FlushAll(bool keepKnown) {
-	for (int i = 1; i < TOTAL_MAPPABLE_IRREGS; ) {
-		if (isImm_[i]) {
-			Flush(i);
+	for (int w = 0; w < MASK_WORDS; w++) {
+		// The zero reg stays known.
+		u32 bits = w == 0 ? isImm_[0] & ~1U : isImm_[w];
+		while (bits != 0) {
+			const IRReg r = (IRReg)(w * 32 + LeastSignificantSetBit(bits));
+			bits &= bits - 1;
+			Flush(r);
 			if (!keepKnown)
-				isImm_[i] = false;
+				ClearBit(isImm_, r);
 		}
-
-		// Most of the time, lots are not.  This speeds it up a lot.
-		bool *next = (bool *)memchr(&isImm_[i + 1], 1, TOTAL_MAPPABLE_IRREGS - i - 1);
-		if (!next)
-			break;
-		i = (int)(next - &isImm_[0]);
 	}
 }
 
diff --git a/Core/MIPS/IR/IRRegCache.h b/Core/MIPS/IR/IRRegCache.h
index 670938e93d..f4546e975b 100644
--- a/Core/MIPS/IR/IRRegCache.h
+++ b/Core/MIPS/IR/IRRegCache.h
@@ -48,13 +48,13 @@ public:
 
 	void SetImm(IRReg r, u32 immVal) {
 		// Setting the value it already holds doesn't need writing again.
-		if (!isImm_[r] || immVal_[r] != immVal)
-			isWritten_[r] = false;
-		isImm_[r] = true;
+		if (!IsImm(r) || immVal_[r] != immVal)
+			ClearBit(isWritten_, r);
+		SetBit(isImm_, r);
 		immVal_[r] = immVal;
 	}
 
-	bool IsImm(IRReg r) const { return isImm_[r]; }
+	bool IsImm(IRReg r) const { return TestBit(isImm_, r); }
 	u32 GetImm(IRReg r) const { return immVal_[r]; }
 
 	// Writes out all pending values. With keepKnown, the values stay known afterward (for a
@@ -74,9 +74,15 @@ private:
 	void Flush(IRReg rd);
 	void Discard(IRReg rd);
 
-	bool isImm_[TOTAL_MAPPABLE_IRREGS];
+	// Bit masks, one bit per reg: cheap to clear for each block, and FlushAll finds the set ones quickly.
+	static constexpr int MASK_WORDS = TOTAL_MAPPABLE_IRREGS / 32;
+	static bool TestBit(const u32 *mask, IRReg r) { return (mask[r >> 5] >> (r & 31)) & 1; }
+	static void SetBit(u32 *mask, IRReg r) { mask[r >> 5] |= 1U << (r & 31); }
+	static void ClearBit(u32 *mask, IRReg r) { mask[r >> 5] &= ~(1U << (r & 31)); }
+
+	u32 isImm_[MASK_WORDS];
 	// Whether the known value has been written to the reg.
-	bool isWritten_[TOTAL_MAPPABLE_IRREGS];
+	u32 isWritten_[MASK_WORDS];
 	uint32_t immVal_[TOTAL_MAPPABLE_IRREGS];
 	IRWriter *ir_;
 };
@@ -101,7 +107,7 @@ struct IRInst;
 
 class IRNativeRegCacheBase {
 protected:
-	enum class MIPSLoc {
+	enum class MIPSLoc : uint8_t {
 		// Known immediate value (only in regcache.)
 		IMM,
 		// In a general reg.
@@ -118,20 +124,22 @@ protected:
 		MEM,
 	};
 
+	// Kept small, since Start() copies all 256 for each block.
 	struct RegStatusMIPS {
 		// Where is this IR/MIPS register?  Note: base reg if vector.
 		MIPSLoc loc = MIPSLoc::MEM;
 		// If in a register, what index (into nr array)?
 		IRNativeReg nReg = -1;
+		// If in a multipart reg (vector or HI/LO), which lane?
+		int8_t lane = -1;
+		// Whether this reg is statically allocated.
+		bool isStatic = false;
 		// If a known immediate value, what value?
 		uint32_t imm = 0;
 		// Locked from spilling (i.e. used by current instruction) as of what IR instruction?
 		int spillLockIRIndex = -1;
-		// If in a multipart reg (vector or HI/LO), which lane?
-		int lane = -1;
-		// Whether this reg is statically allocated.
-		bool isStatic = false;
 	};
+	static_assert(sizeof(RegStatusMIPS) == 12, "RegStatusMIPS grew");
 	struct RegStatusNative {
 		// Which IR/MIPS reg is this currently holding?
 		IRReg mipsReg = IRREG_INVALID;