diff --git a/Core/MIPS/ARM64/Arm64RegCache.cpp b/Core/MIPS/ARM64/Arm64RegCache.cpp
index ac97e33e74..3bfede4dbb 100644
--- a/Core/MIPS/ARM64/Arm64RegCache.cpp
+++ b/Core/MIPS/ARM64/Arm64RegCache.cpp
@@ -777,7 +777,7 @@ void Arm64RegCache::FlushAll() {
 			// Re-pointerify
 			emit_->MOVK(EncodeRegTo64(allocs[i].ar), ((uint64_t)Memory::base) >> 32, SHIFT_32);
 			ar[allocs[i].ar].pointerified = true;
-		} else {
+		} else if (!allocs[i].pointerified) {
 			// If this register got pointerified on the way, mark it as not, so that after save/reload (like in an interpreter fallback), it won't be regarded as such, as it simply won't be.
 			ar[allocs[i].ar].pointerified = false;
 		}
diff --git a/Core/MIPS/RiscV/RiscVRegCache.cpp b/Core/MIPS/RiscV/RiscVRegCache.cpp
index cce041d9f1..ff88d5c2c7 100644
--- a/Core/MIPS/RiscV/RiscVRegCache.cpp
+++ b/Core/MIPS/RiscV/RiscVRegCache.cpp
@@ -795,9 +795,10 @@ void RiscVRegCache::FlushAll() {
 	for (int i = 0; i < count; i++) {
 		if (allocs[i].pointerified && !ar[allocs[i].ar].pointerified && jo_->enablePointerify) {
 			// Re-pointerify
+			_dbg_assert_(mr[allocs[i].mr].loc == MIPSLoc::RVREG);
 			AddMemBase(allocs[i].ar);
 			ar[allocs[i].ar].pointerified = true;
-		} else {
+		} else if (!allocs[i].pointerified) {
 			// If this register got pointerified on the way, mark it as not.
 			// This is so that after save/reload (like in an interpreter fallback),
 			// it won't be regarded as such, as it may no longer be.