32-bit: Keep the memory view range reserved, and mask guest addresses to it

Find4GBBase found 256 MB of free address space for the views and then
unmapped it, so the gaps between the views went back to the system. On a
32-bit Android device, a running game had malloc heaps between VRAM and RAM
and GPU driver buffers just past RAM, and masked guest addresses (bits 30-31
cleared) reach a whole GB above base, mostly heap. A stray guest access
landed in those silently instead of faulting, and another thread could even
map something where a view was about to go, which the view's MAP_FIXED then
replaced. That fits the 32-bit-only crashes deep in system libraries and in
our own heap structures.

Keep the range reserved as PROT_NONE and map the views over it, putting the
reservation back when a view is released. Mask out bits 28-29 too on 32-bit
(no valid address has them), so accesses stay inside the reservation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Common/MemArena.h

@@ -63,5 +63,9 @@ private:
 	size_t reserved_size = 0;
 #else
 	int fd = -1;
+	// Where Find4GBBase keeps the view range reserved (inaccessible) until ReleaseSpace, so nothing else
+	// can be mapped between the views.
+	void *reserved_mem = nullptr;
+	size_t reserved_size = 0;
 #endif
 };

Common/MemArenaAndroid.cpp

@@ -121,6 +121,11 @@ void MemArena::ReleaseSpace() {
 		close(fd);
 	}
 	fd = -1;
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+		reserved_mem = nullptr;
+		reserved_size = 0;
+	}
 }
 
 void *MemArena::CreateView(s64 offset, size_t size, void *base) {
@@ -133,7 +138,14 @@ void *MemArena::CreateView(s64 offset, size_t size, void *base) {
 }
 
 void MemArena::ReleaseView(s64 offset, void* view, size_t size) {
-	munmap(view, size);
+	const uintptr_t start = (uintptr_t)view;
+	if (reserved_mem && start >= (uintptr_t)reserved_mem && start + size <= (uintptr_t)reserved_mem + reserved_size) {
+		// Put the reservation back instead of leaving a hole, which another thread could map into before
+		// ReleaseSpace unmaps the whole range.
+		mmap(view, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED, -1, 0);
+	} else {
+		munmap(view, size);
+	}
 }
 
 u8* MemArena::Find4GBBase() {
@@ -159,14 +171,21 @@ u8* MemArena::Find4GBBase() {
 	}
 #else
 	// Address masking is used in 32-bit mode, so we can get away with less memory.
-	void *base = mmap(0, 0x10000000, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);
+	// Keep the range reserved, and map the views over it. Unmapping it would let the heap and the driver
+	// allocate in the gaps between the views (where stray guest accesses then land), or even where a
+	// view is about to go, which the view's MAP_FIXED would then silently replace.
+	void *base = mmap(0, 0x10000000, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
 
 	if (base == MAP_FAILED) {
 		ERROR_LOG(Log::System, "Failed to map 256 MB of memory space: %s", strerror(errno));
 		return nullptr;
 	}
 
-	munmap(base, 0x10000000);
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+	}
+	reserved_mem = base;
+	reserved_size = 0x10000000;
 	return static_cast<u8*>(base);
 #endif
 }

Common/MemArenaPosix.cpp

@@ -110,6 +110,11 @@ bool MemArena::GrabMemSpace(size_t size) {
 void MemArena::ReleaseSpace() {
 #ifndef NO_MMAP
 	close(fd);
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+		reserved_mem = nullptr;
+		reserved_size = 0;
+	}
 #endif
 }
 
@@ -135,7 +140,14 @@ void *MemArena::CreateView(s64 offset, size_t size, void *base)
 
 void MemArena::ReleaseView(s64 offset, void* view, size_t size) {
 #ifndef NO_MMAP
-	munmap(view, size);
+	const uintptr_t start = (uintptr_t)view;
+	if (reserved_mem && start >= (uintptr_t)reserved_mem && start + size <= (uintptr_t)reserved_mem + reserved_size) {
+		// Put the reservation back instead of leaving a hole, which another thread could map into before
+		// ReleaseSpace unmaps the whole range.
+		mmap(view, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED, -1, 0);
+	} else {
+		munmap(view, size);
+	}
 #endif
 }
 
@@ -165,10 +177,16 @@ u8* MemArena::Find4GBBase() {
     void* base = std::malloc(0x0A000000);
     return static_cast<u8*>(base);
 #else
+	// Keep the range reserved and map the views over it, so nothing else can be allocated in the gaps
+	// between them, or where a view is about to go (see MemArenaAndroid.cpp).
 	size_t size = 0x10000000;
-	void* base = mmap(0, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED | MAP_NORESERVE, -1, 0);
+	void* base = mmap(0, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
 	_assert_msg_(base != MAP_FAILED, "Failed to map 256 MB of memory space: %s", strerror(errno));
-	munmap(base, size);
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+	}
+	reserved_mem = base;
+	reserved_size = size;
 	return static_cast<u8*>(base);
 #endif
 }

Core/MIPS/ARM/ArmIRCompLoadStore.cpp

@@ -46,11 +46,12 @@ void ArmJitBackend::PrepareSrc1Address(IRInst inst) {
 	}
 
 	ARMReg src1 = regs_.MapGPR(inst.src1);
+	// Clears bits 28-31 (see MEMVIEW32_MASK).
 	if (inst.constant == 0) {
-		BIC(SCRATCH1, src1, Operand2(0xC0, 4));
+		BIC(SCRATCH1, src1, Operand2(0xF0, 4));
 	} else {
 		ADDI2R(SCRATCH1, src1, inst.constant, SCRATCH2);
-		BIC(SCRATCH1, SCRATCH1, Operand2(0xC0, 4));
+		BIC(SCRATCH1, SCRATCH1, Operand2(0xF0, 4));
 	}
 }
 

Core/MemMap.h

@@ -108,7 +108,13 @@ enum {
 
 #ifdef MASKED_PSP_MEMORY
 	// This wraparound should work for PSP too.
+#if PPSSPP_ARCH(32BIT)
+	// 32-bit only reserves 256 MB at base. Valid addresses never have bits 28-29 set, so dropping them
+	// too keeps stray accesses inside the reservation, away from the heap.
+	MEMVIEW32_MASK  = 0x0FFFFFFF,
+#else
 	MEMVIEW32_MASK  = 0x3FFFFFFF,
+#endif
 #endif
 
 	// Index mask for the block shadow (see blockShadow below). Dropping bits 30-31 folds the