32-bit: Keep the memory view range reserved, and mask guest addresses to it

Find4GBBase found 256 MB of free address space for the views and then
unmapped it, so the gaps between the views went back to the system. On a
32-bit Android device, a running game had malloc heaps between VRAM and RAM
and GPU driver buffers just past RAM, and masked guest addresses (bits 30-31
cleared) reach a whole GB above base, mostly heap. A stray guest access
landed in those silently instead of faulting, and another thread could even
map something where a view was about to go, which the view's MAP_FIXED then
replaced. That fits the 32-bit-only crashes deep in system libraries and in
our own heap structures.

Keep the range reserved as PROT_NONE and map the views over it, putting the
reservation back when a view is released. Mask out bits 28-29 too on 32-bit
(no valid address has them), so accesses stay inside the reservation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Common/MemArenaAndroid.cpp

@@ -121,6 +121,11 @@ void MemArena::ReleaseSpace() {
 		close(fd);
 	}
 	fd = -1;
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+		reserved_mem = nullptr;
+		reserved_size = 0;
+	}
 }
 
 void *MemArena::CreateView(s64 offset, size_t size, void *base) {
@@ -133,7 +138,14 @@ void *MemArena::CreateView(s64 offset, size_t size, void *base) {
 }
 
 void MemArena::ReleaseView(s64 offset, void* view, size_t size) {
-	munmap(view, size);
+	const uintptr_t start = (uintptr_t)view;
+	if (reserved_mem && start >= (uintptr_t)reserved_mem && start + size <= (uintptr_t)reserved_mem + reserved_size) {
+		// Put the reservation back instead of leaving a hole, which another thread could map into before
+		// ReleaseSpace unmaps the whole range.
+		mmap(view, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED, -1, 0);
+	} else {
+		munmap(view, size);
+	}
 }
 
 u8* MemArena::Find4GBBase() {
@@ -159,14 +171,21 @@ u8* MemArena::Find4GBBase() {
 	}
 #else
 	// Address masking is used in 32-bit mode, so we can get away with less memory.
-	void *base = mmap(0, 0x10000000, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);
+	// Keep the range reserved, and map the views over it. Unmapping it would let the heap and the driver
+	// allocate in the gaps between the views (where stray guest accesses then land), or even where a
+	// view is about to go, which the view's MAP_FIXED would then silently replace.
+	void *base = mmap(0, 0x10000000, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
 
 	if (base == MAP_FAILED) {
 		ERROR_LOG(Log::System, "Failed to map 256 MB of memory space: %s", strerror(errno));
 		return nullptr;
 	}
 
-	munmap(base, 0x10000000);
+	if (reserved_mem) {
+		munmap(reserved_mem, reserved_size);
+	}
+	reserved_mem = base;
+	reserved_size = 0x10000000;
 	return static_cast<u8*>(base);
 #endif
 }