ppsspp/Core/MemMap.cpp

// Copyright (C) 2003 Dolphin Project / 2012 PPSSPP Project

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License 2.0 for more details.

// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/

// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.

#include "ppsspp_config.h"

#if PPSSPP_PLATFORM(UWP)
#include "Common/CommonWindows.h"
#endif

#include <algorithm>
#include <mutex>

#include "Common/CommonTypes.h"
#include "Common/MemArena.h"
#include "Common/MemoryUtil.h"
#include "Common/Serialize/Serializer.h"
#include "Common/Serialize/SerializeFuncs.h"

#include "Core/System.h"
#include "Core/Core.h"
#include "Core/ConfigValues.h"
#include "Core/Debugger/MemBlockInfo.h"
#include "Core/HDRemaster.h"
#include "Core/HLE/ReplaceTables.h"
#include "Core/MemMap.h"
#include "Core/MemFault.h"
#include "Core/MIPS/MIPS.h"
#include "Core/MIPS/JitCommon/JitCommon.h"
#include "Common/Thread/ParallelLoop.h"

namespace Memory {

// The base pointer to the auto-mirrored arena.
u8* base = nullptr;

u8 *blockShadow = nullptr;
AddressSpace g_mainSpace;
bool g_kernelModeAnywhere = false;
static const size_t BLOCK_SHADOW_SIZE = (size_t)BLOCK_SHADOW_MASK + 1;

// The MemArena class
MemArena g_arena;
// ==============

u8 *m_pNullPage;
u8 *m_pPhysicalScratchPad;
u8 *m_pUncachedScratchPad;
u8 *m_pKernelScratchPad;
u8 *m_pUncachedKernelScratchPad;
// 64-bit: Pointers to high-mem mirrors
// 32-bit: Same as above
u8 *m_pPhysicalRAM[3];
u8 *m_pUncachedRAM[3];
u8 *m_pKernelRAM[3];	// RAM mirrored up to "kernel space". Fully accessible at all times currently.
// Technically starts at 0xA0000000, which we don't properly support (but we don't really support kernel code.)
// This matches how we handle 32-bit masking.
u8 *m_pUncachedKernelRAM[3];

// VRAM is mirrored 4 times.  The second and fourth mirrors are swizzled, so actually it's not correct
// to mirror them like we do here unfortunately.
// In practice, a game accessing the mirrors most likely is deswizzling the depth buffer, and things mostly work out
// since when we write to the depth buffer (like with the depth rasterizer or the software renderer)
// we write unswizzled data anyway. There are some exceptions, Silent Hill abuses the swizzling in ways that break
// our software renderer.
static u8 *m_pPhysicalVRAM[4];
static u8 *m_pUncachedVRAM[4];

static u8 *m_pBootPage;
// Only mapped in the Media Engine's space, so these stay null.
static u8 *m_pMEEdram[3];
static u8 *m_pMEUncachedRAM[3];
static u8 *m_pMELocal[12];
// Where masking leaves no room to map the boot page. There's no Media Engine there to share it with.
static u8 g_unmappedBootPage[BOOT_PAGE_SIZE];

// Hardware registers are mapped at 0xbc000000 and forwards (physical address 0x1C000000, but we need
// the 0x80000000 kernel flag and 0x40000000 uncached flag for these). Exception vectors are at 0xbfc00000.
// Since we do HLE emulation, we currently don't bother with any of that.
// Some limited documentation here: https://www.psdevwiki.com/psp/Hardware_Registers#Introduction

// Holds the ending address of the PSP's user space.
// Required for HD Remasters to work properly.
// This replaces RAM_NORMAL_SIZE at runtime.
u32 g_MemorySize;
int g_depthTranslationIndex = DepthTranslationIndex(0x400);

bool DepthMirrorsActive() {
	return PSP_CoreParameter().gpuCore == GPUCORE_SOFTWARE;
}

u32 DepthMirroredSlow(u32 address) {
	if (!DepthMirrorsActive())
		return address;
	const DepthLayout &layout = DEPTH_LAYOUTS[(address >> 22) & 1][g_depthTranslationIndex];
	return (address & 0x40000000) | 0x04000000 | layout.Stored(address & 0x001FFFFF);
}
// Used to store the PSP model on game startup.
u32 g_PSPModel;
u32 g_UserPartitionSize;

static MemMapSetupFlags g_setupFlags;


// We don't declare the IO region in here since its handled by other means (MMIO), except for the
// boot page, which MMIO reads and writes through its view.
static MemoryView views[] = {
	{&m_pNullPage,            0x00000000, 0x00010000, MV_NULL_PAGE | MV_MAIN_CPU_ONLY}, // Null page, usually not enabled. Only used for working around some race condition bugs.
	{&m_pPhysicalScratchPad,  0x00010000, SCRATCHPAD_SIZE, MV_MAIN_CPU_ONLY},
	{&m_pUncachedScratchPad,  0x40010000, SCRATCHPAD_SIZE, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	// Kernel-mode code (e.g. flash0:/reboot.bin) sees the scratchpad through these mirrors -
	// same two address bits as RAM below (0x80000000 = kernel, 0x40000000 = uncached,
	// independently combinable), just missing here until this was noticed via a real SIGSEGV
	// writing 0x80010000 (see docs/VSHBootInvestigation.md).
	{&m_pKernelScratchPad,        0x80010000, SCRATCHPAD_SIZE, MV_MIRROR_PREVIOUS | MV_KERNEL | MV_MAIN_CPU_ONLY},
	{&m_pUncachedKernelScratchPad,0xC0010000, SCRATCHPAD_SIZE, MV_MIRROR_PREVIOUS | MV_KERNEL | MV_MAIN_CPU_ONLY},
	{&m_pPhysicalVRAM[0],     0x04000000, 0x00200000, MV_MAIN_CPU_ONLY},
	{&m_pPhysicalVRAM[1],     0x04200000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pPhysicalVRAM[2],     0x04400000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pPhysicalVRAM[3],     0x04600000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pUncachedVRAM[0],     0x44000000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pUncachedVRAM[1],     0x44200000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pUncachedVRAM[2],     0x44400000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	{&m_pUncachedVRAM[3],     0x44600000, 0x00200000, MV_MIRROR_PREVIOUS | MV_MAIN_CPU_ONLY},
	// The Media Engine doesn't see VRAM: at its addresses it has memory of its own, separate from VRAM
	// through 0x44000000 and 0xA4000000 alike (pspautotest me/vram). mcidclan's libraries keep the VME's
	// buffers and configuration there. Its size isn't known; this covers what VRAM's views do.
	{&m_pMELocal[0],          0x04000000, 0x00200000, MV_ME_ONLY | MV_ME_LOCAL},
	{&m_pMELocal[1],          0x04200000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[2],          0x04400000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[3],          0x04600000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[4],          0x44000000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[5],          0x44200000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[6],          0x44400000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[7],          0x44600000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[8],          0xA4000000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[9],          0xA4200000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[10],         0xA4400000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pMELocal[11],         0xA4600000, 0x00200000, MV_MIRROR_PREVIOUS | MV_ME_ONLY},
	{&m_pPhysicalRAM[0],      0x08000000, g_MemorySize, MV_IS_PRIMARY_RAM},	// only from 0x08800000 is it usable (last 24 megs)
	{&m_pUncachedRAM[0],      0x48000000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_PRIMARY_RAM},
	{&m_pKernelRAM[0],        0x88000000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_PRIMARY_RAM | MV_KERNEL},
	{&m_pUncachedKernelRAM[0],0xC8000000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_PRIMARY_RAM | MV_KERNEL},
	// The ME also reaches physical memory uncached through 0xA0000000.
	{&m_pMEUncachedRAM[0],    0xA8000000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_PRIMARY_RAM | MV_ME_ONLY},
	// Starts at memory + 31 MB.
	{&m_pPhysicalRAM[1],      0x09F00000, g_MemorySize, MV_IS_EXTRA1_RAM},
	{&m_pUncachedRAM[1],      0x49F00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA1_RAM},
	{&m_pKernelRAM[1],        0x89F00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA1_RAM | MV_KERNEL},
	{&m_pUncachedKernelRAM[1],0xC9F00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA1_RAM | MV_KERNEL},
	{&m_pMEUncachedRAM[1],    0xA9F00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA1_RAM | MV_ME_ONLY},
	// Starts at memory + 31 * 2 MB.
	{&m_pPhysicalRAM[2],      0x0BE00000, g_MemorySize, MV_IS_EXTRA2_RAM},
	{&m_pUncachedRAM[2],      0x4BE00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA2_RAM},
	{&m_pKernelRAM[2],        0x8BE00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA2_RAM | MV_KERNEL},
	{&m_pUncachedKernelRAM[2],0xCBE00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA2_RAM | MV_KERNEL},
	{&m_pMEUncachedRAM[2],    0xABE00000, g_MemorySize, MV_MIRROR_PREVIOUS | MV_IS_EXTRA2_RAM | MV_ME_ONLY},

	// Sized for the model in Init().
	{&m_pMEEdram[0],          0x00000000, 0, MV_ME_ONLY | MV_EDRAM},
	{&m_pMEEdram[1],          0x80000000, 0, MV_MIRROR_PREVIOUS | MV_ME_ONLY | MV_EDRAM},
	{&m_pMEEdram[2],          0xA0000000, 0, MV_MIRROR_PREVIOUS | MV_ME_ONLY | MV_EDRAM},

	// Last, so the views before keep their offsets aligned to the host's pages.
	{&m_pBootPage,            BOOT_PAGE_ADDRESS, BOOT_PAGE_SIZE, MV_BOOT_PAGE},

	// TODO: There are a few swizzled mirrors of VRAM, not sure about the best way to
	// implement those.
};

// Where each view is in g_arena, for mapping the same memory into the Media Engine's space.
static size_t viewPositions[ARRAY_SIZE(views)];

inline static bool CanIgnoreView(const MemoryView &view) {
#ifdef MASKED_PSP_MEMORY
	// Basically, 32-bit platforms can ignore views that are masked out anyway.
	return (view.flags & MV_MIRROR_PREVIOUS) && (view.virtual_address & ~MEMVIEW32_MASK) != 0;
#else
	return false;
#endif
}

static bool SkipView(MemMapSetupFlags flags, u32 viewFlags) {
#if PPSSPP_PLATFORM(IOS) && PPSSPP_ARCH(64BIT)
	// We use a limited memory map on iOS with masking, we don't need to allocate the kernel space views.
	if (viewFlags & MV_KERNEL) {
		return true;
	}
#endif
	if ((viewFlags & MV_NULL_PAGE) && !(flags & MemMapSetupFlags::AllocNullPage))
		return true;
#ifdef MASKED_PSP_MEMORY
	// Far above everything else, where a masked map has nothing reserved. And no Media Engine.
	if (viewFlags & (MV_BOOT_PAGE | MV_ME_ONLY))
		return true;
#endif
	return false;
}

#if defined(__APPLE__)
// The address range the views need, from the base.
static size_t ViewSpan(MemMapSetupFlags flags) {
	size_t span = 0;
	for (int i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].size == 0 || SkipView(flags, views[i].flags) || CanIgnoreView(views[i]) || (views[i].flags & MV_ME_ONLY))
			continue;
#ifdef MASKED_PSP_MEMORY
		const u32 address = views[i].virtual_address & MEMVIEW32_MASK;
#else
		const u32 address = views[i].virtual_address;
#endif
		span = std::max(span, (size_t)address + views[i].size);
	}
	return span;
}
#endif

static bool Memory_TryBase(MemMapSetupFlags flags) {
	// OK, we know where to find free space. Now grab it!
	// We just mimic the popular BAT setup.

	size_t position = 0;
	size_t last_position = 0;

	// Zero all the pointers to be sure.
	for (int i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].out_ptr) {
			*views[i].out_ptr = 0;
		}
	}

	int i;  // the final value of i is used in the next loop
	for (i = 0; i < ARRAY_SIZE(views); i++) {
		const MemoryView &view = views[i];
		if (view.size == 0)
			continue;
		if (SkipView(flags, view.flags))
			continue;
		
		if (view.flags & MV_MIRROR_PREVIOUS) {
			position = last_position;
		}
#ifndef MASKED_PSP_MEMORY
		// The ME's own views only get their place in the arena here, for CreateMESpace().
		if (!(view.flags & MV_ME_ONLY)) {
			*view.out_ptr = (u8*)g_arena.CreateView(
				position, view.size, base + view.virtual_address);
			if (!*view.out_ptr) {
				ERROR_LOG(Log::MemMap, "Failed at view %d", i);
				goto bail;
			}
		}
#else
		if (CanIgnoreView(view)) {
			// This is handled by address masking in 32-bit, no view needs to be created.
			*view.out_ptr = *views[i - 1].out_ptr;
		} else {
			*view.out_ptr = (u8*)g_arena.CreateView(
				position, view.size, base + (view.virtual_address & MEMVIEW32_MASK));
			if (!*view.out_ptr) {
				ERROR_LOG(Log::MemMap, "Failed at view %d", i);
				goto bail;
			}
		}
#endif
		viewPositions[i] = position;
		last_position = position;
		position += g_arena.roundup(view.size);
	}

	return true;
bail:
	// Argh! ERROR! Free what we grabbed so far so we can try again.
	for (int j = 0; j <= i; j++) {
		if (views[j].size == 0)
			continue;
		if (SkipView(flags, views[j].flags))
			continue;
		if (views[j].out_ptr && *views[j].out_ptr) {
			if (!CanIgnoreView(views[j])) {
				g_arena.ReleaseView(0, *views[j].out_ptr, views[j].size);
			}
			*views[j].out_ptr = nullptr;
		}
	}
	return false;
}

bool MemoryMap_Setup(MemMapSetupFlags flags) {
	g_setupFlags = flags;
#if PPSSPP_PLATFORM(UWP)
	// We reserve the memory, then simply commit in TryBase.
	base = (u8*)VirtualAllocFromApp(0, 0x10000000, MEM_RESERVE, PAGE_READWRITE);
#else

	// Figure out how much memory we need to allocate in total.
	size_t total_mem = 0;
	for (int i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].size == 0)
			continue;
		if (SkipView(flags, views[i].flags))
			continue;
		if (!CanIgnoreView(views[i]))
			total_mem += g_arena.roundup(views[i].size);
	}

	// Grab some pagefile backed memory out of the void ...
	if (!g_arena.GrabMemSpace(total_mem)) {
		// It'll already have logged.
		return false;
	}
#endif

#if defined(__APPLE__)
	// Let the kernel find room for the views, instead of guessing at bases.
	{
		const size_t span = ViewSpan(flags);
#ifdef MASKED_PSP_MEMORY
		base = MemArena::ReserveViewSpace(span, 0);
#else
		// A 4GB aligned base lets the ARM64 JIT pointerify guest addresses with a single MOVK.
		base = MemArena::ReserveViewSpace(span, 0xFFFFFFFF);
		if (!base) {
			base = MemArena::ReserveViewSpace(span, 0);
		}
#endif
	}
	if (base) {
		if (Memory_TryBase(flags)) {
			INFO_LOG(Log::MemMap, "Mapped the memory views into reserved space at %p", base);
			return true;
		}
		WARN_LOG(Log::MemMap, "Couldn't map the memory views into reserved space, falling back");
		MemArena::ReleaseViewSpace(base);
	}
#endif

#if !PPSSPP_PLATFORM(ANDROID)
	if (g_arena.NeedsProbing()) {
		int base_attempts = 0;
#if PPSSPP_PLATFORM(WINDOWS) && PPSSPP_ARCH(32BIT)
		// Try a whole range of possible bases. Return once we got a valid one.
		uintptr_t max_base_addr = 0x7FFF0000 - 0x10000000;
		uintptr_t min_base_addr = 0x01000000;
		uintptr_t stride = 0x400000;
#elif PPSSPP_ARCH(ARM64) && PPSSPP_PLATFORM(IOS)
		// iOS
		uintptr_t max_base_addr = 0x1FFFF0000ULL - 0x80000000ULL;
		uintptr_t min_base_addr = 0x100000000ULL;
		uintptr_t stride = 0x800000;
#else
		uintptr_t max_base_addr = 0;
		uintptr_t min_base_addr = 0;
		uintptr_t stride = 0;
		ERROR_LOG(Log::MemMap, "MemoryMap_Setup: Hit a wrong path, should not be needed on this platform.");
		return false;
#endif
		for (uintptr_t base_addr = min_base_addr; base_addr < max_base_addr; base_addr += stride) {
			base_attempts++;
			base = (u8 *)base_addr;
			if (Memory_TryBase(flags)) {
				INFO_LOG(Log::MemMap, "Found valid memory base at %p after %i tries.", base, base_attempts);
				return true;
			}
		}
		ERROR_LOG(Log::MemMap, "MemoryMap_Setup: Failed finding a memory base.");
		return false;
	}
	else
#endif
	{
#if !PPSSPP_PLATFORM(UWP)
		base = g_arena.Find4GBBase();
		if (!base) {
			return false;
		}
#endif
	}

	// Should return true...
	return Memory_TryBase(flags);
}

void MemoryMap_Shutdown() {
	size_t position = 0;
	size_t last_position = 0;
	const MemMapSetupFlags flags = g_setupFlags;
	g_setupFlags = MemMapSetupFlags::Default;

	for (int i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].size == 0)
			continue;
		if (SkipView(flags, views[i].flags))
			continue;

		if (views[i].flags & MV_MIRROR_PREVIOUS) {
			position = last_position;
		}

		if (*views[i].out_ptr)
			g_arena.ReleaseView(position, *views[i].out_ptr, views[i].size);
		*views[i].out_ptr = nullptr;

		last_position = position;
		position += g_arena.roundup(views[i].size);
	}
	g_arena.ReleaseSpace();

#if PPSSPP_PLATFORM(UWP)
	VirtualFree(base, 0, MEM_RELEASE);
#elif defined(__APPLE__)
	MemArena::ReleaseViewSpace(base);
#endif
}

// Commits the shadow under every view, so exactly the addresses that are valid PSP memory have entries.
// Mirrors fold onto the same range under the mask, and committing it again is harmless.
static bool BlockShadow_Setup(MemMapSetupFlags flags) {
	blockShadow = (u8 *)ReserveMemoryPages(BLOCK_SHADOW_SIZE);
	if (!blockShadow) {
		return false;
	}
	for (size_t i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].size == 0 || SkipView(flags, views[i].flags) || (views[i].flags & MV_ME_ONLY))
			continue;
		const u32 start = views[i].virtual_address & BLOCK_SHADOW_MASK;
		_dbg_assert_((size_t)start + views[i].size <= BLOCK_SHADOW_SIZE);
		if (!CommitMemoryPages(blockShadow + start, views[i].size)) {
			FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE);
			blockShadow = nullptr;
			return false;
		}
	}
	return true;
}

// On some 32 bit platforms (like old Android, old iOS, etc.), there are/were restrictions on memory map sizes.
// This particular size I can't find any sources for though.
static const int MAX_MMAP_SIZE = 31 * 1024 * 1024;

// Every size we actually use (32MB, 64MB, and the 76MB remasters) is a whole number of megabytes.
static bool IsPlausibleMemorySize(u32 size) {
	return size != 0 && size <= (u32)MAX_MMAP_SIZE * 3 && (size & 0xFFFFF) == 0;
}

bool Init(MemMapSetupFlags flags) {
	_dbg_assert_msg_(g_MemorySize <= MAX_MMAP_SIZE * 3, "ACK - too much memory for three mmap views.");
	for (size_t i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].flags & MV_IS_PRIMARY_RAM)
			views[i].size = std::min((int)g_MemorySize, MAX_MMAP_SIZE);
		if (views[i].flags & MV_IS_EXTRA1_RAM)
			views[i].size = std::min(std::max((int)g_MemorySize - MAX_MMAP_SIZE, 0), MAX_MMAP_SIZE);
		if (views[i].flags & MV_IS_EXTRA2_RAM)
			views[i].size = std::min(std::max((int)g_MemorySize - MAX_MMAP_SIZE * 2, 0), MAX_MMAP_SIZE);
		// The ME's stack in mcidclan's libmecore says 2MB on the PSP-1000 and 4MB on the others. Only the
		// 1000 is hardware-checked (its ME image, which libmecore keys this on).
		if (views[i].flags & MV_EDRAM)
			views[i].size = g_PSPModel == PSP_MODEL_FAT ? 0x00200000 : 0x00400000;
	}

	// The shadow first, so the base search can't put it in a gap of the PSP's 4GB window, where a bad
	// guest access would hit it instead of faulting.
	if (!BlockShadow_Setup(flags)) {
		return false;
	}

	if (!MemoryMap_Setup(flags)) {
		FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE);
		blockShadow = nullptr;
		return false;
	}

	INFO_LOG(Log::MemMap, "Memory system initialized. Base at %p (RAM at @ %p, uncached @ %p)",
		base, m_pPhysicalRAM, m_pUncachedRAM);

	g_mainSpace.base = base;
	g_mainSpace.blockShadow = blockShadow;
	memset(g_unmappedBootPage, 0, sizeof(g_unmappedBootPage));
	MemFault_Init();
	return true;
}

// Returns false if the new map couldn't be set up - in which case there is no memory map at all,
// and base is null. Callers must not carry on writing to guest memory.
bool Reinit() {
	_assert_msg_(PSP_GetBootState() == BootState::Complete, "Cannot reinit during startup/shutdown");
	Core_NotifyLifecycle(CoreLifecycle::MEMORY_REINITING);
	// Held across both halves: between Shutdown() and Init() there is no memory map at all, and a
	// reader that only saw Shutdown()'s own acquire could slip into that gap.
	CoreShutdownLock coreLock = Core_LockAgainstShutdown();
	MemMapSetupFlags flags = g_setupFlags;
	Shutdown();
	const bool success = Init(flags);
	Core_NotifyLifecycle(CoreLifecycle::MEMORY_REINITED);
	return success;
}

// Loads size bytes at start, comparing the parts of it in compare->ranges with what's there as it
// goes. A chunk that already matches doesn't need copying, and the matching parts are reported in
// compare->unchanged. The rest is a plain copy.
static void CopyComparing(uint8_t *dest, const uint8_t *src, u32 start, u32 size, RamCompare *compare) {
	// In the size of the JIT's block lookup pages. One changed byte costs its chunk's blocks.
	constexpr u32 CHUNK_SIZE = 1024;

	// As offsets into the copy, in order, merged.
	std::vector<std::pair<u32, u32>> ranges;
	for (const auto &[address, length] : compare->ranges) {
		const u32 rangeStart = std::max(address & 0x3FFFFFFF, start);
		const u32 rangeEnd = std::min((address & 0x3FFFFFFF) + length, start + size);
		if (rangeStart < rangeEnd) {
			ranges.emplace_back(rangeStart - start, rangeEnd - start);
		}
	}
	std::sort(ranges.begin(), ranges.end());

	u32 pos = 0;
	for (const auto &[rangeStart, rangeEnd] : ranges) {
		const u32 from = std::max(rangeStart, pos);
		if (from >= rangeEnd) {
			continue;
		}
		if (from > pos) {
			ParallelMemcpy(&g_threadManager, dest + pos, src + pos, from - pos);
		}
		u32 runStart = 0;
		bool inRun = false;
		for (u32 chunk = from; chunk < rangeEnd; chunk += CHUNK_SIZE) {
			const u32 chunkSize = std::min(CHUNK_SIZE, rangeEnd - chunk);
			const bool same = memcmp(dest + chunk, src + chunk, chunkSize) == 0;
			if (same && !inRun) {
				runStart = chunk;
				inRun = true;
			} else if (!same) {
				memcpy(dest + chunk, src + chunk, chunkSize);
				if (inRun) {
					compare->unchanged.emplace_back(start + runStart, chunk - runStart);
					inRun = false;
				}
			}
		}
		if (inRun) {
			compare->unchanged.emplace_back(start + runStart, rangeEnd - runStart);
		}
		pos = rangeEnd;
	}
	if (pos < size) {
		ParallelMemcpy(&g_threadManager, dest + pos, src + pos, size - pos);
	}
}

static void DoMemoryVoid(PointerWrap &p, uint32_t start, uint32_t size, RamCompare *compare = nullptr) {
	uint8_t *d = GetPointerWriteOrException(start);
	uint8_t *&storage = *p.ptr;

	// We only handle aligned data and sizes.
	if ((size & 0x3F) != 0 || ((uintptr_t)d & 0x3F) != 0)
		return p.DoVoid(d, size);

	if ((p.mode == PointerWrap::MODE_READ || p.mode == PointerWrap::MODE_VERIFY) && !p.CheckRead(size)) {
		return;
	}

	switch (p.mode) {
	case PointerWrap::MODE_READ:
		if (compare && !compare->ranges.empty()) {
			CopyComparing(d, storage, start, size, compare);
		} else {
			ParallelMemcpy(&g_threadManager, d, storage, size);
		}
		break;
	case PointerWrap::MODE_WRITE:
		ParallelMemcpy(&g_threadManager, storage, d, size);
		break;
	case PointerWrap::MODE_MEASURE:
		// Nothing to do here.
		break;
	case PointerWrap::MODE_VERIFY:
		ParallelRangeLoop(&g_threadManager, [&](int l, int h) {
			for (int i = l; i < h; i++)
				_dbg_assert_msg_(d[i] == storage[i], "Savestate verification failure: %d (0x%X) (at %p) != %d (0x%X) (at %p).\n", d[i], d[i], &d[i], storage[i], storage[i], &storage[i]);
		}, 0, size, 128, 8);
		break;
	case PointerWrap::MODE_NOOP:
		break;
	}
	storage += size;
}

void DoState(PointerWrap &p, RamCompare *compare) {
	auto s = p.Section("Memory", 1, 3);
	if (!s)
		return;

	const u32 memorySizeBefore = g_MemorySize;

	if (s < 2) {
		if (!g_RemasterMode)
			g_MemorySize = RAM_NORMAL_SIZE;
		g_PSPModel = PSP_MODEL_FAT;
	} else if (s == 2) {
		// In version 2, we determine memory size based on PSP model.
		u32 oldMemorySize = g_MemorySize;
		Do(p, g_PSPModel);
		p.DoMarker("PSPModel");
		if (!g_RemasterMode) {
			g_MemorySize = g_PSPModel == PSP_MODEL_FAT ? RAM_NORMAL_SIZE : RAM_DOUBLE_SIZE;
			if (oldMemorySize < g_MemorySize && !Reinit()) {
				ERROR_LOG(Log::MemMap, "Failed to reinit memory to %08x bytes", g_MemorySize);
				p.SetError(PointerWrap::ERROR_FAILURE);
				return;
			}
		}
	} else {
		// In version 3, we started just saving the memory size directly.
		// It's no longer based strictly on the PSP model.
		u32 oldMemorySize = g_MemorySize;
		Do(p, g_PSPModel);
		p.DoMarker("PSPModel");
		Do(p, g_MemorySize);
		if (p.mode == PointerWrap::MODE_READ && !IsPlausibleMemorySize(g_MemorySize)) {
			// Straight out of the file, so don't hand it to Init() - a bogus size makes the map
			// fail to allocate, and we'd carry on writing RAM through a null base.
			ERROR_LOG(Log::MemMap, "Savestate specifies an implausible memory size: %08x", g_MemorySize);
			g_MemorySize = oldMemorySize;
			p.SetError(PointerWrap::ERROR_FAILURE);
			return;
		}
		if (oldMemorySize != g_MemorySize && !Reinit()) {
			ERROR_LOG(Log::MemMap, "Failed to reinit memory to %08x bytes, restoring %08x", g_MemorySize, oldMemorySize);
			g_MemorySize = oldMemorySize;
			Reinit();
			p.SetError(PointerWrap::ERROR_FAILURE);
			return;
		}
	}

	// A new size means Reinit(): RAM is newly mapped, nothing in it to compare with, and the JIT has
	// to start over (its dispatcher points at the old mapping.)
	if (g_MemorySize != memorySizeBefore) {
		compare = nullptr;
	}
	DoMemoryVoid(p, PSP_GetKernelMemoryBase(), g_MemorySize, compare);
	p.DoMarker("RAM");

	DoMemoryVoid(p, PSP_GetVidMemBase(), VRAM_SIZE);
	p.DoMarker("VRAM");
	DoArray(p, m_pPhysicalScratchPad, SCRATCHPAD_SIZE);
	p.DoMarker("ScratchPad");
}

void Shutdown() {
	CoreShutdownLock coreLock = Core_LockAgainstShutdown();
	u32 flags = 0;
	DestroyMESpace();
	MemoryMap_Shutdown();
	base = nullptr;
	FreeMemoryPages(blockShadow, BLOCK_SHADOW_SIZE);
	blockShadow = nullptr;
	g_mainSpace = AddressSpace();
	DEBUG_LOG(Log::MemMap, "Memory system shut down.");
}

bool IsActive() {
	return base != nullptr;
}

u8 *GetBootPage() {
	return m_pBootPage ? m_pBootPage : g_unmappedBootPage;
}

// The Media Engine's map: g_arena's views again, minus the main CPU's own, plus the eDRAM.
static AddressSpace g_meSpace;
static std::vector<std::pair<u8 *, size_t>> g_meViews;

AddressSpace *CreateMESpace() {
	_dbg_assert_(base != nullptr && g_meSpace.base == nullptr);
#if defined(MASKED_PSP_MEMORY) || defined(NO_MMAP)
	// TODO: Masking would fold the eDRAM mirrors onto the scratchpad and boot page addresses.
	static_assert(!CanCreateMESpace());
	return nullptr;
#else
	size_t span = 0;
	for (size_t i = 0; i < ARRAY_SIZE(views); i++) {
		if (views[i].size == 0 || SkipView(g_setupFlags, views[i].flags) || (views[i].flags & MV_MAIN_CPU_ONLY))
			continue;
		span = std::max(span, (size_t)views[i].virtual_address + views[i].size);
		if (views[i].flags & MV_EDRAM)
			g_meSpace.edramSize = views[i].size;
		if (views[i].flags & MV_ME_LOCAL)
			g_meSpace.localSize = views[i].size;
	}

	g_meSpace.blockShadow = (u8 *)ReserveMemoryPages(BLOCK_SHADOW_SIZE);
#if defined(__APPLE__)
	g_meSpace.base = MemArena::ReserveViewSpace(span, 0xFFFFFFFF);
	if (!g_meSpace.base) {
		g_meSpace.base = MemArena::ReserveViewSpace(span, 0);
	}
#else
	g_meSpace.base = g_arena.Find4GBBase();
#endif
	if (!g_meSpace.base || !g_meSpace.blockShadow) {
		DestroyMESpace();
		return nullptr;
	}

	for (size_t i = 0; i < ARRAY_SIZE(views); i++) {
		const MemoryView &view = views[i];
		if (view.size == 0 || SkipView(g_setupFlags, view.flags) || (view.flags & MV_MAIN_CPU_ONLY))
			continue;
		u8 *ptr = (u8 *)g_arena.CreateView(viewPositions[i], view.size, g_meSpace.base + view.virtual_address);
		if (!ptr) {
			ERROR_LOG(Log::MemMap, "Couldn't map the Media Engine's view at %08x", view.virtual_address);
			DestroyMESpace();
			return nullptr;
		}
		g_meViews.emplace_back(ptr, view.size);
		if (!CommitMemoryPages(g_meSpace.blockShadow + (view.virtual_address & BLOCK_SHADOW_MASK), view.size)) {
			DestroyMESpace();
			return nullptr;
		}
	}
	INFO_LOG(Log::MemMap, "Media Engine memory at %p, %d KB of eDRAM", g_meSpace.base, g_meSpace.edramSize / 1024);
	return &g_meSpace;
#endif
}

void DestroyMESpace() {
	for (const auto &[ptr, size] : g_meViews) {
		g_arena.ReleaseView(0, ptr, size);
	}
	g_meViews.clear();
#if defined(__APPLE__)
	if (g_meSpace.base) {
		MemArena::ReleaseViewSpace(g_meSpace.base);
	}
#endif
	if (g_meSpace.blockShadow) {
		FreeMemoryPages(g_meSpace.blockShadow, BLOCK_SHADOW_SIZE);
	}
	g_meSpace = AddressSpace();
}

Opcode Read_Instruction(u32 address) {
	if (!IsValid4AlignedAddress(address)) {
		// BAD!
		_dbg_assert_(false);
		return Opcode(0);
	}
	return Opcode(ReadUnchecked_U32(address));
}

Opcode ReadUnchecked_Instruction(u32 address) {
	_dbg_assert_((address & 3) == 0);
	return Opcode(ReadUnchecked_U32(address));
}

u32 ReadBlockShadow(u32 address) {
	return g_mainSpace.ReadBlockShadow(address);
}

void WriteBlockShadow(u32 address, u32 value) {
	g_mainSpace.WriteBlockShadow(address, value);
}

bool ClearBlockShadow(u32 address, u32 value) {
	return g_mainSpace.ClearBlockShadow(address, value);
}

u32 AddressSpace::ReadBlockShadow(u32 address) const {
	if (!blockShadow || !IsValid4AlignedAddress(address))
		return 0;
	return *GetBlockShadowEntry(address);
}

void AddressSpace::WriteBlockShadow(u32 address, u32 value) const {
	if (!blockShadow || !IsValid4AlignedAddress(address))
		return;
	*GetBlockShadowEntry(address) = value;
}

bool AddressSpace::ClearBlockShadow(u32 address, u32 value) const {
	if (!blockShadow || !IsValid4AlignedAddress(address))
		return false;
	u32 *entry = GetBlockShadowEntry(address);
	if (*entry != value)
		return false;
	// A replacement hook keeps the entry nonzero, see ReplaceTables.h.
	*entry = Replacement_IsHooked(address) ? Replacement_GetBlockShadowHook() : 0;
	return true;
}

void Memset(const u32 addr, const u8 value, const u32 size, const char *tag) {
	if (size == 0) {
		// We ignore invalid addresses etc if the length is zero.
		return;
	}

	if (IsValidRange(addr, size)) {
		uint8_t *ptr = GetPointerWriteUnchecked(addr);
		memset(ptr, value, size);
	} else {
		// TODO: This mainly seems to be produced by GPUCommon::PerformMemorySet, called from
		// Replace_memset_jak().
		if (Memory::IsValidRange(addr, size)) {
			for (size_t i = 0; i < size; i++) {
				Memory::WriteUnchecked_U8(value, (u32)(addr + i));
			}
		}
	}

	if (tag) {
		NotifyMemInfo(MemBlockFlags::WRITE, addr, size, tag, strlen(tag));
	}
}

} // namespace

void PSPPointerNotifyRW(int rw, uint32_t ptr, uint32_t bytes, const char * tag, size_t tagLen) {
	if (MemBlockInfoDetailed(bytes)) {
		if (rw & 1)
			NotifyMemInfo(MemBlockFlags::WRITE, ptr, bytes, tag, tagLen);
		if (rw & 2)
			NotifyMemInfo(MemBlockFlags::READ, ptr, bytes, tag, tagLen);
	}
}