VSH: Start the shell's modules one at a time, fixing game icons

The kernel starts modules one by one, each module_start returning before
the next starts. We started them all as threads side by side, and
vsh_module's start ran before paf's. Its 12MB request for game_plugin's
surface pools then went to the 8.3MB heap we pre-filled scePaf's pool
slot with, failed, and game_plugin threw away every decoded ICON0 and
PIC1 (white tiles on 3.80-5.05, grey placeholders from 5.50).

heaparea1, vshbridge, paf, common_gui and common_util now start in order,
and vsh_module waits for the last. The kd/ drivers still start side by
side, since syscon's start blocks on its hardware interrupt. paf fills
its own pool slot now, so the pre-fill is gone. Icons show on every
version from 1.50 to 6.61, and homebrew launches on the same ones as
before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Core/HLE/sceKernelModule.cpp

@@ -495,6 +495,11 @@ static bool g_runningVSH = false;
 // STATE END
 //////////////////////////////////////////////////////////////////////////
 
+// The VSH's kernel modules that are loaded but not started yet, in boot order. Like the plugin
+// wait it piggybacks on, this only lives through the first moments of a boot and isn't saved.
+static std::vector<SceUID> g_vshModulesToStart;
+static SceUID g_vshModuleStarting = 0;
+
 static void __KernelModuleInit() {
 	actionAfterModule = __KernelRegisterActionType(AfterModuleEntryCall::Create);
 }
@@ -554,6 +559,8 @@ void __KernelModuleDoState(PointerWrap &p) {
 
 void __KernelModuleShutdown() {
 	loadedModules.clear();
+	g_vshModulesToStart.clear();
+	g_vshModuleStarting = 0;
 	MIPSAnalyst::Reset();
 	HLEPlugins::Unload();
 }
@@ -1245,19 +1252,53 @@ static bool ShouldHLEModuleForLoad(std::string_view modname, bool *wasDisabledMa
 // these just load and run through the normal interpreter/JIT like any other PRX, same as the
 // existing 4 VSH-specific modules below. Order matches JPCSP's load order exactly, in case
 // later modules depend on earlier ones having already initialized.
-static void LoadAndStartVshKernelModule(const char *path, SceKernelSMOption *smoption) {
+//
+// The kernel starts them one at a time: each module_start returns before the next module starts,
+// and the VSH itself starts after the last. The drivers can't be waited for here (syscon's start
+// blocks on a semaphore its hardware interrupt would signal), so they start side by side as
+// threads. The shell's own modules (inOrder) are started one by one by StartNextVshKernelModule,
+// with the VSH waiting for the last. Run side by side instead, vsh_module's start allocates its
+// 12MB of paf memory for game_plugin's icon surfaces before paf's start has set up the heap that
+// could hold it, and every game icon from 3.80 on comes out blank.
+static void LoadVshKernelModule(const char *path, bool inOrder) {
 	std::string error_string;
 	SceUID moduleId = KernelLoadModule(path, &error_string);
 	if (moduleId < 0) {
-		WARN_LOG(Log::sceModule, "LoadAndStartVshKernelModules: failed to load %s: %s", path, error_string.c_str());
+		WARN_LOG(Log::sceModule, "LoadVshKernelModules: failed to load %s: %s", path, error_string.c_str());
 		return;
 	}
-	int result = __KernelStartModule(moduleId, 0, 0, 0, smoption, nullptr);
-	if (result < 0) {
-		WARN_LOG(Log::sceModule, "LoadAndStartVshKernelModules: failed to start %s (uid=%d)", path, moduleId);
+	if (inOrder) {
+		g_vshModulesToStart.push_back(moduleId);
+	} else if (__KernelStartModule(moduleId, 0, 0, 0, nullptr, nullptr) < 0) {
+		WARN_LOG(Log::sceModule, "LoadVshKernelModules: failed to start %s (uid=%d)", path, moduleId);
 	}
 }
 
+// Starts the next of the VSH's kernel modules, and has waitingThread (of waitingModule) wait for
+// it with the plugins. Returns false when there was none left that needed waiting for.
+static bool StartNextVshKernelModule(PSPModule *waitingModule, SceUID waitingThread) {
+	g_vshModuleStarting = 0;
+	while (!g_vshModulesToStart.empty()) {
+		SceUID moduleId = g_vshModulesToStart.front();
+		g_vshModulesToStart.erase(g_vshModulesToStart.begin());
+		bool needsWait = false;
+		int result = __KernelStartModule(moduleId, 0, 0, 0, nullptr, &needsWait);
+		if (result < 0) {
+			WARN_LOG(Log::sceModule, "StartNextVshKernelModule: failed to start module %d", moduleId);
+			continue;
+		}
+		u32 error;
+		PSPModule *module = kernelObjects.Get<PSPModule>(moduleId, error);
+		if (needsWait && module) {
+			g_vshModuleStarting = moduleId;
+			waitingModule->startingPlugins.push_back(moduleId);
+			module->pluginWaitingThread = waitingThread;
+			return true;
+		}
+	}
+	return false;
+}
+
 // Some of the kernel's drivers have a per-model build (memlmd, loadexec, wlanfirm, ...), and a
 // firmware installed for one model ships only that model's - so asking for "_01g" unconditionally
 // fails on, say, an 02g install, which is what PPSSPP's own updater unpack produces by default.
@@ -1294,7 +1335,7 @@ static std::string ResolveVshModelModule(const char *path) {
 	return std::string(path);
 }
 
-static void LoadAndStartVshKernelModules() {
+static void LoadVshKernelModules() {
 	// These 11 are small, simple kernel drivers (a few KB to ~100KB of code each) that don't
 	// declare their own module_start_thread_stacksize, so each start thread gets the kernel
 	// module default of 0x1000.
@@ -1316,10 +1357,10 @@ static void LoadAndStartVshKernelModules() {
 		if (!pspFileSystem.GetFileInfo(resolved).exists) {
 			// Older firmwares don't have all of these - lowio.prx only appears around 3.52 - and a
 			// driver that isn't in the dump isn't a failure to report.
-			INFO_LOG(Log::sceModule, "LoadAndStartVshKernelModules: %s isn't in this firmware, skipping", resolved.c_str());
+			INFO_LOG(Log::sceModule, "LoadVshKernelModules: %s isn't in this firmware, skipping", resolved.c_str());
 			continue;
 		}
-		LoadAndStartVshKernelModule(resolved.c_str(), nullptr);
+		LoadVshKernelModule(resolved.c_str(), false);
 	}
 
 	// Firmwares up to about 4.05 keep scePaf's heap allocator in a module of its own, which paf
@@ -1328,7 +1369,7 @@ static void LoadAndStartVshKernelModules() {
 	// the existence check rather than a warning from the loader. heaparea1 and heaparea2 are the
 	// same code with different compiled-in pool sizes; the first is the one the shell asks for.
 	if (pspFileSystem.GetFileInfo("flash0:/vsh/module/heaparea1.prx").exists) {
-		LoadAndStartVshKernelModule("flash0:/vsh/module/heaparea1.prx", nullptr);
+		LoadVshKernelModule("flash0:/vsh/module/heaparea1.prx", true);
 	}
 
 	static const char *const vshUiKernelModulePaths[] = {
@@ -1338,7 +1379,7 @@ static void LoadAndStartVshKernelModules() {
 		"flash0:/vsh/module/common_util.prx",
 	};
 	for (const char *path : vshUiKernelModulePaths) {
-		LoadAndStartVshKernelModule(path, nullptr);
+		LoadVshKernelModule(path, true);
 	}
 }
 
@@ -1749,47 +1790,8 @@ static PSPModule *__KernelLoadELFFromPtr(const u8 *ptr, size_t elfSize, u32 load
 	strncpy(module->nm.name, modinfo->name, ARRAY_SIZE(module->nm.name));
 	module->nm.name[ARRAY_SIZE(module->nm.name) - 1] = '\0';
 
-	// scePaf's heap allocator expects a real memory-pool base address to already be in one of its
-	// BSS slots before any of its code runs. The module that owns the allocator fills that slot in
-	// itself, from its own module_start - but that start thread hasn't been scheduled yet when
-	// vshmain makes its first allocation, so the pointer is still null and the shell writes
-	// through it. Real hardware's kernel bootstrap starts these modules one at a time and waits;
-	// we can't, so pre-fill the slot with a real block instead. Without this the boot dies almost
-	// immediately, either on a null write inside scePaf (5.01+) or on vshmain storing the null the
-	// allocator handed back (up to 4.05). See docs/VSHBootInvestigation.md for the investigation.
-	//
-	// Which module owns it moved: up to about 4.05 the allocator is a separate heaparea1.prx, and
-	// from 5.01 it's compiled into paf.prx. Either way the slot is the second of the two pool
-	// pointers that module's init fills in with sceKernelTryAllocateFpl, and either way its offset
-	// from the module base moves with every build while its offset from gp does not - checked
-	// against paf.prx on 6.00, 6.20, 6.31, 6.37, 6.39, 6.60 and 6.61 (base-relative 0x18CCD8 to
-	// 0x18D728, gp - slot 0x7E88 every time) and heaparea1.prx on 3.95 and 4.05.
-	struct PafHeapOwner {
-		const char *moduleName;
-		u32 poolPointerGpOffset;
-	};
-	static const PafHeapOwner pafHeapOwners[] = {
-		{ "scePaf_Module", 0x7E88 },
-		{ "scePafHeaparea_Module", 0x7FCC },
-	};
-	for (const PafHeapOwner &owner : pafHeapOwners) {
-		if (!equals(module->nm.name, owner.moduleName)) {
-			continue;
-		}
-		// Matches the compiled-in default pool size in both modules.
-		u32 scePafHeapArenaSize = 0x00850000;
-		u32 arenaAddr = userMemory.Alloc(scePafHeapArenaSize, false, "scePafHeapArena");
-		u32 patchAddr = module->nm.gp_value - owner.poolPointerGpOffset;
-		if (arenaAddr != (u32)-1 && Memory::IsValid4AlignedAddress(patchAddr)) {
-			Memory::WriteUnchecked_U32(arenaAddr, patchAddr);
-		} else {
-			WARN_LOG(Log::sceModule, "Failed to patch %s heap arena pointer", owner.moduleName);
-		}
-		break;
-	}
-
 	if (equals(module->nm.name, "vsh_module")) {
-		// Like the above patch, this is likely firmware-version-specific.
+		// This is likely firmware-version-specific.
 		//
 		// vsh_module's SCE_VSH_GRAPHICS thread runs a scan over a small fixed table of
 		// "alarm task" categories (2 categories, each with a count followed by that many
@@ -1809,7 +1811,7 @@ static PSPModule *__KernelLoadELFFromPtr(const u8 *ptr, size_t elfSize, u32 load
 		// for category 0 the same way). This is a narrow, targeted patch of one 4-byte value
 		// vsh_module itself never properly initializes, not a general vsh_module patch.
 		//
-		// Unlike the scePaf patch above, this offset is into rodata rather than at a fixed
+		// This offset is into rodata rather than at a fixed
 		// distance from gp, and it moves with the build - so check that what's there is the
 		// value we identified before overwriting it, rather than writing blind into a firmware
 		// we haven't looked at. 6.60 and 6.61 ship byte-identical builds of vshmain.prx and
@@ -2417,7 +2419,7 @@ static bool __KernelLoadExecFromPtr(MIPSState * mips, const u8 *data, size_t siz
 		// precondition it needs is ready, regressing all the way back to the very first
 		// crash this investigation fixed (an immediate `break` in sceVshBridge_Driver). Left
 		// as the default module-declared attr/priority instead.
-		LoadAndStartVshKernelModules();
+		LoadVshKernelModules();
 	}
 
 	INFO_LOG(Log::System, "Starting modules...");
@@ -2428,9 +2430,13 @@ static bool __KernelLoadExecFromPtr(MIPSState * mips, const u8 *data, size_t siz
 
 	__KernelStartIdleThreads(module->GetUID());
 
-	// Wait until plugins are loaded
+	// Wait until plugins are loaded, and on the VSH, its kernel modules started.
 	module->startingPlugins.clear();
+	bool waitForStarts = g_runningVSH && StartNextVshKernelModule(module, __KernelGetCurThread());
 	if (HLEPlugins::Load(module, __KernelGetCurThread())) {
+		waitForStarts = true;
+	}
+	if (waitForStarts) {
 		__KernelWaitCurThread(WAITTYPE_PLUGIN, module->GetUID(), 1, 0, false, "started plugins");
 		__KernelReSchedule("Started plugins");
 	}
@@ -3107,6 +3113,9 @@ void __KernelReturnFromModuleFunc() {
 						break;
 					}
 				}
+				if (leftModuleID == g_vshModuleStarting) {
+					StartNextVshKernelModule(plugin_waiting_module, module->pluginWaitingThread);
+				}
 				if (plugin_waiting_module->startingPlugins.empty()) {
 					INFO_LOG(Log::sceModule, "Resuming LoadExec thread %d", module->pluginWaitingThread);
 					__KernelResumeThreadFromWait(module->pluginWaitingThread, 0);