diff --git a/Core/Debugger/LineInfo.h b/Core/Debugger/LineInfo.h
index e42bebad4f..4e3993b29b 100644
--- a/Core/Debugger/LineInfo.h
+++ b/Core/Debugger/LineInfo.h
@@ -54,9 +54,11 @@ public:
 	// to be already has final addresses, so the delta is zero.
 	int AddModule(std::string_view elfData, u32 moduleStart, u32 moduleSize, u32 addressDelta);
 
-	// Keyed the same way SymbolMap::UnloadModule is, so that unloading one module drops only its
-	// own lines. Each module owns its rows and its file names outright - there's no shared table
-	// for an unload to have to pick apart.
+	// Each module owns its rows and its file names outright, so dropping one can never disturb
+	// another's. Note this isn't called when a module unloads: a savestate load destroys and
+	// rebuilds every kernel object, and doing it there discarded the table every time - AddModule
+	// replacing by key is what retires a range instead, when something else claims it. See
+	// ~PSPModule.
 	void RemoveModule(u32 moduleStart, u32 moduleSize);
 	void Clear();
 
diff --git a/Core/ELF/ElfReader.cpp b/Core/ELF/ElfReader.cpp
index 068e06c3cb..770bc7f155 100644
--- a/Core/ELF/ElfReader.cpp
+++ b/Core/ELF/ElfReader.cpp
@@ -934,7 +934,12 @@ int LoadCompanionElfDebugInfo(const Path &gameFile, u32 moduleBase, u32 moduleSi
 	if (gameFile.empty() || gameFile.Type() != PathType::NATIVE)
 		return 0;
 
-	const Path dir = gameFile.NavigateUp();
+	// fileToStart is the game's own directory for folder-launched homebrew
+	// (IdentifiedFileType::PSP_PBP_DIRECTORY - the normal case when you pick one in the UI), and
+	// the EBOOT/ISO itself otherwise. Navigating up from a directory lands in PSP/GAME and lists
+	// sibling *games*, so the companion right there next to the EBOOT was never found - which is
+	// exactly the interactive case this exists for.
+	const Path dir = File::IsDirectory(gameFile) ? gameFile : gameFile.NavigateUp();
 	std::vector<File::FileInfo> files;
 	if (!File::GetFilesInDir(dir, &files, "elf:"))
 		return 0;
diff --git a/Core/HLE/sceKernelModule.cpp b/Core/HLE/sceKernelModule.cpp
index 063140197f..da425070ea 100644
--- a/Core/HLE/sceKernelModule.cpp
+++ b/Core/HLE/sceKernelModule.cpp
@@ -211,8 +211,14 @@ PSPModule::~PSPModule() {
 			userMemory.Free(memoryBlockAddr);
 		}
 		g_symbolMap->UnloadModule(memoryBlockAddr, memoryBlockSize);
-		// Keyed identically, so a module going away takes its own line info and nothing else's.
-		g_lineInfo.RemoveModule(memoryBlockAddr, memoryBlockSize);
+		// Deliberately *not* dropping this module's line info here. Loading a savestate deletes
+		// every kernel object and rebuilds it (KernelObjectPool::Clear), so removing on destruction
+		// threw the line table away every time a state was loaded - and for an ELF launched
+		// directly there's no file left to read it back from. SymbolMap has the same problem and
+		// solves it the same way: keep what you have, and let the module that next claims the
+		// address range replace it (LineInfoMap::AddModule replaces by key, and it's called for
+		// every module load whether or not that module has any line info to add). The whole thing
+		// is dropped when the game does, in PSP_Shutdown.
 	}
 
 	if (modulePtr.ptr) {
@@ -308,11 +314,11 @@ void PSPModule::DoState(PointerWrap &p) {
 		char moduleName[29] = { 0 };
 		truncate_cpy(moduleName, nm.name);
 		if (memoryBlockAddr != 0) {
+			// Re-registering is enough to bring both back: SymbolMap keeps every symbol it has ever
+			// seen and just rebuilds its active view from the loaded modules, and line info is no
+			// longer dropped when a module is destroyed (see ~PSPModule). So there's nothing to
+			// re-read here, and a state load doesn't pay for re-parsing the companion ELF.
 			g_symbolMap->AddModule(moduleName, memoryBlockAddr, memoryBlockSize, crc);
-			// Loading a state tears the old module down and re-registers it here, which takes its
-			// symbols and line info with it. The companion ELF is still sitting next to the game,
-			// so read it again rather than coming back from a savestate with none.
-			LoadCompanionElfDebugInfo(PSP_CoreParameter().fileToStart, memoryBlockAddr, memoryBlockSize);
 			if (g_Config.bAutoSaveLoadSymbols) {
 				int idx = g_symbolMap->GetModuleIndexByName(moduleName);
 				if (idx > 0) {
diff --git a/Core/System.cpp b/Core/System.cpp
index 84ad473695..721b41e6d0 100644
--- a/Core/System.cpp
+++ b/Core/System.cpp
@@ -50,6 +50,7 @@
 #include "Core/MIPS/MIPS.h"
 #include "Core/MIPS/MIPSAnalyst.h"
 #include "Core/MIPS/MIPSVFPUUtils.h"
+#include "Core/Debugger/LineInfo.h"
 #include "Core/Debugger/SymbolMap.h"
 #include "Core/System.h"
 #include "Core/HLE/HLE.h"
@@ -626,6 +627,9 @@ void CPU_Shutdown(bool success) {
 	g_CoreParameter.mountIsoLoader = nullptr;
 	delete g_symbolMap;
 	g_symbolMap = nullptr;
+	// Line info outlives individual modules on purpose (see ~PSPModule), so the game going away is
+	// what ends it.
+	g_lineInfo.Clear();
 
 	g_lua.Shutdown();