diff --git a/Core/Debugger/SymbolMap.cpp b/Core/Debugger/SymbolMap.cpp
index e3a0b56dae..9e5f07587b 100644
--- a/Core/Debugger/SymbolMap.cpp
+++ b/Core/Debugger/SymbolMap.cpp
@@ -615,15 +615,21 @@ void SymbolMap::AssignFunctionIndices() {
 void SymbolMap::UpdateActiveSymbols() {
 	// return;   (slow in debug mode)
 	lock_guard guard(lock_);
-	std::map<int, u32> activeModuleIndexes;
-	for (auto it = activeModuleEnds.begin(), end = activeModuleEnds.end(); it != end; ++it) {
-		activeModuleIndexes[it->second.index] = it->second.start;
-	}
 
 	activeFunctions.clear();
 	activeLabels.clear();
 	activeData.clear();
 
+	// On startup and shutdown, we can skip the rest.  Tiny optimization.
+	if (activeModuleEnds.empty() || (functions.empty() && labels.empty() && data.empty())) {
+		return;
+	}
+
+	std::map<int, u32> activeModuleIndexes;
+	for (auto it = activeModuleEnds.begin(), end = activeModuleEnds.end(); it != end; ++it) {
+		activeModuleIndexes[it->second.index] = it->second.start;
+	}
+
 	for (auto it = functions.begin(), end = functions.end(); it != end; ++it) {
 		const auto mod = activeModuleIndexes.find(it->second.module);
 		if (it->second.module == 0) {
diff --git a/headless/StubHost.h b/headless/StubHost.h
index 47ad6886d1..3247abccf1 100644
--- a/headless/StubHost.h
+++ b/headless/StubHost.h
@@ -17,7 +17,8 @@
 
 #pragma once
 
-#include "../Core/Host.h"
+#include "Core/Host.h"
+#include "Core/Debugger/SymbolMap.h"
 
 #define HEADLESSHOST_CLASS HeadlessHost
 
@@ -44,7 +45,7 @@ public:
 	virtual void BootDone() {}
 
 	virtual bool IsDebuggingEnabled() {return false;}
-	virtual bool AttemptLoadSymbolMap() {return false;}
+	virtual bool AttemptLoadSymbolMap() { symbolMap.Clear(); return false; }
 
 	virtual bool ShouldSkipUI() { return true; }