diff --git a/Core/HLE/HLE.cpp b/Core/HLE/HLE.cpp
index 7b23506309..19a18a96d4 100644
--- a/Core/HLE/HLE.cpp
+++ b/Core/HLE/HLE.cpp
@@ -446,6 +446,10 @@ void HLECheckModuleAvailability() {
 	}
 }
 
+DisableHLEFlags HLEGetUnavailableDisableFlags() {
+	return g_unavailableDisableFlags;
+}
+
 void HLEInit() {
 	RegisterAllModules();
 	// Latched lazily rather than here: the compat flags this depends on aren't loaded yet.
diff --git a/Core/HLE/HLE.h b/Core/HLE/HLE.h
index 09933620c2..fb65bbffd0 100644
--- a/Core/HLE/HLE.h
+++ b/Core/HLE/HLE.h
@@ -188,6 +188,10 @@ inline s64 hleDelayResult(s64 result, const char *reason, int usec) {
 
 void HLEInit();
 void HLECheckModuleAvailability();
+// The flags that were asked for but could not be honoured, because the firmware module they need
+// is neither installed nor on the disc - those libraries ran our HLE instead. HLECheckModuleAvailability()
+// fills this in during boot, so it only means anything after that.
+DisableHLEFlags HLEGetUnavailableDisableFlags();
 void HLEDoState(PointerWrap &p);
 void HLEShutdown();
 const HLEFunction *HLEGetFunctionBeingCalled();
diff --git a/docs/debugging.md b/docs/debugging.md
index 0d286ded64..5593578ffc 100644
--- a/docs/debugging.md
+++ b/docs/debugging.md
@@ -171,6 +171,13 @@ produced a round of bogus results here:
 The last three compound: the fix is to treat the run's exit code and a positive "we got here" counter as
 preconditions, and only then believe the error counts.
 
+For the silent-fallback half of this, headless refuses the run rather than substituting: an explicit
+`--disable-hle=` whose firmware module isn't there names the module, prints the `flash0:/kd` and memory stick
+it looked in (usually enough to spot that it's the one beside the exe), and exits 1. Only an *explicit*
+`--disable-hle` binds - sceMpeg and sceMp4 are LLE by default and still fall back quietly, or every run on a
+machine with no firmware would fail. So when a measurement depends on the real module, pass the flag
+explicitly even though it's on by default, and the run will tell you if it didn't get it.
+
 ## Debugging and breakpoint considerations
 
 It might be worth trying the interpreter - all types of breakpoints are the most reliable with this CPU backend.
diff --git a/headless/Headless.cpp b/headless/Headless.cpp
index 0bdd823435..cfc73f1095 100644
--- a/headless/Headless.cpp
+++ b/headless/Headless.cpp
@@ -55,6 +55,7 @@
 #include "Core/Core.h"
 #include "Core/CoreTiming.h"
 #include "Core/EmuThread.h"
+#include "Core/HLE/HLE.h"
 #include "Core/MIPS/MIPSTables.h"
 #include "Core/System.h"
 #include "Core/Util/PSARUnpack.h"
@@ -83,6 +84,9 @@ static bool g_screenshotSaved = false;
 static double g_maxScreenshotError = 0.0;
 static bool g_screenshotFailed = false;
 static std::string g_debugOutputBuffer;
+// Set when a run was asked for a configuration that couldn't be honoured. That isn't a test result,
+// so it fails the process whether or not this run was comparing anything.
+static bool g_configRefused = false;
 static bool g_writeFailureScreenshot = true;
 static bool g_writeDebugOutput = true;
 // Whether the emulated program's stdout/stderr are forwarded to ours. On by default - just running
@@ -325,6 +329,10 @@ struct AutoTestOptions {
 	bool verbose;
 	bool bench;
 	bool printEqualLines;
+	// What --disable-hle asked for, or 0 if it was not passed. Only an explicit request binds:
+	// sceMpeg and sceMp4 run the firmware module by default now and fall back to the HLE wherever
+	// none is installed, which must not fail every run on such a machine.
+	int requiredDisableHLE;
 };
 
 static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &coreParameter, const AutoTestOptions &opt) {
@@ -364,6 +372,29 @@ static bool RunAutoTest(GraphicsContext *graphicsContext, CoreParameter &corePar
 		return false;
 	}
 
+	// Running a different configuration than the one asked for measures the wrong thing without
+	// saying so, which is worse in a test tool than not running at all.
+	const int missingHLE = (int)HLEGetUnavailableDisableFlags() & opt.requiredDisableHLE;
+	if (missingHLE) {
+		for (int i = 0; i < (int)DisableHLEFlags::Count; i++) {
+			if (!(missingHLE & (1 << i))) {
+				continue;
+			}
+			const HLEModuleMeta *meta = GetHLEModuleMetaByFlag((DisableHLEFlags)(1 << i));
+			fprintf(stderr, "--disable-hle asked for %s, but no firmware module for it is installed "
+				"or on the disc - our HLE would run instead.\n", meta ? meta->modname : "an unknown module");
+		}
+		// Nearly always because headless defaulted the memory stick to one beside the exe rather than
+		// the app's, so the firmware installed through the app isn't the firmware it looked at.
+		fprintf(stderr, "Looked in %s (memory stick %s).\n",
+			(g_Config.nandRootDirectory / "flash0" / "kd").c_str(), g_Config.memStickDirectory.c_str());
+		GitHubActionsPrint("error", "Requested --disable-hle unavailable for %s", currentTestName.c_str());
+		g_configRefused = true;
+		// Booted, so it has to come down the same way a finished run does.
+		PSP_Shutdown(true);
+		return false;
+	}
+
 	System_Notify(SystemNotification::BOOT_DONE);
 
 	PSP_UpdateDebugStats((DebugOverlay)g_Config.iDebugOverlay == DebugOverlay::DEBUG_STATS || g_Config.bLogFrameDrops);
@@ -607,6 +638,10 @@ int RunTests(GraphicsContext *graphicsContext, CoreParameter &coreParameter, con
 		}
 	}
 
+	if (g_configRefused) {
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -668,6 +703,7 @@ int main(int argc, const char* argv[]) {
 	testOptions.verbose = cmdLineOptions.verbose.value_or(false);
 	testOptions.printEqualLines = cmdLineOptions.printEqualLines.value_or(false);
 	testOptions.maxScreenshotError = cmdLineOptions.maxScreenshotError.value_or(0.0);
+	testOptions.requiredDisableHLE = cmdLineOptions.disableHLE.value_or(0);
 
 	bool fullLog = cmdLineOptions.enableLogging.value_or(false);
 	const char *stateToLoad = cmdLineOptions.stateToLoad.has_value() ? cmdLineOptions.stateToLoad.value().c_str() : nullptr;