diff --git a/Common/ExceptionHandlerSetup.cpp b/Common/ExceptionHandlerSetup.cpp
index 6df5cc4a56..e26b9c7d85 100644
--- a/Common/ExceptionHandlerSetup.cpp
+++ b/Common/ExceptionHandlerSetup.cpp
@@ -20,9 +20,42 @@
 #include "Common/MachineContext.h"
 #include "Common/ExceptionHandlerSetup.h"
 
+#if defined(_MSC_VER)
+#include <crtdbg.h>
+#include "Common/CommonWindows.h"
+#endif
+
 static BadAccessHandler g_badAccessHandler;
 static void *altStack = nullptr;
 
+void SetupCRT(bool suppressDialogs) {
+#if defined(_MSC_VER)
+	_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
+
+	if (suppressDialogs) {
+		// 1. Redirect CRT assertions/errors/warnings to stderr.
+		const _HFILE reportTarget = _CRTDBG_FILE_STDERR;
+
+		_CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE);
+		_CrtSetReportFile(_CRT_ASSERT, reportTarget);
+
+		_CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
+		_CrtSetReportFile(_CRT_ERROR, reportTarget);
+
+		_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE);
+		_CrtSetReportFile(_CRT_WARN, reportTarget);
+
+		// 2. Suppress the abort() message box & crash reporting dialogs.
+		_set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+
+#if !PPSSPP_PLATFORM(UWP)
+		// 3. Suppress Windows OS-level "Program has stopped working" modal dialogs.
+		SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
+#endif
+	}
+#endif
+}
+
 #ifdef MACHINE_CONTEXT_SUPPORTED
 
 // We cannot handle exceptions in UWP builds. Bleh.
diff --git a/Common/ExceptionHandlerSetup.h b/Common/ExceptionHandlerSetup.h
index da47a55687..caaccaee8b 100644
--- a/Common/ExceptionHandlerSetup.h
+++ b/Common/ExceptionHandlerSetup.h
@@ -21,3 +21,8 @@ void InstallExceptionHandler(BadAccessHandler accessHandler, bool logStackTraceO
 
 // Implementation note: This must be a no-op if InstallExceptionHandler hasn't been called.
 void UninstallExceptionHandler();
+
+// MSVC-only, no-op elsewhere. Turns on the debug CRT's leak checking, and with suppressDialogs
+// set, routes CRT assertions and abort() to stderr instead of a modal dialog - required for
+// anything run non-interactively (headless, the unit tests, CI), where a dialog just hangs.
+void SetupCRT(bool suppressDialogs);
diff --git a/headless/Headless.cpp b/headless/Headless.cpp
index ba2ef8b647..b583eb6a22 100644
--- a/headless/Headless.cpp
+++ b/headless/Headless.cpp
@@ -38,6 +38,7 @@
 #include <csignal>
 #endif
 #include "Common/CPUDetect.h"
+#include "Common/ExceptionHandlerSetup.h"
 #include "Common/File/VFS/VFS.h"
 #include "Common/File/VFS/ZipFileReader.h"
 #include "Common/File/VFS/DirectoryReader.h"
@@ -520,35 +521,6 @@ public:
 	}
 };
 
-// Has a parameter because we might start using this from the main build.
-void SetupCRT(bool headless) {
-#if defined(_MSC_VER)
-	_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
-
-	if (headless) {
-		// Suppress abort dialogs and similar.
-		// 1. Redirect CRT Assertions/Errors/Warnings to stdout/stderr
-
-		const _HFILE reportTarget = _CRTDBG_FILE_STDERR;
-
-		_CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE);
-		_CrtSetReportFile(_CRT_ASSERT, reportTarget);
-
-		_CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
-		_CrtSetReportFile(_CRT_ERROR, reportTarget);
-
-		_CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE);
-		_CrtSetReportFile(_CRT_WARN, reportTarget);
-
-		// 2. Suppress the abort() message box & crash reporting dialogs
-		_set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
-
-		// 3. Suppress Windows OS-level "Program has stopped working" modal dialogs
-		SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
-	}
-#endif
-}
-
 int main(int argc, const char* argv[]) {
 	PROFILE_INIT();
 	TimeInit();
diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp
index 2859a87dff..b28ecf8a5f 100644
--- a/unittest/UnitTest.cpp
+++ b/unittest/UnitTest.cpp
@@ -77,6 +77,7 @@
 #include "Common/ArmEmitter.h"
 #include "Common/BitScan.h"
 #include "Common/CPUDetect.h"
+#include "Common/ExceptionHandlerSetup.h"
 #include "Common/Log.h"
 #include "Common/StringUtils.h"
 #include "Core/Config.h"
@@ -1762,6 +1763,8 @@ TestItem availableTests[] = {
 };
 
 int main(int argc, const char *argv[]) {
+	// Never block on a modal dialog - these get run from CI and from tooling.
+	SetupCRT(true);
 	SetCurrentThreadName("UnitTest");
 	TimeInit();