VSH: Play MP4 video from the memory stick

- Fixes #22763

The XMB's video player boots the ME in mode 3 through sceMeCore_driver's
sceMeBootStart (5dff5c50), where games get mode 2. Mode 3 only changes the
argument to ME command 0x185 (me_wrapper.prx), and the player sets the AAC
context flag at 0x2d on anything newer than a PSP-1000 and plays those
files, so sceAudiocodecInit now accepts the 0x2c/0x2d flags in mode 3.
Games still get the failure pspautotests audio/audiocodec/context records.

Also implements the two vaudio calls the player uses: a ChReserveBuffering
that routes each buffer through an ME effect (E8E78DC8), and the setter for
its parameter (504E4745). The effect isn't known, so audio goes out as is.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Core/HLE/HLETables.cpp

@@ -439,6 +439,7 @@ void RegisterAllModules() {
 	Register_PopsBootCalls();
 	Register_sceVaudio_driver();
 	Register_sceNetApctl_lib();
+	Register_sceMeCore_driver();
 
 	// Not ready to enable this due to apparent softlocks in Patapon 3.
 	// Register_sceNpMatching2();

Core/HLE/sceAudiocodec.cpp

@@ -19,6 +19,7 @@
 
 #include "Common/Serialize/SerializeFuncs.h"
 #include "Common/Serialize/SerializeMap.h"
+#include "Core/ConfigValues.h"
 #include "Core/HLE/HLE.h"
 #include "Core/HLE/FunctionWrappers.h"
 #include "Core/HLE/sceAudiocodec.h"
@@ -45,6 +46,8 @@ static std::map<u32, int> g_at3PlusFrameBytes;
 static std::map<u32, int> g_primingFrames;
 
 static bool oldStateLoaded = false;
+// What sceMeBootStart booted the ME for. Games get mode 2, the VSH mode 3.
+static int g_meBootMode = 2;
 
 static_assert(sizeof(SceAudiocodecCodec) == 128);
 // Games allocate this structure and the ME writes into it, so every offset is load-bearing.
@@ -111,8 +114,8 @@ static_assert(offsetof(SceAudiocodecCodec, allocMem) == 0x68);
 // ------------------------------------------------
 // Sample rate is at offset 0x28. The input frame size is 0x609 when the byte at 0x2c is nonzero
 // and 0x600 when it is zero (avcodec.prx decodeUtility, case 0x1003); the output size comes from
-// the byte at 0x2d and is 0x1000 or 0x2000. Setting the byte at 0x2c alone (to 1) fails Init on
-// hardware, so something else has to go with it.
+// the byte at 0x2d and is 0x1000 or 0x2000. Either byte fails Init in a game, though (see
+// sceAudiocodecInit).
 
 // MP3 (0x1002)
 // ------------------------------------------------
@@ -269,6 +272,7 @@ void __AudioCodecInit() {
 	// game around for the next one to find.
 	clearDecoders();
 	oldStateLoaded = false;
+	g_meBootMode = 2;
 }
 
 void __AudioCodecShutdown() {
@@ -405,8 +409,13 @@ static int __AudioCodecInitCommon(u32 ctxPtr, int codec, bool mono) {
 			// Refused before it gets to the ME.
 			return hleLogError(Log::ME, SCE_AVCODEC_ERROR_UNSUPPORTED, "bad AAC sample rate %d", ctx->fmt.aac.sampleRate);
 		}
-		// Either flag makes Init send another ME command (0x95, 0x97), and both fail.
-		if (ctx->fmt.aac.flag2c || ctx->fmt.aac.flag2d) {
+		// Either flag makes Init send another ME command (0x95, 0x97). Both fail with the ME booted
+		// for games (pspautotests audio/audiocodec/context), but the VSH's video player sets 0x2d on
+		// anything newer than a PSP-1000, and 0x2c for 24 and 48 kHz (msvideo_main_plugin.prx), and
+		// plays those files. It runs with the ME booted in mode 3, which differs from the games'
+		// mode 2 only by the argument to ME command 0x185 (me_wrapper.prx), so that's taken to be
+		// what turns them on.
+		if ((ctx->fmt.aac.flag2c || ctx->fmt.aac.flag2d) && g_meBootMode != 3) {
 			return MECall(hleLogError(Log::ME, SCE_AVCODEC_ERROR_UNSUPPORTED, "AAC flags %02x %02x", ctx->fmt.aac.flag2c, ctx->fmt.aac.flag2d), InitUs(codec, ctx));
 		}
 		primingFrames = 2;
@@ -935,12 +944,36 @@ void Register_sceAudiocodec() {
 	RegisterHLEModule("sceAudiocodec", ARRAY_SIZE(sceAudiocodec), sceAudiocodec);
 }
 
+// Reboots the ME with the firmware image for a mode, 1-4 (me_wrapper.prx). Its module_start boots
+// mode 2, which is what games run with, and the VSH asks for 3 through sceVshBridge. Modes 2 and 3
+// share an image and then send ME command 0x185 with 0 or 1. From the PSP-2000 on, every mode but 2
+// is taken as 3. Only the mode is modelled.
+static int sceMeBootStart(int mode) {
+	if (mode < 1 || mode > 4) {
+		return hleLogError(Log::ME, SCE_KERNEL_ERROR_INVALID_INDEX, "bad mode");
+	}
+	if (Memory::g_PSPModel != PSP_MODEL_FAT && mode != 2) {
+		mode = 3;
+	}
+	g_meBootMode = mode;
+	return hleLogDebug(Log::ME, 0);
+}
+
+const HLEFunction sceMeCore_driver[] = {
+	{0X5DFF5C50, &WrapI_I<sceMeBootStart>,               "sceMeBootStart",            'i', "i", HLE_KERNEL_SYSCALL },
+};
+
+void Register_sceMeCore_driver() {
+	RegisterHLEModule("sceMeCore_driver", ARRAY_SIZE(sceMeCore_driver), sceMeCore_driver);
+}
+
 void __sceAudiocodecDoState(PointerWrap &p){
-	auto s = p.Section("AudioList", 0, 3);
+	auto s = p.Section("AudioList", 0, 4);
 	if (!s) {
 		if (p.mode == PointerWrap::MODE_READ) {
 			clearDecoders();
 			oldStateLoaded = true;
+			g_meBootMode = 2;
 		}
 		return;
 	}
@@ -1017,4 +1050,9 @@ void __sceAudiocodecDoState(PointerWrap &p){
 	} else if (p.mode == PointerWrap::MODE_READ) {
 		g_primingFrames.clear();
 	}
+	if (s >= 4) {
+		Do(p, g_meBootMode);
+	} else {
+		g_meBootMode = 2;
+	}
 }

Core/HLE/sceAudiocodec.h

@@ -125,7 +125,8 @@ struct SceAudiocodecCodec {
 
 		// AAC (0x1003). The input frame size is 0x609 when flag2c is nonzero and 0x600 when it is
 		// zero, and the output size 0x2000 when flag2d is set (avcodec.prx decodeUtility). Init
-		// sends extra ME commands for either flag (0x95, 0x97), which fail on the firmware tested.
+		// sends extra ME commands for either flag (0x95, 0x97), which fail unless the ME was booted
+		// for the VSH (sceMeBootStart).
 		struct {
 			// 0x28  One of the AAC rates from 8000 to 96000 (not 12000 or 7350), or Init fails.
 			u32 sampleRate;
@@ -148,6 +149,7 @@ struct SceAudiocodecCodec {
 void __AudioCodecInit();
 void __AudioCodecShutdown();
 void Register_sceAudiocodec();
+void Register_sceMeCore_driver();
 void __sceAudiocodecDoState(PointerWrap &p);
 
 class AudioDecoder;

Core/HLE/sceVaudio.cpp

@@ -28,17 +28,26 @@
 // Ultra hacky Vaudio implementation. Not sure what the point of this API is.
 
 bool vaudioReserved = false;
+// For a channel reserved with sceVaudio_E8E78DC8, the parameter vaudio.prx hands the ME with each
+// buffer (ME command 0x125). -1 otherwise.
+static int vaudioMeParam = -1;
 
 void __VaudioInit() {
 	vaudioReserved = false;
+	vaudioMeParam = -1;
 }
 
 void __VaudioDoState(PointerWrap &p) {
-	auto s = p.Section("sceVaudio", 1);
+	auto s = p.Section("sceVaudio", 1, 2);
 	if (!s)
 		return;
 
 	Do(p, vaudioReserved);
+	if (s >= 2) {
+		Do(p, vaudioMeParam);
+	} else {
+		vaudioMeParam = -1;
+	}
 }
 
 static u32 sceVaudioChReserve(int sampleCount, int freq, int format) {
@@ -92,6 +101,7 @@ static u32 sceVaudioChRelease() {
 	// happily release a reservation that Output2 made. pspautotests calls that the "wrong
 	// release" and the hardware allows it.
 	vaudioReserved = false;
+	vaudioMeParam = -1;
 	if (!g_audioSRC.reserved) {
 		return SCE_ERROR_AUDIO_CHANNEL_NOT_RESERVED;
 	}
@@ -138,15 +148,39 @@ static u32 sceVaudioChReserveBuffering(int sampleCount, int freq, int format) {
 	return sceVaudioChReserve(sampleCount, freq, format);
 }
 
+// ChReserveBuffering, after setting up an effect on the ME (command 0x124, with the frequency and
+// sample count): vaudio.prx then passes every buffer through the ME with a parameter that starts at
+// 0x800, and ChRelease tears it down (0x126). The VSH's video player uses it, and only it changes
+// the parameter (sceVaudio_504E4745). Unknown what the effect is, so the audio goes out unchanged.
+static u32 sceVaudio_E8E78DC8(int sampleCount, int freq, int format) {
+	u32 result = sceVaudioChReserve(sampleCount, freq, format);
+	if (result == 0) {
+		vaudioMeParam = 0x800;
+	}
+	return result;
+}
+
+static u32 sceVaudio_504E4745(int param) {
+	// As vaudio.prx checks it.
+	if (vaudioMeParam < 0) {
+		return hleLogError(Log::sceAudio, 0x80000001, "not reserved through the ME");
+	}
+	if (param < 0x400 || param > 0x1000) {
+		return hleLogError(Log::sceAudio, SCE_KERNEL_ERROR_INVALID_VALUE, "out of range");
+	}
+	vaudioMeParam = param;
+	return hleLogDebug(Log::sceAudio, 0);
+}
+
 const HLEFunction sceVaudio[] = {
 	{0X8986295E, &WrapU_IU<sceVaudioOutputBlocking>, "sceVaudioOutputBlocking",     'x', "ix" },
 	{0X03B6807D, &WrapU_III<sceVaudioChReserve>,     "sceVaudioChReserve",          'x', "iii"},
 	{0X67585DFD, &WrapU_V<sceVaudioChRelease>,       "sceVaudioChRelease",          'x', ""   },
 	{0X346FBE94, &WrapU_II<sceVaudioSetEffectType>,  "sceVaudioSetEffectType",      'x', "ii" },
 	{0XCBD4AC51, &WrapU_I<sceVaudioSetAlcMode>,      "sceVaudioSetAlcMode",         'x', "i"  },
-	{0X504E4745, nullptr,                            "sceVaudio_504E4745",          '?', ""   },
+	{0X504E4745, &WrapU_I<sceVaudio_504E4745>,       "sceVaudio_504E4745",          'x', "x"  },
 	{0X27ACC20B, &WrapU_III<sceVaudioChReserveBuffering>, "sceVaudioChReserveBuffering", 'x', "iii"},
-	{0XE8E78DC8, nullptr,                            "sceVaudio_E8E78DC8",          '?', ""   },
+	{0XE8E78DC8, &WrapU_III<sceVaudio_E8E78DC8>,     "sceVaudio_E8E78DC8",          'x', "iii"},
 };
 
 void Register_sceVaudio() {