Merge pull request #22681 from hrydgard/mmio-jit

Compile kernel loads/stores (MMIO under the JITs), sceKernelRegisterIntrHandler, and an eret hardware test

Core/HLE/sceKernelInterrupt.cpp

@@ -191,6 +191,7 @@ SubIntrHandler* IntrHandler::get(int subIntrNum)
 void IntrHandler::clear()
 {
 	subIntrHandlers.clear();
+	setBaseHandler(0, 0);
 }
 
 int IntrHandler::queueUp(int subintr) {
@@ -212,12 +213,19 @@ int IntrHandler::queueUp(int subintr) {
 
 void IntrHandler::DoState(PointerWrap &p)
 {
-	auto s = p.Section("IntrHandler", 1);
+	auto s = p.Section("IntrHandler", 1, 2);
 	if (!s)
 		return;
 
 	Do(p, intrNumber);
 	Do<int, SubIntrHandler>(p, subIntrHandlers);
+	if (s >= 2) {
+		Do(p, baseHandler_);
+		Do(p, baseArg_);
+		Do(p, baseEnabled_);
+	} else {
+		setBaseHandler(0, 0);
+	}
 	if (p.mode == p.MODE_READ && (intrNumber < 0 || intrNumber >= PSP_NUMBER_INTERRUPTS)) {
 		ERROR_LOG(Log::sceIntc, "Savestate failure: invalid interrupt handler %d", intrNumber);
 		p.SetError(p.ERROR_FAILURE);
@@ -1039,6 +1047,46 @@ void Register_InterruptManager()
 }
 
 
+// What these return is from tests/intr/kernel/intrhandler, recorded on a PSP. Interrupts there go up to
+// 67, one more than PSP_NUMBER_INTERRUPTS, which the firmware's drivers use anyway.
+static int sceKernelRegisterIntrHandler(int intrNumber, u32 unknown, u32 handler, u32 handlerArg, u32 opt) {
+	if (intrNumber < 0 || intrNumber >= PSP_NUMBER_INTERRUPTS || handler == 0)
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE);
+	IntrHandler *intr = intrHandlers[intrNumber];
+	if (intr->hasBaseHandler())
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_FOUND_HANDLER);
+	intr->setBaseHandler(handler, handlerArg);
+	return hleLogDebug(Log::sceIntc, 0);
+}
+
+static int sceKernelReleaseIntrHandler(int intrNumber) {
+	if (intrNumber < 0 || intrNumber >= PSP_NUMBER_INTERRUPTS)
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_ILLEGAL_INTRCODE);
+	IntrHandler *intr = intrHandlers[intrNumber];
+	if (!intr->hasBaseHandler())
+		return hleLogError(Log::sceIntc, SCE_KERNEL_ERROR_NOTFOUND_HANDLER);
+	intr->setBaseHandler(0, 0);
+	return hleLogDebug(Log::sceIntc, 0);
+}
+
+static int SetIntrEnabled(int intrNumber, bool enable) {
+	if (intrNumber < 0 || intrNumber >= PSP_NUMBER_INTERRUPTS)
+		return SCE_KERNEL_ERROR_ILLEGAL_INTRCODE;
+	IntrHandler *intr = intrHandlers[intrNumber];
+	if (!intr->hasBaseHandler())
+		return SCE_KERNEL_ERROR_NOTFOUND_HANDLER;
+	intr->enableBase(enable);
+	return 0;
+}
+
+static int sceKernelEnableIntr(int intrNumber) {
+	return hleLogDebugOrError(Log::sceIntc, SetIntrEnabled(intrNumber, true));
+}
+
+static int sceKernelDisableIntr(int intrNumber) {
+	return hleLogDebugOrError(Log::sceIntc, SetIntrEnabled(intrNumber, false));
+}
+
 static int sceKernelIsIntrContext() {
 	return hleLogDebug(Log::sceKernel, __IsInInterrupt() ? 1 : 0);
 }
@@ -1065,15 +1113,10 @@ const HLEFunction InterruptManagerForKernel[] =
 	{0X05572A5F, &WrapV_V<sceKernelExitGame>,                  "sceKernelExitGame",                   'v', ""    ,HLE_KERNEL_SYSCALL },
 	{0X4AC57943, &WrapI_I<sceKernelRegisterExitCallback>,      "sceKernelRegisterExitCallback",       'i', "i"   ,HLE_KERNEL_SYSCALL },
 	{0XFE28C6D9, &WrapI_V<sceKernelIsIntrContext>,             "sceKernelIsIntrContext",              'i', ""    ,HLE_KERNEL_SYSCALL },
-	// NOT added on purpose, even though JPCSP implements all four: sceKernelRegisterIntrHandler
-	// (0x58DD8978), sceKernelReleaseIntrHandler (0xF987B1F0), sceKernelEnableIntr (0x4D6E7305)
-	// and sceKernelDisableIntr (0xD774BA45). JPCSP can honour them because it emulates the
-	// interrupt controller as MMIO; we dispatch the few interrupts we emulate ourselves (see
-	// __RegisterIntrHandler and its callers in sceGe/sceKernelAlarm/sceKernelVTimer) and have no
-	// way to run a guest handler for one. Stubbing them to return success is therefore a lie the
-	// real flash0 drivers act on - measured while booting the VSH, they make 31 such calls
-	// (interrupts 4, 12, 15-18, 20-24, 31), and the boot then stalls in GE list execution without
-	// ever starting a plugin module, where leaving them unresolved reaches the shell.
+	{0X58DD8978, &WrapI_IUUUU<sceKernelRegisterIntrHandler>,   "sceKernelRegisterIntrHandler",        'i', "ixxxx", HLE_KERNEL_SYSCALL },
+	{0XF987B1F0, &WrapI_I<sceKernelReleaseIntrHandler>,        "sceKernelReleaseIntrHandler",         'i', "i"   ,HLE_KERNEL_SYSCALL },
+	{0X4D6E7305, &WrapI_I<sceKernelEnableIntr>,                "sceKernelEnableIntr",                 'i', "i"   ,HLE_KERNEL_SYSCALL },
+	{0XD774BA45, &WrapI_I<sceKernelDisableIntr>,               "sceKernelDisableIntr",                'i', "i"   ,HLE_KERNEL_SYSCALL },
 };
 
 void Register_InterruptManagerForKernel()

Core/HLE/sceKernelInterrupt.h

@@ -129,12 +129,24 @@ public:
 	SubIntrHandler *get(int subIntrNum);
 	void clear();
 
+	// The handler a kernel driver registered for the whole interrupt (sceKernelRegisterIntrHandler).
+	// Nothing raises those interrupts yet, so it's only kept.
+	bool hasBaseHandler() const { return baseHandler_ != 0; }
+	void setBaseHandler(u32 handler, u32 arg) {
+		baseHandler_ = handler;
+		baseArg_ = arg;
+		baseEnabled_ = false;
+	}
+	void enableBase(bool enable) { baseEnabled_ = enable; }
 
 	void DoState(PointerWrap &p);
 
 private:
 	int intrNumber;
 	std::map<int, SubIntrHandler> subIntrHandlers;
+	u32 baseHandler_ = 0;
+	u32 baseArg_ = 0;
+	bool baseEnabled_ = false;
 	int entryCycles_ = 0;
 	int exitCycles_ = 0;
 };

Core/HW/MMIO.cpp

@@ -19,19 +19,12 @@
 
 #include "Common/Log.h"
 #include "Common/Serialize/Serializer.h"
-#include "Core/Core.h"
 #include "Core/HW/GpioMMIO.h"
 #include "Core/HW/MMIO.h"
-#include "Core/MemMap.h"
 #include "Core/MIPS/MIPS.h"
 
 namespace MMIO {
 
-// Unhandled reads return a distinctive poison value (not 0) so it's obvious in a live trace when
-// some later computation's input traces back to an unimplemented register, rather than looking
-// like an ordinary, legitimate zero.
-constexpr u32 UNKNOWN_MMIO_POISON = 0x1337BEEF;
-
 static std::vector<MMIODevice *> g_devices;
 
 void Init() {
@@ -65,27 +58,16 @@ static MMIODevice *FindDevice(u32 address) {
 	return nullptr;
 }
 
-static bool CheckKernelMode(MIPSState *mips, u32 address, int size, MemoryExceptionType type) {
-	if (Memory::IsKernelCodeAddress(mips->pc))
-		return true;
-	Core_MemoryException(address, size, mips->pc, type, "Kernel mode only");
-	return false;
-}
-
 u32 Read(MIPSState *mips, u32 address, int size) {
-	if (!CheckKernelMode(mips, address, size, MemoryExceptionType::READ_WORD))
-		return UNKNOWN_MMIO_POISON;
 	MMIODevice *device = FindDevice(address);
 	u32 value;
 	if (device && device->Read(mips, address - device->Base(), size, &value))
 		return value;
 	WARN_LOG(Log::CPU, "Unhandled MMIO Read%d at %08x%s%s", size * 8, address, device ? " in " : "", device ? device->Name() : "");
-	return UNKNOWN_MMIO_POISON;
+	return UNHANDLED_READ;
 }
 
 void Write(MIPSState *mips, u32 address, int size, u32 value) {
-	if (!CheckKernelMode(mips, address, size, MemoryExceptionType::WRITE_WORD))
-		return;
 	MMIODevice *device = FindDevice(address);
 	if (device && device->Write(mips, address - device->Base(), size, value))
 		return;

Core/HW/MMIO.h

@@ -51,13 +51,18 @@ private:
 
 namespace MMIO {
 
+// What an unhandled read returns. Distinctive (not 0) so it's obvious in a live trace when some later
+// computation's input traces back to an unimplemented register, rather than looking like an
+// ordinary, legitimate zero.
+constexpr u32 UNHANDLED_READ = 0x1337BEEF;
+
 // Registers the devices, in their power-on state.
 void Init();
 void Shutdown();
 void DoState(PointerWrap &p);
 
-// For an address Memory::IsMMIOAccess() accepts. size is 1, 2 or 4. An access no device handles is
-// logged, and reads as poison.
+// For an address Memory::IsMMIOAccess() accepts, from kernel code (the caller checks). size is 1, 2
+// or 4. An access no device handles is logged, and reads as UNHANDLED_READ.
 u32 Read(MIPSState *mips, u32 address, int size);
 void Write(MIPSState *mips, u32 address, int size, u32 value);
 

Core/MIPS/IR/IRCompLoadStore.cpp

@@ -53,18 +53,28 @@ namespace MIPSComp {
 
 		CheckMemoryBreakpoint(rs, offset);
 
-		if (js.kernelMode) {
-			// Send all memory accesses to the interpreter in kernel mode.
-			// TODO: Do something faster - but it hardly matters, currently this is VSH-only.
-			DISABLE;
-			return;
-		}
-
 		if (((op >> 29) & 1) == 0 && rt == MIPS_REG_ZERO) {
 			// Don't load anything into $zr
 			return;
 		}
 
+		if (js.kernelMode) {
+			// Kernel code may reach hardware registers. ResolveKernelMemory turns the plain memory
+			// accesses back into ordinary ones where it can tell. The rest go to the interpreter.
+			switch (o) {
+			case 32: ir.Write(IROp::LoadKernel8Ext, rt, rs, 0, offset); return;
+			case 33: ir.Write(IROp::LoadKernel16Ext, rt, rs, 0, offset); return;
+			case 35: ir.Write(IROp::LoadKernel32, rt, rs, 0, offset); return;
+			case 36: ir.Write(IROp::LoadKernel8, rt, rs, 0, offset); return;
+			case 37: ir.Write(IROp::LoadKernel16, rt, rs, 0, offset); return;
+			case 40: ir.Write(IROp::StoreKernel8, rt, rs, 0, offset); return;
+			case 41: ir.Write(IROp::StoreKernel16, rt, rs, 0, offset); return;
+			case 43: ir.Write(IROp::StoreKernel32, rt, rs, 0, offset); return;
+			default:
+				DISABLE;
+			}
+		}
+
 		switch (o) {
 			// Load
 		case 35:

Core/MIPS/IR/IRFrontend.cpp

@@ -347,6 +347,7 @@ void IRFrontend::DoJit(u32 em_address, std::vector<IRInst> &instructions, u32 &m
 			&RemoveLoadStoreLeftRight,
 			&OptimizeFPMoves,
 			&PropagateConstants,
+			&ResolveKernelMemory,
 			&PurgeTemps,
 			&ReduceVec4Flush,
 			&OptimizeLoadsAfterStores,

Core/MIPS/IR/IRInst.cpp

@@ -95,6 +95,15 @@ static const IRMeta irMeta[] = {
 	{ IROp::Store32Conditional, "Store32Conditional", "GGC", IRFLAG_SRC3DST },
 	{ IROp::StoreFloat, "StoreFloat", "FGC", IRFLAG_SRC3 },
 	{ IROp::StoreVec4, "StoreVec4", "VGC", IRFLAG_SRC3 },
+	// Barriers: a hardware register access has side effects, and mustn't move or merge.
+	{ IROp::LoadKernel8, "LoadKernel8", "GGC", IRFLAG_BARRIER },
+	{ IROp::LoadKernel8Ext, "LoadKernel8Ext", "GGC", IRFLAG_BARRIER },
+	{ IROp::LoadKernel16, "LoadKernel16", "GGC", IRFLAG_BARRIER },
+	{ IROp::LoadKernel16Ext, "LoadKernel16Ext", "GGC", IRFLAG_BARRIER },
+	{ IROp::LoadKernel32, "LoadKernel32", "GGC", IRFLAG_BARRIER },
+	{ IROp::StoreKernel8, "StoreKernel8", "GGC", IRFLAG_SRC3 | IRFLAG_BARRIER },
+	{ IROp::StoreKernel16, "StoreKernel16", "GGC", IRFLAG_SRC3 | IRFLAG_BARRIER },
+	{ IROp::StoreKernel32, "StoreKernel32", "GGC", IRFLAG_SRC3 | IRFLAG_BARRIER },
 	{ IROp::FAdd, "FAdd", "FFF" },
 	{ IROp::FSub, "FSub", "FFF" },
 	{ IROp::FMul, "FMul", "FFF" },

Core/MIPS/IR/IRInst.h

@@ -110,6 +110,17 @@ enum class IROp : uint8_t {
 	StoreFloat,
 	StoreVec4,
 
+	// Kernel code's loads and stores, which may reach hardware registers (MMIO). See
+	// ResolveKernelMemory.
+	LoadKernel8,
+	LoadKernel8Ext,
+	LoadKernel16,
+	LoadKernel16Ext,
+	LoadKernel32,
+	StoreKernel8,
+	StoreKernel16,
+	StoreKernel32,
+
 	Ext8to32,
 	Ext16to32,
 	ReverseBits,

Core/MIPS/IR/IRInterpreter.cpp

@@ -15,6 +15,7 @@
 #include "Core/Debugger/Breakpoints.h"
 #include "Core/HLE/HLE.h"
 #include "Core/HLE/ReplaceTables.h"
+#include "Core/HW/MMIO.h"
 #include "Core/MemMap.h"
 #include "Core/MIPS/MIPS.h"
 #include "Core/MIPS/MIPSTables.h"
@@ -70,6 +71,51 @@ u32 IRRunMemCheck(u32 pc, u32 addr) {
 	return coreState != CORE_RUNNING_CPU ? 1 : 0;
 }
 
+// LoadKernel* and StoreKernel*: memory, or a hardware register, like the interpreter's loads and stores.
+template <int size>
+static bool IsValidAlignedAddress(u32 addr) {
+	switch (size) {
+	case 1: return Memory::IsValidAddress(addr);
+	case 2: return Memory::IsValid2AlignedAddress(addr);
+	default: return Memory::IsValid4AlignedAddress(addr);
+	}
+}
+
+template <int size, bool sign>
+static u32 RunKernelLoad(MIPSState *mips, u32 addr) {
+	u32 value;
+	if (IsValidAlignedAddress<size>(addr)) {
+		switch (size) {
+		case 1: value = Memory::ReadUnchecked_U8(addr); break;
+		case 2: value = Memory::ReadUnchecked_U16(addr); break;
+		default: value = Memory::ReadUnchecked_U32(addr); break;
+		}
+	} else if (Memory::IsMMIOAccess(addr)) {
+		value = MMIO::Read(mips, addr, size);
+	} else {
+		Core_MemoryException(addr, size, mips->pc, MemoryExceptionType::READ_WORD, "kernel load");
+		return 0;
+	}
+	if (sign)
+		value = size == 1 ? SignExtend8ToU32(value) : SignExtend16ToU32(value);
+	return value;
+}
+
+template <int size>
+static void RunKernelStore(MIPSState *mips, u32 addr, u32 value) {
+	if (IsValidAlignedAddress<size>(addr)) {
+		switch (size) {
+		case 1: Memory::WriteUnchecked_U8((u8)value, addr); break;
+		case 2: Memory::WriteUnchecked_U16((u16)value, addr); break;
+		default: Memory::WriteUnchecked_U32(value, addr); break;
+		}
+	} else if (Memory::IsMMIOAccess(addr)) {
+		MMIO::Write(mips, addr, size, value);
+	} else {
+		Core_MemoryException(addr, size, mips->pc, MemoryExceptionType::WRITE_WORD, "kernel store");
+	}
+}
+
 // With GCC and Clang, each op jumps straight to the next op's handler through a table ("threaded"
 // dispatch), which predicts much better than the one shared indirect jump of a switch. Other
 // compilers get the switch. Ops missing from the table fall back to the switch, so they still work.
@@ -158,6 +204,14 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
 		dispatch[(int)IROp::StoreFloat] = &&L_StoreFloat;
 		dispatch[(int)IROp::LoadVec4] = &&L_LoadVec4;
 		dispatch[(int)IROp::StoreVec4] = &&L_StoreVec4;
+		dispatch[(int)IROp::LoadKernel8] = &&L_LoadKernel8;
+		dispatch[(int)IROp::LoadKernel8Ext] = &&L_LoadKernel8Ext;
+		dispatch[(int)IROp::LoadKernel16] = &&L_LoadKernel16;
+		dispatch[(int)IROp::LoadKernel16Ext] = &&L_LoadKernel16Ext;
+		dispatch[(int)IROp::LoadKernel32] = &&L_LoadKernel32;
+		dispatch[(int)IROp::StoreKernel8] = &&L_StoreKernel8;
+		dispatch[(int)IROp::StoreKernel16] = &&L_StoreKernel16;
+		dispatch[(int)IROp::StoreKernel32] = &&L_StoreKernel32;
 		dispatch[(int)IROp::Vec4Init] = &&L_Vec4Init;
 		dispatch[(int)IROp::Vec4Shuffle] = &&L_Vec4Shuffle;
 		dispatch[(int)IROp::Vec4Blend] = &&L_Vec4Blend;
@@ -467,6 +521,31 @@ u32 IRInterpret(MIPSState *mips, const IRInst *inst, const IRInst *chainArena) {
 			IR_NEXT;
 		}
 
+		IR_CASE(LoadKernel8)
+			mips->r[inst->dest] = RunKernelLoad<1, false>(mips, mips->r[inst->src1] + inst->constant);
+			IR_NEXT;
+		IR_CASE(LoadKernel8Ext)
+			mips->r[inst->dest] = RunKernelLoad<1, true>(mips, mips->r[inst->src1] + inst->constant);
+			IR_NEXT;
+		IR_CASE(LoadKernel16)
+			mips->r[inst->dest] = RunKernelLoad<2, false>(mips, mips->r[inst->src1] + inst->constant);
+			IR_NEXT;
+		IR_CASE(LoadKernel16Ext)
+			mips->r[inst->dest] = RunKernelLoad<2, true>(mips, mips->r[inst->src1] + inst->constant);
+			IR_NEXT;
+		IR_CASE(LoadKernel32)
+			mips->r[inst->dest] = RunKernelLoad<4, false>(mips, mips->r[inst->src1] + inst->constant);
+			IR_NEXT;
+		IR_CASE(StoreKernel8)
+			RunKernelStore<1>(mips, mips->r[inst->src1] + inst->constant, mips->r[inst->src3]);
+			IR_NEXT;
+		IR_CASE(StoreKernel16)
+			RunKernelStore<2>(mips, mips->r[inst->src1] + inst->constant, mips->r[inst->src3]);
+			IR_NEXT;
+		IR_CASE(StoreKernel32)
+			RunKernelStore<4>(mips, mips->r[inst->src1] + inst->constant, mips->r[inst->src3]);
+			IR_NEXT;
+
 		IR_CASE(Vec4Init)
 		{
 			memcpy(&mips->f[inst->dest], vec4InitValues[inst->src1], 4 * sizeof(float));

Core/MIPS/IR/IRNativeCommon.cpp

@@ -463,6 +463,18 @@ void IRNativeBackend::CompileIRInst(IRInst inst) {
 		CompIR_Interpret(inst);
 		break;
 
+	case IROp::LoadKernel8:
+	case IROp::LoadKernel8Ext:
+	case IROp::LoadKernel16:
+	case IROp::LoadKernel16Ext:
+	case IROp::LoadKernel32:
+	case IROp::StoreKernel8:
+	case IROp::StoreKernel16:
+	case IROp::StoreKernel32:
+		// Kernel code only, and mostly hardware registers once ResolveKernelMemory has run.
+		CompIR_Generic(inst);
+		break;
+
 	case IROp::Syscall:
 	case IROp::SyscallUnresolved:
 	case IROp::CallReplacement:

Core/MIPS/IR/IRPassSimplify.cpp

@@ -691,6 +691,27 @@ bool PropagateConstants(const IRWriter &in, IRWriter &out, const IROptions &opts
 			}
 			break;
 
+		case IROp::LoadKernel8:
+		case IROp::LoadKernel8Ext:
+		case IROp::LoadKernel16:
+		case IROp::LoadKernel16Ext:
+		case IROp::LoadKernel32:
+		case IROp::StoreKernel8:
+		case IROp::StoreKernel16:
+		case IROp::StoreKernel32:
+			// Barriers, so everything is written out, but they can't change the GPRs, so the values
+			// stay known. A constant address goes in the op, for ResolveKernelMemory.
+			if (gpr.IsImm(inst.src1)) {
+				const u32 addr = gpr.GetImm(inst.src1) + inst.constant;
+				gpr.FlushAll(true);
+				if ((GetIRMeta(inst.op)->flags & IRFLAG_SRC3) == 0)
+					gpr.MapDirty(inst.dest);
+				out.Write(inst.op, inst.dest, 0, 0, addr);
+				break;
+			}
+			gpr.FlushAll(true);
+			goto doDefault;
+
 		case IROp::Load8:
 		case IROp::Load8Ext:
 		case IROp::Load16:
@@ -2373,6 +2394,42 @@ bool OptimizeLoadsAfterStores(const IRWriter &in, IRWriter &out, const IROptions
 	return false;
 }
 
+// The plain op for a kernel load or store, and its size, or Nop.
+static IROp PlainMemoryOp(IROp op, int *size) {
+	switch (op) {
+	case IROp::LoadKernel8: *size = 1; return IROp::Load8;
+	case IROp::LoadKernel8Ext: *size = 1; return IROp::Load8Ext;
+	case IROp::LoadKernel16: *size = 2; return IROp::Load16;
+	case IROp::LoadKernel16Ext: *size = 2; return IROp::Load16Ext;
+	case IROp::LoadKernel32: *size = 4; return IROp::Load32;
+	case IROp::StoreKernel8: *size = 1; return IROp::Store8;
+	case IROp::StoreKernel16: *size = 2; return IROp::Store16;
+	case IROp::StoreKernel32: *size = 4; return IROp::Store32;
+	default: return IROp::Nop;
+	}
+}
+
+bool ResolveKernelMemory(const IRWriter &in, IRWriter &out, const IROptions &opts) {
+	CONDITIONAL_DISABLE;
+	for (IRInst inst : in.GetInstructions()) {
+		int size = 0;
+		const IROp plain = PlainMemoryOp(inst.op, &size);
+		if (plain != IROp::Nop && inst.src1 == MIPS_REG_ZERO) {
+			const u32 addr = inst.constant;
+			bool valid;
+			switch (size) {
+			case 1: valid = Memory::IsValidAddress(addr); break;
+			case 2: valid = Memory::IsValid2AlignedAddress(addr); break;
+			default: valid = Memory::IsValid4AlignedAddress(addr); break;
+			}
+			if (valid)
+				inst.op = plain;
+		}
+		out.Write(inst);
+	}
+	return false;
+}
+
 bool OptimizeForInterpreter(const IRWriter &in, IRWriter &out, const IROptions &opts) {
 	CONDITIONAL_DISABLE;
 	// This tells us to skip an AND op that has been optimized out.

Core/MIPS/IR/IRPassSimplify.h

@@ -18,4 +18,7 @@ bool ApplyMemoryValidation(const IRWriter &in, IRWriter &out, const IROptions &o
 bool ReduceVec4Flush(const IRWriter &in, IRWriter &out, const IROptions &opts);
 
 bool OptimizeLoadsAfterStores(const IRWriter &in, IRWriter &out, const IROptions &opts);
+// Kernel code's loads and stores at a constant address become plain ones where that's memory, so only
+// hardware registers (and addresses only known when it runs) stay LoadKernel*/StoreKernel*.
+bool ResolveKernelMemory(const IRWriter &in, IRWriter &out, const IROptions &opts);
 bool OptimizeForInterpreter(const IRWriter &in, IRWriter &out, const IROptions &opts);

Core/MIPS/Interpreter.cpp

@@ -99,6 +99,23 @@ int MIPS_InterpretSingleStep(MIPSState *mips) {
 	return 1;
 }
 
+// Hardware registers, for kernel code only.
+static u32 ReadMMIO(MIPSState *mips, u32 addr, int size) {
+	if (!Memory::IsKernelCodeAddress(mips->pc)) {
+		Core_MemoryException(addr, size, mips->pc, MemoryExceptionType::READ_WORD, "Kernel mode only");
+		return MMIO::UNHANDLED_READ;
+	}
+	return MMIO::Read(mips, addr, size);
+}
+
+static void WriteMMIO(MIPSState *mips, u32 addr, int size, u32 value) {
+	if (!Memory::IsKernelCodeAddress(mips->pc)) {
+		Core_MemoryException(addr, size, mips->pc, MemoryExceptionType::WRITE_WORD, "Kernel mode only");
+		return;
+	}
+	MMIO::Write(mips, addr, size, value);
+}
+
 namespace MIPSInt {
 	void Int_Cache(MIPSState *mips, MIPSOpcode op) {
 		int imm = SignExtend16ToS32(op);
@@ -356,7 +373,7 @@ namespace MIPSInt {
 			if (rt != 0) {
 				if (!Memory::IsValid4AlignedAddress(addr)) {
 					if (Memory::IsMMIOAccess(addr)) {
-						R(rt) = MMIO::Read(mips, addr, 4);
+						R(rt) = ReadMMIO(mips, addr, 4);
 					} else {
 						Core_MemoryException(addr, 4, PC, MemoryExceptionType::READ_WORD, "ll");
 						R(rt) = 0;
@@ -371,7 +388,7 @@ namespace MIPSInt {
 			if (mips->llBit) {
 				if (!Memory::IsValid4AlignedAddress(addr)) {
 					if (Memory::IsMMIOAccess(addr))
-						MMIO::Write(mips, addr, 4, R(rt));
+						WriteMMIO(mips, addr, 4, R(rt));
 					else
 						Core_MemoryException(addr, 4, PC, MemoryExceptionType::WRITE_WORD, "sc");
 				} else {
@@ -448,7 +465,7 @@ namespace MIPSInt {
 			if (!Memory::IsValidAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a read from MMIO, so we can handle it.
-					R(rt) = SignExtend8ToU32(MMIO::Read(mips, addr, 1));
+					R(rt) = SignExtend8ToU32(ReadMMIO(mips, addr, 1));
 					break;
 				}
 				Core_MemoryException(addr, 1, PC, MemoryExceptionType::READ_WORD, "lb");
@@ -461,7 +478,7 @@ namespace MIPSInt {
 			if (!Memory::IsValid2AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a read from MMIO, so we can handle it.
-					R(rt) = SignExtend16ToU32(MMIO::Read(mips, addr, 2));
+					R(rt) = SignExtend16ToU32(ReadMMIO(mips, addr, 2));
 					break;
 				}
 				Core_MemoryException(addr, 2, PC, MemoryExceptionType::READ_WORD, "lh");
@@ -474,7 +491,7 @@ namespace MIPSInt {
 			if (!Memory::IsValid4AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a read from MMIO, so we can handle it.
-					R(rt) = MMIO::Read(mips, addr, 4);
+					R(rt) = ReadMMIO(mips, addr, 4);
 					break;
 				}
 				Core_MemoryException(addr, 4, PC, MemoryExceptionType::READ_WORD, "lw");
@@ -487,7 +504,7 @@ namespace MIPSInt {
 			if (!Memory::IsValidAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a read from MMIO, so we can handle it.
-					R(rt) = MMIO::Read(mips, addr, 1);
+					R(rt) = ReadMMIO(mips, addr, 1);
 					break;
 				}
 				Core_MemoryException(addr, 1, PC, MemoryExceptionType::READ_WORD, "lbu");
@@ -500,7 +517,7 @@ namespace MIPSInt {
 			if (!Memory::IsValid2AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a read from MMIO, so we can handle it.
-					R(rt) = MMIO::Read(mips, addr, 2);
+					R(rt) = ReadMMIO(mips, addr, 2);
 					break;
 				}
 				Core_MemoryException(addr, 2, PC, MemoryExceptionType::READ_WORD, "lhu");
@@ -513,7 +530,7 @@ namespace MIPSInt {
 			if (!Memory::IsValidAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a write to MMIO, so we can handle it.
-					MMIO::Write(mips, addr, 1, (u8)R(rt));
+					WriteMMIO(mips, addr, 1, (u8)R(rt));
 					break;
 				}
 				Core_MemoryException(addr, 1, PC, MemoryExceptionType::WRITE_WORD, "sb");
@@ -525,7 +542,7 @@ namespace MIPSInt {
 			if (!Memory::IsValid2AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a write to MMIO, so we can handle it.
-					MMIO::Write(mips, addr, 2, (u16)R(rt));
+					WriteMMIO(mips, addr, 2, (u16)R(rt));
 					break;
 				}
 				Core_MemoryException(addr, 2, PC, MemoryExceptionType::WRITE_WORD, "sh");
@@ -537,7 +554,7 @@ namespace MIPSInt {
 			if (!Memory::IsValid4AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
 					// This is a write to MMIO, so we can handle it.
-					MMIO::Write(mips, addr, 4, R(rt));
+					WriteMMIO(mips, addr, 4, R(rt));
 					break;
 				}
 				Core_MemoryException(addr, 4, PC, MemoryExceptionType::WRITE_WORD, "sw");
@@ -555,7 +572,7 @@ namespace MIPSInt {
 				if (Memory::IsValidAddress(addr)) {
 					mem = Memory::ReadUnchecked_U32(addr & 0xfffffffc);
 				} else if (Memory::IsMMIOAccess(addr)) {
-					mem = MMIO::Read(mips, addr & 0xfffffffc, 4);
+					mem = ReadMMIO(mips, addr & 0xfffffffc, 4);
 				} else {
 					Core_MemoryException(addr, 4, PC, MemoryExceptionType::READ_WORD, "lwl");
 				}
@@ -572,7 +589,7 @@ namespace MIPSInt {
 				if (Memory::IsValidAddress(addr)) {
 					mem = Memory::ReadUnchecked_U32(addr & 0xfffffffc);
 				} else if (Memory::IsMMIOAccess(addr)) {
-					mem = MMIO::Read(mips, addr & 0xfffffffc, 4);
+					mem = ReadMMIO(mips, addr & 0xfffffffc, 4);
 				} else {
 					Core_MemoryException(addr, 4, PC, MemoryExceptionType::READ_WORD, "lwr");
 				}
@@ -628,7 +645,7 @@ namespace MIPSInt {
 		case 49:
 			if (!Memory::IsValid4AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
-					FI(ft) = MMIO::Read(mips, addr, 4);
+					FI(ft) = ReadMMIO(mips, addr, 4);
 					break;
 				}
 				Core_MemoryException(addr, 4, PC, MemoryExceptionType::READ_WORD, "lwc1");
@@ -640,7 +657,7 @@ namespace MIPSInt {
 		case 57:
 			if (!Memory::IsValid4AlignedAddress(addr)) {
 				if (Memory::IsMMIOAccess(addr)) {
-					MMIO::Write(mips, addr, 4, FI(ft));
+					WriteMMIO(mips, addr, 4, FI(ft));
 					break;
 				}
 				Core_MemoryException(addr, 4, PC, MemoryExceptionType::WRITE_WORD, "swc1");
@@ -763,8 +780,8 @@ namespace MIPSInt {
 		PC += 4;
 	}
 
-	// Return from an exception, as MIPS32 defines it (not checked on a PSP): to ErrorEPC (30) if
-	// Status.ERL is set, clearing it, otherwise to EPC (14), clearing Status.EXL. No delay slot.
+	// Return from an exception, as MIPS32 defines it (cpu/cop0/eret): to ErrorEPC (30) if Status.ERL
+	// is set, clearing only that, otherwise to EPC (14), clearing Status.EXL. No delay slot.
 	void Int_Eret(MIPSState *mips, MIPSOpcode op) {
 		u32 &status = mips->cop0[12];
 		if (status & 4) {

test.py

@@ -85,6 +85,7 @@ class Command(object):
 # These have worked and should keep working always - regression tests.
 # -g flag runs these.
 tests_good = [
+  "cpu/cop0/eret",
   "cpu/cpu_alu/cpu_alu",
   "cpu/cpu_alu/cpu_branch",
   "cpu/cpu_alu/branch_link_ra",
@@ -321,6 +322,7 @@ tests_good = [
   "hash/sha1ctx",
   "hle/check_not_used_uids",
   "intr/intr",
+  "intr/kernel/intrhandler",
   "intr/mfic",
   "intr/enablesub",
   "intr/registersub",

unittest/JitHarness.cpp

@@ -513,8 +513,9 @@ bool TestCPUEret() {
 }
 
 // Kernel code reaching the GPIO block's registers on every core: a write, an acknowledge clearing a
-// status bit, reads back through lw and lwc1, and poison from a register nothing handles. Then the
-// registers through a savestate.
+// status bit, reads back through lw and lwc1, and poison from a register nothing handles. Kernel
+// RAM too, at a constant address (which the IR makes a plain access) and through a register. Then
+// the registers through a savestate.
 bool TestMMIO() {
 	SetupJitHarness();
 	MMIO::Init();
@@ -523,6 +524,9 @@ bool TestMMIO() {
 	const u32 base = 0x88000100;
 	auto sw = [](int rt, int rs, int imm) { return (0x2Bu << 26) | (rs << 21) | (rt << 16) | (imm & 0xFFFF); };
 	auto lwc1 = [](int ft, int rs, int imm) { return (0x31u << 26) | (rs << 21) | (ft << 16) | (imm & 0xFFFF); };
+	auto sb = [](int rt, int rs, int imm) { return (0x28u << 26) | (rs << 21) | (rt << 16) | (imm & 0xFFFF); };
+	auto lb = [](int rt, int rs, int imm) { return (0x20u << 26) | (rs << 21) | (rt << 16) | (imm & 0xFFFF); };
+	const u32 ram = 0x88001000;
 	const u32 code[] = {
 		MIPS_MAKE_LUI(MIPS_REG_T0, 0xBE24),  // GPIO
 		MIPS_MAKE_ORI(MIPS_REG_T1, MIPS_REG_ZERO, 0x30),
@@ -533,6 +537,12 @@ bool TestMMIO() {
 		lwc1(0, MIPS_REG_T0, 0x20),
 		MIPS_MAKE_LUI(MIPS_REG_T2, 0xBC10),
 		MIPS_MAKE_LW(MIPS_REG_V1, MIPS_REG_T2, 0),
+		MIPS_MAKE_LUI(MIPS_REG_T3, ram >> 16),
+		sw(MIPS_REG_T1, MIPS_REG_T3, ram & 0xFFFF),
+		MIPS_MAKE_LW(MIPS_REG_A0, MIPS_REG_T3, ram & 0xFFFF),
+		MIPS_MAKE_ORI(MIPS_REG_T4, MIPS_REG_ZERO, 0x80),
+		sb(MIPS_REG_T4, MIPS_REG_A1, 4),  // a1 is ram, unknown to the IR.
+		lb(MIPS_REG_A2, MIPS_REG_A1, 4),
 		MIPS_MAKE_SYSCALL("UnitTestFakeSyscalls", "UnitTestTerminator"),
 		MIPS_MAKE_BREAK(1),
 	};
@@ -550,17 +560,42 @@ bool TestMMIO() {
 		MMIO::Init();
 		currentMIPS->r[MIPS_REG_V0] = 0;
 		currentMIPS->r[MIPS_REG_V1] = 0;
+		currentMIPS->r[MIPS_REG_A0] = 0;
+		currentMIPS->r[MIPS_REG_A1] = ram;
+		currentMIPS->r[MIPS_REG_A2] = 0;
 		currentMIPS->fi[0] = 0;
 		currentMIPS->pc = base;
 		coreState = CORE_RUNNING_CPU;
 		while (coreState == CORE_RUNNING_CPU)
 			mipsr4k.RunLoopUntil(CoreTiming::GetTicks(currentMIPS) + 1000000);
 
-		if (currentMIPS->r[MIPS_REG_V0] != 0x20 || currentMIPS->fi[0] != 0x20 || currentMIPS->r[MIPS_REG_V1] != 0x1337BEEF) {
-			printf("MMIO on core %d: v0=%08x f0=%08x v1=%08x, expected 00000020, 00000020, 1337beef\n", (int)core,
-				currentMIPS->r[MIPS_REG_V0], currentMIPS->fi[0], currentMIPS->r[MIPS_REG_V1]);
+		if (currentMIPS->r[MIPS_REG_V0] != 0x20 || currentMIPS->fi[0] != 0x20 || currentMIPS->r[MIPS_REG_V1] != MMIO::UNHANDLED_READ ||
+			currentMIPS->r[MIPS_REG_A0] != 0x10 || currentMIPS->r[MIPS_REG_A2] != 0xFFFFFF80) {
+			printf("MMIO on core %d: v0=%08x f0=%08x v1=%08x a0=%08x a2=%08x, expected 00000020, 00000020, %08x, 00000010, ffffff80\n", (int)core,
+				currentMIPS->r[MIPS_REG_V0], currentMIPS->fi[0], currentMIPS->r[MIPS_REG_V1], currentMIPS->r[MIPS_REG_A0], currentMIPS->r[MIPS_REG_A2], MMIO::UNHANDLED_READ);
 			success = false;
 		}
+
+		if (mipsr4k.jit) {
+			// The registers stay checked accesses, the constant RAM ones become plain.
+			JitBlockCacheDebugInterface *blocks = mipsr4k.jit->GetBlockCacheDebugInterface();
+			const JitBlockDebugInfo info = blocks->GetBlockDebugInfo(blocks->GetBlockNumberFromStartAddress(base));
+			int kernelOps = 0, plainOps = 0;
+			for (const std::string &line : info.irDisasm) {
+				if (line.find("LoadKernel") != std::string::npos || line.find("StoreKernel") != std::string::npos)
+					kernelOps++;
+				else if (line.find("Load32 ") != std::string::npos || line.find("Store32 ") != std::string::npos)
+					plainOps++;
+			}
+			// sw/sw/lw/lw to registers, sb/lb through a1. lwc1 is left to the interpreter. The RAM store
+			// is plain, and the load after it plain too, or gone (the value forwarded).
+			if (kernelOps != 6 || plainOps < 1) {
+				printf("MMIO on core %d: %d kernel and %d plain loads/stores, expected 6 and at least 1\n", (int)core, kernelOps, plainOps);
+				for (const std::string &line : info.irDisasm)
+					printf("  %s\n", line.c_str());
+				success = false;
+			}
+		}
 	}
 	mipsr4k.UpdateCore(CPUCore::INTERPRETER);