From be84bc9934a006e8f1d170ceeea3b08894040e89 Mon Sep 17 00:00:00 2001
From: "Unknown W. Brackets" <checkins@unknownbrackets.org>
Date: Sun, 1 Jan 2023 06:59:14 -0800
Subject: [PATCH] softgpu: Detect binner alloc fail and bail.

---
 Core/System.cpp                |  8 ++++++++
 GPU/GPU.cpp                    | 11 ++++++++++-
 GPU/GPU.h                      |  1 +
 GPU/GPUCommon.h                |  3 +++
 GPU/GPUInterface.h             |  1 +
 GPU/Software/SoftGpu.cpp       | 10 +++++++++-
 GPU/Software/SoftGpu.h         |  1 +
 GPU/Software/TransformUnit.cpp |  6 ++++++
 GPU/Software/TransformUnit.h   |  2 ++
 9 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/Core/System.cpp b/Core/System.cpp
index 35da73d353..8303f08a15 100644
--- a/Core/System.cpp
+++ b/Core/System.cpp
@@ -487,6 +487,14 @@ bool PSP_InitUpdate(std::string *error_string) {
 	if (pspIsInited) {
 		Core_NotifyLifecycle(CoreLifecycle::START_COMPLETE);
 		pspIsRebooting = false;
+
+		// If GPU init failed during IsReady checks, bail.
+		if (!GPU_IsStarted()) {
+			*error_string = "Unable to initialize rendering engine.";
+			pspIsRebooting = false;
+			PSP_Shutdown();
+			return true;
+		}
 	}
 	return pspIsInited;
 }
diff --git a/GPU/GPU.cpp b/GPU/GPU.cpp
index 9ff5b8fd35..abfe5a5a0b 100644
--- a/GPU/GPU.cpp
+++ b/GPU/GPU.cpp
@@ -58,6 +58,12 @@ bool GPU_IsReady() {
 	return false;
 }
 
+bool GPU_IsStarted() {
+	if (gpu)
+		return gpu->IsReady() && gpu->IsStarted();
+	return false;
+}
+
 bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw) {
 	const auto &gpuCore = PSP_CoreParameter().gpuCore;
 	_assert_(draw || gpuCore == GPUCORE_SOFTWARE);
@@ -106,7 +112,10 @@ bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw) {
 #endif
 	}
 
-	return gpu != NULL;
+	if (gpu && gpu->IsReady() && !gpu->IsStarted())
+		SetGPU<SoftGPU>(nullptr);
+
+	return gpu != nullptr;
 #endif
 }
 #ifdef USE_CRT_DBG
diff --git a/GPU/GPU.h b/GPU/GPU.h
index 1fd455c662..7ff35b1401 100644
--- a/GPU/GPU.h
+++ b/GPU/GPU.h
@@ -147,4 +147,5 @@ namespace Draw {
 
 bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw);
 bool GPU_IsReady();
+bool GPU_IsStarted();
 void GPU_Shutdown();
diff --git a/GPU/GPUCommon.h b/GPU/GPUCommon.h
index 28c09034ed..9529be025e 100644
--- a/GPU/GPUCommon.h
+++ b/GPU/GPUCommon.h
@@ -86,6 +86,9 @@ public:
 	bool IsReady() override {
 		return true;
 	}
+	bool IsStarted() override {
+		return true;
+	}
 	void CancelReady() override {}
 	void Reinitialize() override;
 
diff --git a/GPU/GPUInterface.h b/GPU/GPUInterface.h
index 9dc7d82886..2e1af82a66 100644
--- a/GPU/GPUInterface.h
+++ b/GPU/GPUInterface.h
@@ -191,6 +191,7 @@ public:
 	// Initialization
 	virtual bool IsReady() = 0;
 	virtual void CancelReady() = 0;
+	virtual bool IsStarted() = 0;
 	virtual void InitClear() = 0;
 	virtual void Reinitialize() = 0;
 
diff --git a/GPU/Software/SoftGpu.cpp b/GPU/Software/SoftGpu.cpp
index 1b7a4cac82..56e9a3b60a 100644
--- a/GPU/Software/SoftGpu.cpp
+++ b/GPU/Software/SoftGpu.cpp
@@ -433,11 +433,15 @@ SoftGPU::SoftGPU(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	Rasterizer::Init();
 	Sampler::Init();
 	drawEngine_ = new SoftwareDrawEngine();
+	if (!drawEngine_)
+		return;
+
 	drawEngine_->Init();
 	drawEngineCommon_ = drawEngine_;
 
 	// Push the initial CLUT buffer in case it's all zero (we push only on change.)
-	drawEngine_->transformUnit.NotifyClutUpdate(clut);
+	if (drawEngine_->transformUnit.IsStarted())
+		drawEngine_->transformUnit.NotifyClutUpdate(clut);
 
 	// No need to flush for simple parameter changes.
 	flushOnParams_ = false;
@@ -762,6 +766,10 @@ void SoftGPU::FastRunLoop(DisplayList &list) {
 	dirtyFlags_ = dirty;
 }
 
+bool SoftGPU::IsStarted() {
+	return drawEngine_ && drawEngine_->transformUnit.IsStarted();
+}
+
 void SoftGPU::ExecuteOp(u32 op, u32 diff) {
 	const u8 cmd = op >> 24;
 	const auto info = softgpuCmdInfo[cmd];
diff --git a/GPU/Software/SoftGpu.h b/GPU/Software/SoftGpu.h
index 508baa6d0e..2496c7b474 100644
--- a/GPU/Software/SoftGpu.h
+++ b/GPU/Software/SoftGpu.h
@@ -128,6 +128,7 @@ public:
 	~SoftGPU();
 
 	u32 CheckGPUFeatures() const override { return 0; }
+	bool IsStarted() override;
 	void InitClear() override {}
 	void ExecuteOp(u32 op, u32 diff) override;
 	void FinishDeferred() override;
diff --git a/GPU/Software/TransformUnit.cpp b/GPU/Software/TransformUnit.cpp
index e109f6d87a..e4ee125736 100644
--- a/GPU/Software/TransformUnit.cpp
+++ b/GPU/Software/TransformUnit.cpp
@@ -42,6 +42,8 @@
 
 TransformUnit::TransformUnit() {
 	decoded_ = (u8 *)AllocateMemoryPages(TRANSFORM_BUF_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
+	if (!decoded_)
+		return;
 	binner_ = new BinManager();
 }
 
@@ -50,6 +52,10 @@ TransformUnit::~TransformUnit() {
 	delete binner_;
 }
 
+bool TransformUnit::IsStarted() {
+	return binner_ && decoded_;
+}
+
 SoftwareDrawEngine::SoftwareDrawEngine() {
 	// All this is a LOT of memory, need to see if we can cut down somehow.  Used for splines.
 	decoded = (u8 *)AllocateMemoryPages(DECODED_VERTEX_BUFFER_SIZE, MEM_PROT_READ | MEM_PROT_WRITE);
diff --git a/GPU/Software/TransformUnit.h b/GPU/Software/TransformUnit.h
index ed7476a13b..b1dd318540 100644
--- a/GPU/Software/TransformUnit.h
+++ b/GPU/Software/TransformUnit.h
@@ -117,6 +117,8 @@ public:
 	TransformUnit();
 	~TransformUnit();
 
+	bool IsStarted();
+
 	static WorldCoords ModelToWorldNormal(const ModelCoords& coords);
 	static WorldCoords ModelToWorld(const ModelCoords& coords);
 	static ViewCoords WorldToView(const WorldCoords& coords);