Merge pull request #16156 from hrydgard/opengl-thread-refactor

OpenGL threading refactor

Common/GPU/OpenGL/GLQueueRunner.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <vector>
+#include <set>
 #include <unordered_map>
 
 #include "Common/GPU/OpenGL/GLCommon.h"

Common/GPU/OpenGL/GLRenderManager.cpp

@@ -6,6 +6,7 @@
 #include "Common/VR/PPSSPPVR.h"
 
 #include "Common/Log.h"
+#include "Common/TimeUtil.h"
 #include "Common/MemoryUtil.h"
 #include "Common/Math/math_util.h"
 
@@ -112,6 +113,8 @@ void GLDeleter::Perform(GLRenderManager *renderManager, bool skipGLCalls) {
 }
 
 GLRenderManager::~GLRenderManager() {
+	_dbg_assert_(!run_);
+
 	for (int i = 0; i < MAX_INFLIGHT_FRAMES; i++) {
 		_assert_(frameData_[i].deleter.IsEmpty());
 		_assert_(frameData_[i].deleter_prev.IsEmpty());
@@ -123,7 +126,6 @@ GLRenderManager::~GLRenderManager() {
 
 void GLRenderManager::ThreadStart(Draw::DrawContext *draw) {
 	queueRunner_.CreateDeviceObjects();
-	threadFrame_ = threadInitFrame_;
 	renderThreadId = std::this_thread::get_id();
 
 	if (newInflightFrames_ != -1) {
@@ -169,24 +171,16 @@ void GLRenderManager::ThreadStart(Draw::DrawContext *draw) {
 void GLRenderManager::ThreadEnd() {
 	INFO_LOG(G3D, "ThreadEnd");
 
-	// Wait for any shutdown to complete in StopThread().
-	std::unique_lock<std::mutex> lock(mutex_);
 	queueRunner_.DestroyDeviceObjects();
-	VLOG("PULL: Quitting");
+	VLOG("  PULL: Quitting");
 
-	// Good point to run all the deleters to get rid of leftover objects.
+	// Good time to run all the deleters to get rid of leftover objects.
 	for (int i = 0; i < MAX_INFLIGHT_FRAMES; i++) {
 		// Since we're in shutdown, we should skip the GL calls on Android.
 		frameData_[i].deleter.Perform(this, skipGLCalls_);
 		frameData_[i].deleter_prev.Perform(this, skipGLCalls_);
-		for (int j = 0; j < (int)frameData_[i].steps.size(); j++) {
-			delete frameData_[i].steps[j];
-		}
-		frameData_[i].steps.clear();
-		frameData_[i].initSteps.clear();
 	}
 	deleter_.Perform(this, skipGLCalls_);
-
 	for (int i = 0; i < (int)steps_.size(); i++) {
 		delete steps_[i];
 	}
@@ -194,104 +188,68 @@ void GLRenderManager::ThreadEnd() {
 	initSteps_.clear();
 }
 
+// Unlike in Vulkan, this isn't a full independent function, instead it gets called every frame.
+//
+// This means that we have to block and run the render queue until we've presented one frame,
+// at which point we can leave.
+//
+// NOTE: If run_ is true, we WILL run a task!
 bool GLRenderManager::ThreadFrame() {
-	std::unique_lock<std::mutex> lock(mutex_);
-	if (!run_)
+	if (!run_) {
 		return false;
+	}
+
+	GLRRenderThreadTask task;
 
 	// In case of syncs or other partial completion, we keep going until we complete a frame.
-	do {
-		if (nextFrame) {
-			threadFrame_++;
-			if (threadFrame_ >= inflightFrames_)
-				threadFrame_ = 0;
-		}
-		FrameData &frameData = frameData_[threadFrame_];
-		{
-			std::unique_lock<std::mutex> lock(frameData.pull_mutex);
-			while (!frameData.readyForRun && run_) {
-				VLOG("PULL: Waiting for frame[%d].readyForRun", threadFrame_);
-				frameData.pull_condVar.wait(lock);
-			}
-			if (!frameData.readyForRun && !run_) {
-				// This means we're out of frames to render and run_ is false, so bail.
-				return false;
-			}
-			VLOG("PULL: Setting frame[%d].readyForRun = false", threadFrame_);
-			frameData.readyForRun = false;
-			frameData.deleter_prev.Perform(this, skipGLCalls_);
-			frameData.deleter_prev.Take(frameData.deleter);
-			// Previously we had a quick exit here that avoided calling Run() if run_ was suddenly false,
-			// but that created a race condition where frames could end up not finished properly on resize etc.
+	while (true) {
+		// Pop a task of the queue and execute it.
+		// NOTE: We need to actually wait for a task, we can't just bail!
 
-			// Only increment next time if we're done.
-			nextFrame = frameData.type == GLRRunType::END;
-			_assert_(frameData.type == GLRRunType::END || frameData.type == GLRRunType::SYNC);
+		{
+			std::unique_lock<std::mutex> lock(pushMutex_);
+			while (renderThreadQueue_.empty()) {
+				pushCondVar_.wait(lock);
+			}
+			task = renderThreadQueue_.front();
+			renderThreadQueue_.pop();
 		}
-		VLOG("PULL: Running frame %d", threadFrame_);
-		if (firstFrame) {
-			INFO_LOG(G3D, "Running first frame (%d)", threadFrame_);
-			firstFrame = false;
+
+		// We got a task! We can now have pushMutex_ unlocked, allowing the host to
+		// push more work when it feels like it, and just start working.
+		if (task.runType == GLRRunType::EXIT) {
+			// Oh, host wanted out. Let's leave, and also let's notify the host.
+			// This is unlike Vulkan too which can just block on the thread existing.
+			std::unique_lock<std::mutex> lock(syncMutex_);
+			syncCondVar_.notify_one();
+			syncDone_ = true;
+			break;
 		}
 
 		// Render the scene.
-		Run(threadFrame_);
-
-		VLOG("PULL: Finished frame %d", threadFrame_);
-	} while (!nextFrame);
+		VLOG("  PULL: Frame %d RUN (%0.3f)", task.frame, time_now_d());
+		if (Run(task)) {
+			// Swap requested, so we just bail the loop.
+			break;
+		}
+	};
 
 	return true;
 }
 
 void GLRenderManager::StopThread() {
-	// Since we don't control the thread directly, this will only pause the thread.
-
+	// There's not really a lot to do here anymore.
+	INFO_LOG(G3D, "GLRenderManager::StopThread()");
 	if (run_) {
 		run_ = false;
-		for (int i = 0; i < MAX_INFLIGHT_FRAMES; i++) {
-			auto &frameData = frameData_[i];
-			{
-				std::unique_lock<std::mutex> lock(frameData.push_mutex);
-				frameData.push_condVar.notify_all();
-			}
-			{
-				std::unique_lock<std::mutex> lock(frameData.pull_mutex);
-				frameData.pull_condVar.notify_all();
-			}
-		}
 
-		// Wait until we've definitely stopped the threadframe.
-		std::unique_lock<std::mutex> lock(mutex_);
-
-		INFO_LOG(G3D, "GL submission thread paused. Frame=%d", curFrame_);
-
-		// Eat whatever has been queued up for this frame if anything.
-		Wipe();
-
-		// Wait for any fences to finish and be resignaled, so we don't have sync issues.
-		// Also clean out any queued data, which might refer to things that might not be valid
-		// when we restart...
-		for (int i = 0; i < MAX_INFLIGHT_FRAMES; i++) {
-			auto &frameData = frameData_[i];
-			std::unique_lock<std::mutex> lock(frameData.push_mutex);
-			if (frameData.readyForRun || frameData.steps.size() != 0) {
-				Crash();
-			}
-			frameData.readyForRun = false;
-			frameData.readyForSubmit = false;
-			for (size_t i = 0; i < frameData.steps.size(); i++) {
-				delete frameData.steps[i];
-			}
-			frameData.steps.clear();
-			frameData.initSteps.clear();
-
-			while (!frameData.readyForFence) {
-				VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (stop)", i);
-				frameData.push_condVar.wait(lock);
-			}
-		}
+		std::unique_lock<std::mutex> lock(pushMutex_);
+		GLRRenderThreadTask exitTask{};
+		exitTask.runType = GLRRunType::EXIT;
+		renderThreadQueue_.push(exitTask);
+		pushCondVar_.notify_one();
 	} else {
-		INFO_LOG(G3D, "GL submission thread was already paused.");
+		WARN_LOG(G3D, "GL submission thread was already paused.");
 	}
 }
 
@@ -450,60 +408,51 @@ void GLRenderManager::CopyImageToMemorySync(GLRTexture *texture, int mipLevel, i
 }
 
 void GLRenderManager::BeginFrame() {
-	VLOG("BeginFrame");
-
 #ifdef _DEBUG
 	curProgram_ = nullptr;
 #endif
 
 	int curFrame = GetCurFrame();
-	FrameData &frameData = frameData_[curFrame];
 
-	// Make sure the very last command buffer from the frame before the previous has been fully executed.
+	FrameData &frameData = frameData_[curFrame];
 	{
-		std::unique_lock<std::mutex> lock(frameData.push_mutex);
+		VLOG("PUSH: BeginFrame (curFrame = %d, readyForFence = %d, time=%0.3f)", curFrame, (int)frameData.readyForFence, time_now_d());
+		std::unique_lock<std::mutex> lock(frameData.fenceMutex);
 		while (!frameData.readyForFence) {
-			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1", curFrame);
-			frameData.push_condVar.wait(lock);
+			frameData.fenceCondVar.wait(lock);
 		}
 		frameData.readyForFence = false;
-		frameData.readyForSubmit = true;
 	}
 
-	VLOG("PUSH: Fencing %d", curFrame);
-
-	// glFenceSync(&frameData.fence...)
-
-	// Must be after the fence - this performs deletes.
-	VLOG("PUSH: BeginFrame %d", curFrame);
 	if (!run_) {
 		WARN_LOG(G3D, "BeginFrame while !run_!");
 	}
 
-	// vulkan_->BeginFrame();
-	// In GL, we have to do deletes on the submission thread.
-
 	insideFrame_ = true;
 }
 
 void GLRenderManager::Finish() {
-	curRenderStep_ = nullptr;
+	curRenderStep_ = nullptr;  // EndCurRenderStep is this simple here.
+
 	int curFrame = GetCurFrame();
 	FrameData &frameData = frameData_[curFrame];
-	{
-		std::unique_lock<std::mutex> lock(frameData.pull_mutex);
-		VLOG("PUSH: Frame[%d].readyForRun = true, notifying pull", curFrame);
-		frameData.steps = std::move(steps_);
-		steps_.clear();
-		frameData.initSteps = std::move(initSteps_);
-		initSteps_.clear();
-		frameData.readyForRun = true;
-		frameData.type = GLRRunType::END;
-		frameData_[curFrame_].deleter.Take(deleter_);
-	}
 
-	// Notify calls do not in fact need to be done with the mutex locked.
-	frameData.pull_condVar.notify_all();
+	frameData_[curFrame].deleter.Take(deleter_);
+
+	VLOG("PUSH: Finish, pushing task. curFrame = %d", curFrame);
+	GLRRenderThreadTask task;
+	task.frame = curFrame;
+	task.runType = GLRRunType::PRESENT;
+
+	{
+		std::unique_lock<std::mutex> lock(pushMutex_);
+		renderThreadQueue_.push(task);
+		renderThreadQueue_.back().initSteps = std::move(initSteps_);
+		renderThreadQueue_.back().steps = std::move(steps_);
+		initSteps_.clear();
+		steps_.clear();
+		pushCondVar_.notify_one();
+	}
 
 	curFrame_++;
 	if (curFrame_ >= inflightFrames_)
@@ -512,18 +461,19 @@ void GLRenderManager::Finish() {
 	insideFrame_ = false;
 }
 
-// Render thread
-void GLRenderManager::Run(int frame) {
-	FrameData &frameData = frameData_[frame];
+// Render thread. Returns true if the caller should handle a swap.
+bool GLRenderManager::Run(GLRRenderThreadTask &task) {
+	FrameData &frameData = frameData_[task.frame];
+
 	if (!frameData.hasBegun) {
 		frameData.hasBegun = true;
+
+		frameData.deleter_prev.Perform(this, skipGLCalls_);
+		frameData.deleter_prev.Take(frameData.deleter);
 	}
 
-	auto &stepsOnThread = frameData_[frame].steps;
-	auto &initStepsOnThread = frameData_[frame].initSteps;
 	// queueRunner_.LogSteps(stepsOnThread);
-	queueRunner_.RunInitSteps(initStepsOnThread, skipGLCalls_);
-	initStepsOnThread.clear();
+	queueRunner_.RunInitSteps(task.initSteps, skipGLCalls_);
 
 	// Run this after RunInitSteps so any fresh GLRBuffers for the pushbuffers can get created.
 	if (!skipGLCalls_) {
@@ -537,13 +487,12 @@ void GLRenderManager::Run(int frame) {
 		int passes = GetVRPassesCount();
 		for (int i = 0; i < passes; i++) {
 			PreVRFrameRender(i);
-			queueRunner_.RunSteps(stepsOnThread, skipGLCalls_, i < passes - 1, true);
+			queueRunner_.RunSteps(task.steps, skipGLCalls_, i < passes - 1, true);
 			PostVRFrameRender();
 		}
 	} else {
-		queueRunner_.RunSteps(stepsOnThread, skipGLCalls_, false, false);
+		queueRunner_.RunSteps(task.steps, skipGLCalls_, false, false);
 	}
-	stepsOnThread.clear();
 
 	if (!skipGLCalls_) {
 		for (auto iter : frameData.activePushBuffers) {
@@ -551,20 +500,10 @@ void GLRenderManager::Run(int frame) {
 		}
 	}
 
-	switch (frameData.type) {
-	case GLRRunType::END:
-		frameData.hasBegun = false;
-
-		VLOG("PULL: Frame %d.readyForFence = true", frame);
-
-		{
-			std::unique_lock<std::mutex> lock(frameData.push_mutex);
-			_assert_(frameData.readyForSubmit);
-			frameData.readyForFence = true;
-			frameData.readyForSubmit = false;
-			frameData.push_condVar.notify_all();
-		}
+	bool swapRequest = false;
 
+	switch (task.runType) {
+	case GLRRunType::PRESENT:
 		if (!frameData.skipSwap) {
 			if (swapIntervalChanged_) {
 				swapIntervalChanged_ = false;
@@ -572,78 +511,77 @@ void GLRenderManager::Run(int frame) {
 					swapIntervalFunction_(swapInterval_);
 				}
 			}
+			// This is the swapchain framebuffer flip.
 			if (swapFunction_) {
+				VLOG("  PULL: SwapFunction()");
 				swapFunction_();
+				if (!retainControl_) {
+					// get out of here.
+					swapRequest = true;
+				}
+			} else {
+				VLOG("  PULL: SwapRequested");
+				swapRequest = true;
 			}
 		} else {
 			frameData.skipSwap = false;
 		}
+		frameData.hasBegun = false;
+
+		VLOG("  PULL: Frame %d.readyForFence = true", task.frame);
+
+		{
+			std::lock_guard<std::mutex> lock(frameData.fenceMutex);
+			frameData.readyForFence = true;
+			frameData.fenceCondVar.notify_one();
+			// At this point, we're done with this framedata (for now).
+		}
+
 		break;
 
 	case GLRRunType::SYNC:
+		frameData.hasBegun = false;
+
 		// glFinish is not actually necessary here, and won't be unless we start using
 		// glBufferStorage. Then we need to use fences.
 		{
-			std::unique_lock<std::mutex> lock(frameData.push_mutex);
-			frameData.readyForFence = true;
-			frameData.readyForSubmit = true;
-			frameData.push_condVar.notify_all();
+			std::unique_lock<std::mutex> lock(syncMutex_);
+			syncDone_ = true;
+			syncCondVar_.notify_one();
 		}
 		break;
 
 	default:
 		_assert_(false);
 	}
-
-	VLOG("PULL: Finished running frame %d", frame);
+	VLOG("  PULL: ::Run(): Done running tasks");
+	return swapRequest;
 }
 
 void GLRenderManager::FlushSync() {
-	int curFrame = curFrame_;
-	FrameData &frameData = frameData_[curFrame];
 	{
-		std::unique_lock<std::mutex> lock(frameData.pull_mutex);
-		VLOG("PUSH: Frame[%d].readyForRun = true (sync)", curFrame);
-		frameData.initSteps = std::move(initSteps_);
-		initSteps_.clear();
-		frameData.steps = std::move(steps_);
+		VLOG("PUSH: Frame[%d].readyForRun = true (sync)", curFrame_);
+
+		GLRRenderThreadTask task;
+		task.frame = curFrame_;
+		task.runType = GLRRunType::SYNC;
+
+		std::unique_lock<std::mutex> lock(pushMutex_);
+		renderThreadQueue_.push(task);
+		renderThreadQueue_.back().initSteps = std::move(initSteps_);
+		renderThreadQueue_.back().steps = std::move(steps_);
+		pushCondVar_.notify_one();
 		steps_.clear();
-		frameData.readyForRun = true;
-		_assert_(frameData.readyForFence == false);
-		frameData.type = GLRRunType::SYNC;
-		frameData.pull_condVar.notify_all();
 	}
+
 	{
-		std::unique_lock<std::mutex> lock(frameData.push_mutex);
+		std::unique_lock<std::mutex> lock(syncMutex_);
 		// Wait for the flush to be hit, since we're syncing.
-		while (!frameData.readyForFence) {
-			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (sync)", curFrame);
-			frameData.push_condVar.wait(lock);
-		}
-		frameData.readyForFence = false;
-		frameData.readyForSubmit = true;
-	}
-}
-
-void GLRenderManager::Wipe() {
-	initSteps_.clear();
-	for (auto step : steps_) {
-		delete step;
-	}
-	steps_.clear();
-}
-
-void GLRenderManager::WaitUntilQueueIdle() {
-	// Just wait for all frames to be ready.
-	for (int i = 0; i < MAX_INFLIGHT_FRAMES; i++) {
-		FrameData &frameData = frameData_[i];
-
-		std::unique_lock<std::mutex> lock(frameData.push_mutex);
-		// Ignore unsubmitted frames.
-		while (!frameData.readyForFence && frameData.readyForRun) {
-			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (wait idle)", i);
-			frameData.push_condVar.wait(lock);
+		while (!syncDone_) {
+			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (sync)", curFrame_);
+			syncCondVar_.wait(lock);
 		}
+		syncDone_ = false;
 	}
 }
 

Common/GPU/OpenGL/GLRenderManager.h

@@ -7,6 +7,7 @@
 #include <set>
 #include <string>
 #include <mutex>
+#include <queue>
 #include <condition_variable>
 
 #include "Common/GPU/OpenGL/GLCommon.h"
@@ -349,11 +350,29 @@ private:
 	GLBufferStrategy strategy_ = GLBufferStrategy::SUBDATA;
 };
 
-enum class GLRRunType {
-	END,
-	SYNC,
+class GLRInputLayout {
+public:
+	struct Entry {
+		int location;
+		int count;
+		GLenum type;
+		GLboolean normalized;
+		int stride;
+		intptr_t offset;
+	};
+	std::vector<Entry> entries;
+	int semanticsMask_ = 0;
 };
 
+enum class GLRRunType {
+	PRESENT,
+	SYNC,
+	EXIT,
+};
+
+class GLRenderManager;
+class GLPushBuffer;
+
 class GLDeleter {
 public:
 	void Perform(GLRenderManager *renderManager, bool skipGLCalls);
@@ -373,18 +392,14 @@ public:
 	std::vector<GLPushBuffer *> pushBuffers;
 };
 
-class GLRInputLayout {
-public:
-	struct Entry {
-		int location;
-		int count;
-		GLenum type;
-		GLboolean normalized;
-		int stride;
-		intptr_t offset;
-	};
-	std::vector<Entry> entries;
-	int semanticsMask_ = 0;
+// These are enqueued from the main thread,
+// and the render thread pops them off
+struct GLRRenderThreadTask {
+	std::vector<GLRStep *> steps;
+	std::vector<GLRInitStep> initSteps;
+
+	int frame;
+	GLRRunType runType;
 };
 
 // Note: The GLRenderManager is created and destroyed on the render thread, and the latter
@@ -402,7 +417,7 @@ public:
 
 	void ThreadStart(Draw::DrawContext *draw);
 	void ThreadEnd();
-	bool ThreadFrame();  // Returns false to request exiting the loop.
+	bool ThreadFrame();  // Returns true if it did anything. False means the queue was empty.
 
 	void SetErrorCallback(ErrorCallbackFn callback, void *userdata) {
 		queueRunner_.SetErrorCallback(callback, userdata);
@@ -415,20 +430,14 @@ public:
 	// Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again.
 	void BeginFrame();
 	// Can run on a different thread!
-	void Finish();
-	void Run(int frame);
-
-	// Zaps queued up commands. Use if you know there's a risk you've queued up stuff that has already been deleted. Can happen during in-game shutdown.
-	void Wipe();
-
-	// Wait until no frames are pending.  Use during shutdown before freeing pointers.
-	void WaitUntilQueueIdle();
+	void Finish(); 
+	bool Run(GLRRenderThreadTask &task);
 
 	// Creation commands. These were not needed in Vulkan since there we can do that on the main thread.
 	// We pass in width/height here even though it's not strictly needed until we support glTextureStorage
 	// and then we'll also need formats and stuff.
 	GLRTexture *CreateTexture(GLenum target, int width, int height, int depth, int numMips) {
-		GLRInitStep step{ GLRInitStepType::CREATE_TEXTURE };
+		GLRInitStep step { GLRInitStepType::CREATE_TEXTURE };
 		step.create_texture.texture = new GLRTexture(caps_, width, height, depth, numMips);
 		step.create_texture.texture->target = target;
 		initSteps_.push_back(step);
@@ -983,8 +992,9 @@ public:
 		_dbg_assert_(foundCount == 1);
 	}
 
-	void SetSwapFunction(std::function<void()> swapFunction) {
+	void SetSwapFunction(std::function<void()> swapFunction, bool retainControl) {
 		swapFunction_ = swapFunction;
+		retainControl_ = retainControl;
 	}
 
 	void SetSwapIntervalFunction(std::function<void(int)> swapIntervalFunction) {
@@ -1027,26 +1037,14 @@ private:
 
 	// Per-frame data, round-robin so we can overlap submission with execution of the previous frame.
 	struct FrameData {
-		std::mutex push_mutex;
-		std::condition_variable push_condVar;
-
-		std::mutex pull_mutex;
-		std::condition_variable pull_condVar;
-
-		bool readyForFence = true;
-		bool readyForRun = false;
-		bool readyForSubmit = false;
-
 		bool skipSwap = false;
-		GLRRunType type = GLRRunType::END;
 
-		// GLuint fence; For future AZDO stuff?
-		std::vector<GLRStep *> steps;
-		std::vector<GLRInitStep> initSteps;
+		std::mutex fenceMutex;
+		std::condition_variable fenceCondVar;
+		bool readyForFence = true;
 
 		// Swapchain.
 		bool hasBegun = false;
-		uint32_t curSwapchainImage = -1;
 
 		GLDeleter deleter;
 		GLDeleter deleter_prev;
@@ -1064,16 +1062,23 @@ private:
 
 	// Execution time state
 	bool run_ = true;
+
 	// Thread is managed elsewhere, and should call ThreadFrame.
-	std::mutex mutex_;
-	int threadInitFrame_ = 0;
 	GLQueueRunner queueRunner_;
 
-	// Thread state
-	int threadFrame_ = -1;
+	// For pushing data on the queue.
+	std::mutex pushMutex_;
+	std::condition_variable pushCondVar_;
 
-	bool nextFrame = false;
-	bool firstFrame = true;
+	std::queue<GLRRenderThreadTask> renderThreadQueue_;
+
+	// For readbacks and other reasons we need to sync with the render thread.
+	std::mutex syncMutex_;
+	std::condition_variable syncCondVar_;
+
+	bool firstFrame_ = true;
+	bool vrRenderStarted_ = false;
+	bool syncDone_ = false;
 
 	GLDeleter deleter_;
 	bool skipGLCalls_ = false;
@@ -1082,6 +1087,7 @@ private:
 
 	std::function<void()> swapFunction_;
 	std::function<void(int)> swapIntervalFunction_;
+	bool retainControl_ = false;
 	GLBufferStrategy bufferStrategy_ = GLBufferStrategy::SUBDATA;
 
 	int inflightFrames_ = MAX_INFLIGHT_FRAMES;

Common/GPU/OpenGL/thin3d_gl.cpp

@@ -212,7 +212,6 @@ GLuint ShaderStageToOpenGL(ShaderStage stage) {
 class OpenGLShaderModule : public ShaderModule {
 public:
 	OpenGLShaderModule(GLRenderManager *render, ShaderStage stage, const std::string &tag) : render_(render), stage_(stage), tag_(tag) {
-		DEBUG_LOG(G3D, "Shader module created (%p)", this);
 		glstage_ = ShaderStageToOpenGL(stage);
 	}
 

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -1197,11 +1197,11 @@ void VulkanRenderManager::Finish() {
 	int curFrame = vulkan_->GetCurFrame();
 	FrameData &frameData = frameData_[curFrame];
 
+	VLOG("PUSH: Frame[%d]", curFrame);
+	VKRRenderThreadTask task;
+	task.frame = curFrame;
+	task.runType = VKRRunType::PRESENT;
 	{
-		VLOG("PUSH: Frame[%d]", curFrame);
-		VKRRenderThreadTask task;
-		task.frame = curFrame;
-		task.runType = VKRRunType::PRESENT;
 		std::unique_lock<std::mutex> lock(pushMutex_);
 		renderThreadQueue_.push(task);
 		renderThreadQueue_.back().steps = std::move(steps_);
@@ -1327,7 +1327,7 @@ void VulkanRenderManager::FlushSync() {
 		std::unique_lock<std::mutex> lock(syncMutex_);
 		// Wait for the flush to be hit, since we're syncing.
 		while (!frameData.syncDone) {
-			VLOG("PUSH: Waiting for frame[%d].readyForFence = 1 (sync)", curFrame);
+			VLOG("PUSH: Waiting for frame[%d].syncDone = 1 (sync)", curFrame);
 			syncCondVar_.wait(lock);
 		}
 		frameData.syncDone = false;

Common/GPU/Vulkan/thin3d_vulkan.cpp

@@ -198,7 +198,6 @@ public:
 	const std::string &GetSource() const { return source_; }
 	~VKShaderModule() {
 		if (module_) {
-			DEBUG_LOG(G3D, "Queueing %s (shmodule %p) for release", tag_.c_str(), module_);
 			VkShaderModule shaderModule = module_->BlockUntilReady();
 			vulkan_->Delete().QueueDeleteShaderModule(shaderModule);
 			vulkan_->Delete().QueueCallback([](VulkanContext *context, void *m) {
@@ -268,7 +267,6 @@ public:
 		vkrDesc = new VKRGraphicsPipelineDesc();
 	}
 	~VKPipeline() {
-		DEBUG_LOG(G3D, "Queueing %s (pipeline) for release", tag_.c_str());
 		if (pipeline) {
 			pipeline->QueueForDeletion(vulkan_);
 		}

Core/Core.cpp

@@ -327,14 +327,14 @@ void Core_ProcessStepping() {
 
 // Many platforms, like Android, do not call this function but handle things on their own.
 // Instead they simply call NativeRender and NativeUpdate directly.
-void Core_Run(GraphicsContext *ctx) {
+bool Core_Run(GraphicsContext *ctx) {
 	host->UpdateDisassembly();
 	while (true) {
 		if (GetUIState() != UISTATE_INGAME) {
 			Core_StateProcessed();
 			if (GetUIState() == UISTATE_EXIT) {
 				UpdateRunLoop();
-				return;
+				return false;
 			}
 			Core_RunLoop(ctx);
 			continue;
@@ -347,7 +347,7 @@ void Core_Run(GraphicsContext *ctx) {
 			Core_RunLoop(ctx);
 			if (coreState == CORE_POWERDOWN) {
 				Core_StateProcessed();
-				return;
+				return true;
 			}
 			break;
 
@@ -358,10 +358,10 @@ void Core_Run(GraphicsContext *ctx) {
 			// Exit loop!!
 			Core_StateProcessed();
 
-			return;
+			return true;
 
 		case CORE_NEXTFRAME:
-			return;
+			return true;
 		}
 	}
 }

Core/Core.h

@@ -27,8 +27,10 @@ class GraphicsContext;
 // called from emu thread
 void UpdateRunLoop();
 
-void Core_Run(GraphicsContext *ctx);
+// Returns false when an UI exit state is detected.
+bool Core_Run(GraphicsContext *ctx);
 void Core_Stop();
+
 // For platforms that don't call Core_Run
 void Core_SetGraphicsContext(GraphicsContext *ctx);
 

Core/CoreTiming.cpp

@@ -233,16 +233,14 @@ void Shutdown()
 	ClearPendingEvents();
 	UnregisterAllEvents();
 
-	while(eventPool)
-	{
+	while (eventPool) {
 		Event *ev = eventPool;
 		eventPool = ev->next;
 		delete ev;
 	}
 
 	std::lock_guard<std::mutex> lk(externalEventLock);
-	while(eventTsPool)
-	{
+	while (eventTsPool) {
 		Event *ev = eventTsPool;
 		eventTsPool = ev->next;
 		delete ev;
@@ -251,7 +249,12 @@ void Shutdown()
 
 u64 GetTicks()
 {
-	return (u64)globalTimer + slicelength - currentMIPS->downcount;
+	if (currentMIPS) {
+		return (u64)globalTimer + slicelength - currentMIPS->downcount;
+	} else {
+		// Reporting can actually end up here during weird task switching sequences on Android
+		return false;
+	}
 }
 
 u64 GetIdleTicks()

GPU/GLES/GPU_GLES.cpp

@@ -128,11 +128,6 @@ GPU_GLES::GPU_GLES(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 }
 
 GPU_GLES::~GPU_GLES() {
-	if (draw_) {
-		GLRenderManager *render = (GLRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
-		render->Wipe();
-	}
-
 	// If we're here during app shutdown (exiting the Windows app in-game, for example)
 	// everything should already be cleared since DeviceLost has been run.
 

Qt/QtMain.h

@@ -90,7 +90,6 @@ public:
 	}
 
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}
 

SDL/SDLGLGraphicsContext.cpp

@@ -429,7 +429,7 @@ int SDLGLGraphicsContext::Init(SDL_Window *&window, int x, int y, int mode, std:
 #else
 		SDL_GL_SwapWindow(window_);
 #endif
-	});
+	}, false);
 
 	renderManager_->SetSwapIntervalFunction([&](int interval) {
 		INFO_LOG(G3D, "SDL SwapInterval: %d", interval);

SDL/SDLGLGraphicsContext.h

@@ -42,7 +42,6 @@ public:
 	}
 
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}
 

Windows/EmuThread.cpp

@@ -82,7 +82,9 @@ static void EmuThreadFunc(GraphicsContext *graphicsContext) {
 		// This way they can load a new game.
 		if (!Core_IsActive())
 			UpdateUIState(UISTATE_MENU);
-		Core_Run(g_graphicsContext);
+		if (!Core_Run(g_graphicsContext)) {
+			emuThreadState = (int)EmuThreadState::QUIT_REQUESTED;
+		}
 	}
 
 	emuThreadState = (int)EmuThreadState::STOPPED;
@@ -99,12 +101,14 @@ static void EmuThreadStart(GraphicsContext *graphicsContext) {
 }
 
 static void EmuThreadStop() {
-	emuThreadState = (int)EmuThreadState::QUIT_REQUESTED;
+	if (emuThreadState != (int)EmuThreadState::QUIT_REQUESTED &&
+		emuThreadState != (int)EmuThreadState::STOPPED) {
+		emuThreadState = (int)EmuThreadState::QUIT_REQUESTED;
+	}
 }
 
 static void EmuThreadJoin() {
 	emuThread.join();
-	emuThread = std::thread();
 	INFO_LOG(SYSTEM, "EmuThreadJoin - joined");
 }
 

Windows/GPU/WindowsGLContext.cpp

@@ -432,7 +432,7 @@ bool WindowsGLContext::InitFromRenderThread(std::string *error_message) {
 	renderManager_ = (GLRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
 	renderManager_->SetInflightFrames(g_Config.iInflightFrames);
 	SetGPUBackend(GPUBackend::OPENGL);
-	renderManager_->SetSwapFunction([&]() {::SwapBuffers(hDC); });
+	renderManager_->SetSwapFunction([&]() {::SwapBuffers(hDC); }, true);
 	if (wglSwapIntervalEXT) {
 		// glew loads wglSwapIntervalEXT if available
 		renderManager_->SetSwapIntervalFunction([&](int interval) {
@@ -500,6 +500,5 @@ void WindowsGLContext::ThreadEnd() {
 }
 
 void WindowsGLContext::StopThread() {
-	renderManager_->WaitUntilQueueIdle();
 	renderManager_->StopThread();
 }

Windows/PPSSPP.vcxproj.filters

@@ -838,4 +838,4 @@
       <Filter>Other Platforms\SDL</Filter>
     </Text>
   </ItemGroup>
-</Project>
+</Project>

android/jni/AndroidJavaGLContext.cpp

@@ -38,7 +38,6 @@ bool AndroidJavaEGLGraphicsContext::InitFromRenderThread(ANativeWindow *wnd, int
 
 void AndroidJavaEGLGraphicsContext::ShutdownFromRenderThread() {
 	INFO_LOG(G3D, "AndroidJavaEGLGraphicsContext::Shutdown");
-	renderManager_->WaitUntilQueueIdle();
 	renderManager_ = nullptr;  // owned by draw_.
 	delete draw_;
 	draw_ = nullptr;

android/jni/AndroidJavaGLContext.h

@@ -40,7 +40,6 @@ public:
 	}
 
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}
 

android/jni/app-android.cpp

@@ -335,7 +335,7 @@ static void EmuThreadFunc() {
 	while (emuThreadState != (int)EmuThreadState::QUIT_REQUESTED) {
 		UpdateRunLoopAndroid(env);
 	}
-	INFO_LOG(SYSTEM, "QUIT_REQUESTED found, left loop. Setting state to STOPPED.");
+	INFO_LOG(SYSTEM, "QUIT_REQUESTED found, left EmuThreadFunc loop. Setting state to STOPPED.");
 	emuThreadState = (int)EmuThreadState::STOPPED;
 
 	NativeShutdownGraphics();
@@ -868,17 +868,18 @@ extern "C" void Java_org_ppsspp_ppsspp_NativeApp_shutdown(JNIEnv *, jclass) {
 		EmuThreadStop("shutdown");
 		INFO_LOG(SYSTEM, "BeginAndroidShutdown");
 		graphicsContext->BeginAndroidShutdown();
-		// Skipping GL calls, the old context is gone.
-		while (graphicsContext->ThreadFrame()) {
-			INFO_LOG(SYSTEM, "graphicsContext->ThreadFrame executed to clear buffers");
-		}
-		INFO_LOG(SYSTEM, "Joining emuthread");
-		EmuThreadJoin();
-		INFO_LOG(SYSTEM, "Joined emuthread");
-
+		// Now, it could be that we had some frames queued up. Get through them.
+		// We're on the render thread, so this is synchronous.
+		do {
+			INFO_LOG(SYSTEM, "Executing graphicsContext->ThreadFrame to clear buffers");
+		} while (graphicsContext->ThreadFrame());
 		graphicsContext->ThreadEnd();
+		INFO_LOG(SYSTEM, "ThreadEnd called.");
 		graphicsContext->ShutdownFromRenderThread();
 		INFO_LOG(SYSTEM, "Graphics context now shut down from NativeApp_shutdown");
+
+		INFO_LOG(SYSTEM, "Joining emuthread");
+		EmuThreadJoin();
 	}
 
 	INFO_LOG(SYSTEM, "NativeApp.shutdown() -- begin");

headless/SDLHeadlessHost.cpp

@@ -83,7 +83,6 @@ public:
 
 	void StopThread() override {
 		if (renderManager_) {
-			renderManager_->WaitUntilQueueIdle();
 			renderManager_->StopThread();
 		}
 	}
@@ -167,7 +166,7 @@ bool GLDummyGraphicsContext::InitFromRenderThread(std::string *errorMessage) {
 	_assert_(success);
 	renderManager_->SetSwapFunction([&]() {
 		SDL_GL_SwapWindow(screen_);
-	});
+	}, false);
 
 	return success;
 }

ios/ViewController.mm

@@ -80,7 +80,6 @@ public:
 	}
 
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}
 

libretro/LibretroGLContext.h

@@ -28,7 +28,6 @@ public:
 	bool ThreadFrame() override { return renderManager_->ThreadFrame(); }
 	void ThreadEnd() override { renderManager_->ThreadEnd(); }
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}
 

libretro/LibretroGLCoreContext.h

@@ -24,7 +24,6 @@ public:
 	bool ThreadFrame() override { return renderManager_->ThreadFrame(); }
 	void ThreadEnd() override { renderManager_->ThreadEnd(); }
 	void StopThread() override {
-		renderManager_->WaitUntilQueueIdle();
 		renderManager_->StopThread();
 	}