diff --git a/GPU/Common/FramebufferCommon.cpp b/GPU/Common/FramebufferCommon.cpp
index 4b902a26da..484c9d0de4 100644
--- a/GPU/Common/FramebufferCommon.cpp
+++ b/GPU/Common/FramebufferCommon.cpp
@@ -140,11 +140,18 @@ void FramebufferManagerCommon::Init() {
 	BeginFrame();
 }
 
-void FramebufferManagerCommon::UpdateSize() {
+bool FramebufferManagerCommon::UpdateSize() {
+	const bool newRender = renderWidth_ != (float)PSP_CoreParameter().renderWidth || renderHeight_ != (float)PSP_CoreParameter().renderHeight;
+	const bool newSettings = bloomHack_ != g_Config.iBloomHack || trueColor_ != g_Config.bTrueColor;
+
 	renderWidth_ = (float)PSP_CoreParameter().renderWidth;
 	renderHeight_ = (float)PSP_CoreParameter().renderHeight;
 	pixelWidth_ = PSP_CoreParameter().pixelWidth;
 	pixelHeight_ = PSP_CoreParameter().pixelHeight;
+	bloomHack_ = g_Config.iBloomHack;
+	trueColor_ = g_Config.bTrueColor;
+
+	return newRender || newSettings;
 }
 
 void FramebufferManagerCommon::BeginFrame() {
@@ -1105,7 +1112,7 @@ void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, u16 w,
 
 	SetRenderSize(vfb);
 
-	bool trueColor = g_Config.bTrueColor;
+	bool trueColor = trueColor_;
 	if (PSP_CoreParameter().compat.flags().Force04154000Download && vfb->fb_address == 0x00154000) {
 		trueColor = true;
 	}
@@ -1660,7 +1667,7 @@ void FramebufferManagerCommon::SetRenderSize(VirtualFramebuffer *vfb) {
 	float renderWidthFactor = renderWidth_ / 480.0f;
 	float renderHeightFactor = renderHeight_ / 272.0f;
 	bool force1x = false;
-	switch (g_Config.iBloomHack) {
+	switch (bloomHack_) {
 	case 1:
 		force1x = vfb->bufferWidth <= 128 || vfb->bufferHeight <= 64;
 		break;
diff --git a/GPU/Common/FramebufferCommon.h b/GPU/Common/FramebufferCommon.h
index 9336cdd2d1..20ee9f9405 100644
--- a/GPU/Common/FramebufferCommon.h
+++ b/GPU/Common/FramebufferCommon.h
@@ -285,7 +285,7 @@ protected:
 	// Cardboard Settings Calculator
 	void GetCardboardSettings(CardboardSettings *cardboardSettings);
 
-	void UpdateSize();
+	bool UpdateSize();
 	void SetNumExtraFBOs(int num);
 
 	virtual void DisableState() = 0;
@@ -368,6 +368,8 @@ protected:
 	float renderHeight_;
 	int pixelWidth_;
 	int pixelHeight_;
+	int bloomHack_ = 0;
+	bool trueColor_ = false;
 
 	// Used by post-processing shaders
 	std::vector<Draw::Framebuffer *> extraFBOs_;
diff --git a/GPU/D3D11/FramebufferManagerD3D11.cpp b/GPU/D3D11/FramebufferManagerD3D11.cpp
index b8b317bfbe..ce0cf12796 100644
--- a/GPU/D3D11/FramebufferManagerD3D11.cpp
+++ b/GPU/D3D11/FramebufferManagerD3D11.cpp
@@ -859,8 +859,6 @@ void FramebufferManagerD3D11::PackDepthbuffer(VirtualFramebuffer *vfb, int x, in
 
 void FramebufferManagerD3D11::EndFrame() {
 	if (resized_) {
-		DestroyAllFBOs();
-
 		// Check if postprocessing shader is doing upscaling as it requires native resolution
 		const ShaderInfo *shaderInfo = 0;
 		if (g_Config.sPostShaderName != "Off") {
@@ -891,7 +889,10 @@ void FramebufferManagerD3D11::EndFrame() {
 			PSP_CoreParameter().renderHeight = 272 * zoom;
 		}
 
-		UpdateSize();
+		if (UpdateSize() || g_Config.iRenderingMode == FB_NON_BUFFERED_MODE) {
+			DestroyAllFBOs();
+		}
+
 		// Seems related - if you're ok with numbers all the time, show some more :)
 		if (g_Config.iShowFPSCounter != 0) {
 			ShowScreenResolution();
diff --git a/GPU/Directx9/FramebufferDX9.cpp b/GPU/Directx9/FramebufferDX9.cpp
index e841242155..46c719555b 100644
--- a/GPU/Directx9/FramebufferDX9.cpp
+++ b/GPU/Directx9/FramebufferDX9.cpp
@@ -752,7 +752,6 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 
 	void FramebufferManagerDX9::EndFrame() {
 		if (resized_) {
-			DestroyAllFBOs();
 			// Actually, auto mode should be more granular...
 			// Round up to a zoom factor for the render size.
 			int zoom = g_Config.iInternalResolution;
@@ -775,7 +774,9 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 				PSP_CoreParameter().renderHeight = 272 * zoom;
 			}
 
-			UpdateSize();
+			if (UpdateSize() || g_Config.iRenderingMode == FB_NON_BUFFERED_MODE) {
+				DestroyAllFBOs();
+			}
 			// Seems related - if you're ok with numbers all the time, show some more :)
 			if (g_Config.iShowFPSCounter != 0) {
 				ShowScreenResolution();
diff --git a/GPU/GLES/FramebufferManagerGLES.cpp b/GPU/GLES/FramebufferManagerGLES.cpp
index d7db65d4da..faff214096 100644
--- a/GPU/GLES/FramebufferManagerGLES.cpp
+++ b/GPU/GLES/FramebufferManagerGLES.cpp
@@ -1150,11 +1150,8 @@ void FramebufferManagerGLES::PackDepthbuffer(VirtualFramebuffer *vfb, int x, int
 void FramebufferManagerGLES::EndFrame() {
 	CHECK_GL_ERROR_IF_DEBUG();
 	if (resized_) {
-		// TODO: Only do this if the new size actually changed the renderwidth/height.
-		DestroyAllFBOs();
-
 		// Check if postprocessing shader is doing upscaling as it requires native resolution
-		const ShaderInfo *shaderInfo = 0;
+		const ShaderInfo *shaderInfo = nullptr;
 		if (g_Config.sPostShaderName != "Off") {
 			shaderInfo = GetPostShaderInfo(g_Config.sPostShaderName);
 		}
@@ -1183,7 +1180,9 @@ void FramebufferManagerGLES::EndFrame() {
 			PSP_CoreParameter().renderHeight = 272 * zoom;
 		}
 
-		UpdateSize();
+		if (UpdateSize() || g_Config.iRenderingMode == FB_NON_BUFFERED_MODE) {
+			DestroyAllFBOs();
+		}
 
 		resized_ = false;
 #ifdef _WIN32
@@ -1192,7 +1191,6 @@ void FramebufferManagerGLES::EndFrame() {
 			ShowScreenResolution();
 		}
 #endif
-		ClearBuffer();
 		DestroyDraw2DProgram();
 		SetLineWidth();
 	}
diff --git a/GPU/Vulkan/FramebufferVulkan.cpp b/GPU/Vulkan/FramebufferVulkan.cpp
index e3be6cec11..d7be33ab65 100644
--- a/GPU/Vulkan/FramebufferVulkan.cpp
+++ b/GPU/Vulkan/FramebufferVulkan.cpp
@@ -1039,9 +1039,6 @@ void FramebufferManagerVulkan::BeginFrameVulkan() {
 
 void FramebufferManagerVulkan::EndFrame() {
 	if (resized_) {
-		// TODO: Only do this if the new size actually changed the renderwidth/height.
-		DestroyAllFBOs();
-
 		// Check if postprocessing shader is doing upscaling as it requires native resolution
 		const ShaderInfo *shaderInfo = 0;
 		if (g_Config.sPostShaderName != "Off") {
@@ -1073,7 +1070,9 @@ void FramebufferManagerVulkan::EndFrame() {
 			PSP_CoreParameter().renderHeight = 272 * zoom;
 		}
 
-		UpdateSize();
+		if (UpdateSize() || g_Config.iRenderingMode == FB_NON_BUFFERED_MODE) {
+			DestroyAllFBOs();
+		}
 
 		resized_ = false;
 #ifdef _WIN32
@@ -1082,7 +1081,6 @@ void FramebufferManagerVulkan::EndFrame() {
 			ShowScreenResolution();
 		}
 #endif
-		ClearBuffer();
 	}
 
 	// We flush to memory last requested framebuffer, if any.