diff --git a/GPU/Common/FramebufferCommon.cpp b/GPU/Common/FramebufferCommon.cpp
index 3a997f59ae..9478826b28 100644
--- a/GPU/Common/FramebufferCommon.cpp
+++ b/GPU/Common/FramebufferCommon.cpp
@@ -440,7 +440,6 @@ VirtualFramebuffer *FramebufferManagerCommon::DoSetRenderFrameBuffer(const Frame
 		vfb->drawnFormat = params.fmt;
 		vfb->usageFlags = FB_USAGE_RENDERTARGET;
 		SetColorUpdated(vfb, skipDrawReason);
-		vfb->depthUpdated = false;
 
 		u32 byteSize = FramebufferByteSize(vfb);
 		u32 fb_address_mem = (params.fb_address & 0x3FFFFFFF) | 0x04000000;
@@ -588,7 +587,8 @@ void FramebufferManagerCommon::NotifyRenderFramebufferSwitched(VirtualFramebuffe
 
 	// Copy depth pixel value from the read framebuffer to the draw framebuffer
 	if (prevVfb && !g_Config.bDisableSlowFramebufEffects) {
-		if (!prevVfb->fbo || !vfb->fbo || !useBufferedRendering_ || !prevVfb->depthUpdated || isClearingDepth) {
+		bool hasNewerDepth = prevVfb->last_frame_depth_render != 0 && prevVfb->last_frame_depth_render >= vfb->last_frame_depth_updated;
+		if (!prevVfb->fbo || !vfb->fbo || !useBufferedRendering_ || !hasNewerDepth || isClearingDepth) {
 			// If depth wasn't updated, then we're at least "two degrees" away from the data.
 			// This is an optimization: it probably doesn't need to be copied in this case.
 		} else {
diff --git a/GPU/Common/FramebufferCommon.h b/GPU/Common/FramebufferCommon.h
index 926f3211c3..13b8a840ec 100644
--- a/GPU/Common/FramebufferCommon.h
+++ b/GPU/Common/FramebufferCommon.h
@@ -79,9 +79,10 @@ struct VirtualFramebuffer {
 	int last_frame_displayed;
 	int last_frame_clut;
 	int last_frame_failed;
+	int last_frame_depth_updated;
+	int last_frame_depth_render;
 	u32 clutUpdatedBytes;
 	bool memoryUpdated;
-	bool depthUpdated;
 	bool firstFrameSaved;
 
 	u32 fb_address;
@@ -283,7 +284,8 @@ public:
 
 	void SetDepthUpdated() {
 		if (currentRenderVfb_) {
-			currentRenderVfb_->depthUpdated = true;
+			currentRenderVfb_->last_frame_depth_render = gpuStats.numFlips;
+			currentRenderVfb_->last_frame_depth_updated = gpuStats.numFlips;
 		}
 	}
 	void SetColorUpdated(int skipDrawReason) {
diff --git a/GPU/D3D11/FramebufferManagerD3D11.cpp b/GPU/D3D11/FramebufferManagerD3D11.cpp
index 075b39ae30..4d29240d2a 100644
--- a/GPU/D3D11/FramebufferManagerD3D11.cpp
+++ b/GPU/D3D11/FramebufferManagerD3D11.cpp
@@ -539,6 +539,7 @@ void FramebufferManagerD3D11::BlitFramebufferDepth(VirtualFramebuffer *src, Virt
 		// TODO: Currently, this copies depth AND stencil, which is a problem.  See #9740.
 		draw_->CopyFramebufferImage(src->fbo, 0, 0, 0, 0, dst->fbo, 0, 0, 0, 0, src->renderWidth, src->renderHeight, 1, Draw::FB_DEPTH_BIT);
 		RebindFramebuffer();
+		dst->last_frame_depth_updated = gpuStats.numFlips;
 	}
 }
 
diff --git a/GPU/GLES/FramebufferManagerGLES.cpp b/GPU/GLES/FramebufferManagerGLES.cpp
index ff379d910e..c51b454706 100644
--- a/GPU/GLES/FramebufferManagerGLES.cpp
+++ b/GPU/GLES/FramebufferManagerGLES.cpp
@@ -529,7 +529,7 @@ void FramebufferManagerGLES::BlitFramebufferDepth(VirtualFramebuffer *src, Virtu
 			// Let's only do this if not clearing depth.
 			glstate.scissorTest.force(false);
 			draw_->BlitFramebuffer(src->fbo, 0, 0, w, h, dst->fbo, 0, 0, w, h, Draw::FB_DEPTH_BIT, Draw::FB_BLIT_NEAREST);
-			// WARNING: If we set dst->depthUpdated here, our optimization above would be pointless.
+			dst->last_frame_depth_updated = gpuStats.numFlips;
 			glstate.scissorTest.restore();
 		}
 	}
diff --git a/GPU/Vulkan/FramebufferVulkan.cpp b/GPU/Vulkan/FramebufferVulkan.cpp
index 36fa278291..f174ffeb31 100644
--- a/GPU/Vulkan/FramebufferVulkan.cpp
+++ b/GPU/Vulkan/FramebufferVulkan.cpp
@@ -412,6 +412,7 @@ void FramebufferManagerVulkan::BlitFramebufferDepth(VirtualFramebuffer *src, Vir
 	if (matchingDepthBuffer && matchingRenderSize && matchingSize) {
 		// TODO: Currently, this copies depth AND stencil, which is a problem.  See #9740.
 		draw_->CopyFramebufferImage(src->fbo, 0, 0, 0, 0, dst->fbo, 0, 0, 0, 0, src->renderWidth, src->renderHeight, 1, Draw::FB_DEPTH_BIT);
+		dst->last_frame_depth_updated = gpuStats.numFlips;
 	} else if (matchingDepthBuffer && matchingSize) {
 		/*
 		int w = std::min(src->renderWidth, dst->renderWidth);