diff --git a/pcsx2/GS/GSExtra.h b/pcsx2/GS/GSExtra.h
index 2941f704a8..5598778b70 100644
--- a/pcsx2/GS/GSExtra.h
+++ b/pcsx2/GS/GSExtra.h
@@ -107,7 +107,7 @@ inline FILE* px_fopen(const std::string& filename, const std::string& mode)
 #ifdef ENABLE_ACCURATE_BUFFER_EMULATION
 static const GSVector2i default_rt_size(2048, 2048);
 #else
-static const GSVector2i default_rt_size(1280, 1024);
+static const GSVector2i default_rt_size(0, 0);
 #endif
 
 // Maximum texture size to skip preload/hash path.
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 1b1c958811..c3dd259101 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -135,7 +135,7 @@ void GSRendererHW::SetScaling()
 
 	// No need to resize for native/custom resolutions as default size will be enough for native and we manually get RT Buffer size for custom.
 	// don't resize until the display rectangle and register states are stabilized.
-	if (GSConfig.UpscaleMultiplier <= 1 || good_rt_size)
+	if (good_rt_size)
 		return;
 
 	m_tc->RemovePartial();
@@ -293,9 +293,6 @@ void GSRendererHW::Reset()
 
 void GSRendererHW::VSync(u32 field, bool registers_written)
 {
-	//Check if the frame buffer width or display width has changed
-	SetScaling();
-
 	if (m_reset)
 	{
 		m_tc->RemoveAll();
@@ -308,6 +305,9 @@ void GSRendererHW::VSync(u32 field, bool registers_written)
 		m_reset = false;
 	}
 
+	//Check if the frame buffer width or display width has changed
+	SetScaling();
+
 	GSRenderer::VSync(field, registers_written);
 
 	m_tc->IncAge();
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 82b2415460..33522bc93a 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -2243,15 +2243,8 @@ void GSTextureCache::Target::Update()
 	// Alternate
 	// 1/ uses multiple vertex rectangle
 
-	GSVector2i t_size = default_rt_size;
-
-	// Ensure buffer width is at least of the minimum required value.
-	// Probably not necessary but doesn't hurt to be on the safe side.
-	// I've seen some games use buffer sizes over 1024, which might bypass our default limit
-	int buffer_width = m_TEX0.TBW << 6;
-	t_size.x = std::max(buffer_width, t_size.x);
-
-	GSVector4i r = m_dirty.GetDirtyRectAndClear(m_TEX0, t_size);
+	GSVector4i unscaled_size = GSVector4i(GSVector4(m_texture->GetSize()) / GSVector4(m_texture->GetScale()));
+	GSVector4i r = m_dirty.GetDirtyRectAndClear(m_TEX0, GSVector2i(unscaled_size.x, unscaled_size.y));
 
 	if (r.rempty())
 		return;