diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h
index 8d8dfcd977..20df538c5e 100644
--- a/GPU/Common/TextureCacheCommon.h
+++ b/GPU/Common/TextureCacheCommon.h
@@ -80,7 +80,8 @@ public:
 			STATUS_CLUT_RECHECK = 0x20,    // Another texture with same addr had a hashfail.
 			STATUS_DEPALETTIZE = 0x40,     // Needs to go through a depalettize pass.
 			STATUS_TO_SCALE = 0x80,        // Pending texture scaling in a later frame.
-			STATUS_FREE_CHANGE = 0x100,    // Allow one change before marking "frequent".
+			STATUS_IS_SCALED = 0x100,      // Has been scaled (can't be replaceImages'd.)
+			STATUS_FREE_CHANGE = 0x200,    // Allow one change before marking "frequent".
 		};
 
 		// Status, but int so we can zero initialize.
diff --git a/GPU/Directx9/TextureCacheDX9.cpp b/GPU/Directx9/TextureCacheDX9.cpp
index b21f04d71f..d0f8976a3a 100644
--- a/GPU/Directx9/TextureCacheDX9.cpp
+++ b/GPU/Directx9/TextureCacheDX9.cpp
@@ -1235,7 +1235,7 @@ bool TextureCacheDX9::HandleTextureChange(TexCacheEntry *const entry, const char
 	gpuStats.numTextureInvalidations++;
 	DEBUG_LOG(G3D, "Texture different or overwritten, reloading at %08x: %s", entry->addr, reason);
 	if (doDelete) {
-		if (initialMatch && standardScaleFactor_ == 1) {
+		if (initialMatch && standardScaleFactor_ == 1 && (entry->status & TexCacheEntry::STATUS_IS_SCALED) == 0) {
 			// Actually, if size and number of levels match, let's try to avoid deleting and recreating.
 			// Instead, let's use glTexSubImage to replace the images.
 			replaceImages = true;
@@ -1244,6 +1244,7 @@ bool TextureCacheDX9::HandleTextureChange(TexCacheEntry *const entry, const char
 				lastBoundTexture = INVALID_TEX;
 			}
 			ReleaseTexture(entry);
+			entry->status &= ~TexCacheEntry::STATUS_IS_SCALED;
 		}
 	}
 	// Clear the reliable bit if set.
@@ -1329,6 +1330,7 @@ void TextureCacheDX9::BuildTexture(TexCacheEntry *const entry, bool replaceImage
 	if (replaced.GetSize(0, w, h)) {
 		// We're replacing, so we won't scale.
 		scaleFactor = 1;
+		entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 		if (g_Config.bMipMap) {
 			maxLevel = replaced.MaxLevel();
 			badMipSizes = false;
@@ -1350,6 +1352,7 @@ void TextureCacheDX9::BuildTexture(TexCacheEntry *const entry, bool replaceImage
 			scaleFactor = 1;
 		} else {
 			entry->status &= ~TexCacheEntry::STATUS_TO_SCALE;
+			entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 			texelsScaledThisFrame_ += w * h;
 		}
 	}
diff --git a/GPU/GLES/TextureCache.cpp b/GPU/GLES/TextureCache.cpp
index 956acad3a6..cb39a2a7ed 100644
--- a/GPU/GLES/TextureCache.cpp
+++ b/GPU/GLES/TextureCache.cpp
@@ -1336,7 +1336,7 @@ bool TextureCache::HandleTextureChange(TexCacheEntry *const entry, const char *r
 	gpuStats.numTextureInvalidations++;
 	DEBUG_LOG(G3D, "Texture different or overwritten, reloading at %08x: %s", entry->addr, reason);
 	if (doDelete) {
-		if (initialMatch && standardScaleFactor_ == 1) {
+		if (initialMatch && standardScaleFactor_ == 1 && (entry->status & TexCacheEntry::STATUS_IS_SCALED) == 0) {
 			// Actually, if size and number of levels match, let's try to avoid deleting and recreating.
 			// Instead, let's use glTexSubImage to replace the images.
 			replaceImages = true;
@@ -1345,6 +1345,7 @@ bool TextureCache::HandleTextureChange(TexCacheEntry *const entry, const char *r
 				lastBoundTexture = INVALID_TEX;
 			}
 			glDeleteTextures(1, &entry->textureName);
+			entry->status &= ~TexCacheEntry::STATUS_IS_SCALED;
 		}
 	}
 	// Clear the reliable bit if set.
@@ -1435,6 +1436,7 @@ void TextureCache::BuildTexture(TexCacheEntry *const entry, bool replaceImages)
 	if (replaced.GetSize(0, w, h)) {
 		// We're replacing, so we won't scale.
 		scaleFactor = 1;
+		entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 		if (g_Config.bMipMap) {
 			maxLevel = replaced.MaxLevel();
 			badMipSizes = false;
@@ -1456,6 +1458,7 @@ void TextureCache::BuildTexture(TexCacheEntry *const entry, bool replaceImages)
 			scaleFactor = 1;
 		} else {
 			entry->status &= ~TexCacheEntry::STATUS_TO_SCALE;
+			entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 			texelsScaledThisFrame_ += w * h;
 		}
 	}
diff --git a/GPU/Vulkan/TextureCacheVulkan.cpp b/GPU/Vulkan/TextureCacheVulkan.cpp
index 2d0a03891c..eb79d4134a 100644
--- a/GPU/Vulkan/TextureCacheVulkan.cpp
+++ b/GPU/Vulkan/TextureCacheVulkan.cpp
@@ -1209,6 +1209,7 @@ bool TextureCacheVulkan::HandleTextureChange(TexCacheEntry *const entry, const c
 		}
 		delete entry->vkTex;
 		entry->vkTex = nullptr;
+		entry->status &= ~TexCacheEntry::STATUS_IS_SCALED;
 	}
 	// Clear the reliable bit if set.
 	if (entry->GetHashStatus() == TexCacheEntry::STATUS_RELIABLE) {
@@ -1290,6 +1291,7 @@ void TextureCacheVulkan::BuildTexture(TexCacheEntry *const entry,VulkanPushBuffe
 	if (replaced.GetSize(0, w, h)) {
 		// We're replacing, so we won't scale.
 		scaleFactor = 1;
+		entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 		if (g_Config.bMipMap) {
 			maxLevel = replaced.MaxLevel();
 			badMipSizes = false;
@@ -1311,6 +1313,7 @@ void TextureCacheVulkan::BuildTexture(TexCacheEntry *const entry,VulkanPushBuffe
 			scaleFactor = 1;
 		} else {
 			entry->status &= ~TexCacheEntry::STATUS_TO_SCALE;
+			entry->status |= TexCacheEntry::STATUS_IS_SCALED;
 			texelsScaledThisFrame_ += w * h;
 		}
 	}