diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 11f9f725a6..3ade5bd273 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -1614,6 +1614,15 @@ GSTextureCache::HashCacheEntry* GSTextureCache::LookupHashCache(const GIFRegTEX0
 			// so that when the replacement comes back, there's something for it to swap with.
 			can_cache = true;
 		}
+		else if (paltex)
+		{
+			// there's an edge case here; when there's multiple textures with the same vram data, but different
+			// palettes, if we don't replace all of them, the first one to get loaded in will prevent any of the
+			// others from getting tested for replacement. so, disable paltex for the textures when any of the
+			// palette variants have replacements.
+			if (GSTextureReplacements::HasReplacementTextureWithOtherPalette(key))
+				paltex = false;
+		}
 	}
 
 	// if this texture isn't cacheable, bail out now since we don't want to waste time preloading it
diff --git a/pcsx2/GS/Renderers/HW/GSTextureReplacements.cpp b/pcsx2/GS/Renderers/HW/GSTextureReplacements.cpp
index 63284a7d4a..22621d8bb5 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureReplacements.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureReplacements.cpp
@@ -137,6 +137,9 @@ namespace GSTextureReplacements
 	/// Lookup map of texture names to replacements, if they exist.
 	static std::unordered_map<TextureName, std::string> s_replacement_texture_filenames;
 
+	/// Lookup map of texture names without CLUT hash, to know when we need to disable paltex.
+	static std::unordered_set<TextureName> s_replacement_textures_without_clut_hash;
+
 	/// Lookup map of texture names to replacement data which has been cached.
 	static std::unordered_map<TextureName, ReplacementTexture> s_replacement_texture_cache;
 	static std::mutex s_replacement_texture_cache_mutex;
@@ -290,9 +293,11 @@ void GSTextureReplacements::ReloadReplacementMap()
 
 	// clear out the caches
 	{
+		s_replacement_texture_filenames.clear();
+		s_replacement_textures_without_clut_hash.clear();
+
 		std::unique_lock<std::mutex> lock(s_replacement_texture_cache_mutex);
 		s_replacement_texture_cache.clear();
-		s_replacement_texture_filenames.clear();
 		s_pending_async_load_textures.clear();
 		s_async_loaded_textures.clear();
 	}
@@ -321,7 +326,11 @@ void GSTextureReplacements::ReloadReplacementMap()
 			continue;
 
 		DevCon.WriteLn("Found %ux%u replacement '%.*s'", name->Width(), name->Height(), static_cast<int>(filename.size()), filename.data());
-		s_replacement_texture_filenames.emplace(std::move(name.value()), std::move(fd.FileName));
+		s_replacement_texture_filenames.emplace(name.value(), std::move(fd.FileName));
+
+		// zero out the CLUT hash, because we need this for checking if there's any replacements with this hash when using paltex
+		name->CLUTHash = 0;
+		s_replacement_textures_without_clut_hash.insert(name.value());
 	}
 
 	if (GSConfig.PrecacheTextureReplacements)
@@ -369,6 +378,12 @@ u32 GSTextureReplacements::CalcMipmapLevelsForReplacement(u32 width, u32 height)
 	return static_cast<u32>(std::log2(std::max(width, height))) + 1u;
 }
 
+bool GSTextureReplacements::HasReplacementTextureWithOtherPalette(const GSTextureCache::HashCacheKey& hash)
+{
+	const TextureName name(CreateTextureName(hash.WithRemovedCLUTHash(), 0));
+	return s_replacement_textures_without_clut_hash.find(name) != s_replacement_textures_without_clut_hash.end();
+}
+
 GSTexture* GSTextureReplacements::LookupReplacementTexture(const GSTextureCache::HashCacheKey& hash, bool mipmap, bool* pending)
 {
 	const TextureName name(CreateTextureName(hash, 0));
@@ -482,6 +497,7 @@ void GSTextureReplacements::PrecacheReplacementTextures()
 void GSTextureReplacements::ClearReplacementTextures()
 {
 	s_replacement_texture_filenames.clear();
+	s_replacement_textures_without_clut_hash.clear();
 
 	std::unique_lock<std::mutex> lock(s_replacement_texture_cache_mutex);
 	s_replacement_texture_cache.clear();
diff --git a/pcsx2/GS/Renderers/HW/GSTextureReplacements.h b/pcsx2/GS/Renderers/HW/GSTextureReplacements.h
index 4759336976..46c2a60cc2 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureReplacements.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureReplacements.h
@@ -44,6 +44,7 @@ namespace GSTextureReplacements
 
 	u32 CalcMipmapLevelsForReplacement(u32 width, u32 height);
 
+	bool HasReplacementTextureWithOtherPalette(const GSTextureCache::HashCacheKey& hash);
 	GSTexture* LookupReplacementTexture(const GSTextureCache::HashCacheKey& hash, bool mipmap, bool* pending);
 	GSTexture* CreateReplacementTexture(const ReplacementTexture& rtex, const GSVector2& scale, bool mipmap);
 	void ProcessAsyncLoadedTextures();