From 3863e2f90f7476572c4e3d4230303c289a7736b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Tue, 3 Nov 2026 15:03:01 +0100 Subject: [PATCH 1/2] TextureCache: Find the textures a block transfer overlaps through a flat index Invalidate() runs for every block transfer, and walked the cache_ map from 1 MB before the address, setting a flag on the few textures that overlap. MX vs. ATV Untamed does about a thousand a second, each walking about 380 of its 745 textures to find six, and its one 1 MB texture means a tighter bound on texture size wouldn't help. Now an array of the textures' address ranges sits next to cache_, sorted by address, with textures over 64 KB in a short list of their own, so a search only starts 64 KB back. It's updated wherever an entry enters or leaves cache_ (including the moves to and from secondCache_) or changes its bufw. It marks the same textures as the map walk did (compared on every call in MX vs. ATV, Wipeout Pure, God of War: Chains of Olympus and Tekken 6). In MX on a Cortex-A53, Invalidate() went from 889 samples to 27. Co-Authored-By: Claude Opus 5.5 --- GPU/Common/TextureCacheCommon.cpp | 72 +++++++++++++++++++++++++------ GPU/Common/TextureCacheCommon.h | 14 ++++++ 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp index 83aff9e4bf..9838521450 100644 --- a/GPU/Common/TextureCacheCommon.cpp +++ b/GPU/Common/TextureCacheCommon.cpp @@ -731,7 +731,11 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { if (rehash) { // Update in case any of these changed. - entry->bufw = bufw; + if (entry->bufw != bufw) { + RemoveTexRange(entry); + entry->bufw = bufw; + AddTexRange(entry); + } if (entry->cluthash != cluthash) { WARN_LOG(Log::TexCache, "Mysterious cluthash miss: %08x vs %08x", entry->cluthash, cluthash); } @@ -777,6 +781,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { } // Release the texture from the cache entry, since we're about to throw it away. + RemoveTexRange(entry); entryIter->second.release(); const u64 secondKeyOld = (u64)entry->fullhash | ((u64)cluthash << 32); @@ -832,6 +837,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { DEBUG_LOG(Log::TexCache, "%08x: Texture matched with secondary cache entry with hash %08x at other address %08x. Updating address.", texaddr, newFullHash, entry->addr); entry->addr = texaddr; } + AddTexRange(entry); return ApplyTextureFinish(entry, doBind); } else { DEBUG_LOG(Log::TexCache, "%08x: Entry in secondary cache not suitable, ignoring and creating new: %08x (%dx%d)", texaddr, newFullHash, w, h); @@ -862,6 +868,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // Wasn't a match even in format. Let's just delete it right away, since we know we need to rebuild it, // and it's unlikely that putting it in the secondary cache will do us any good. We do that for things we rehash, though. ReleaseTexture(entryIter->second.get(), true); + RemoveTexRange(entryIter->second.get()); cache_.erase(entryIter); entryIter = cache_.end(); // make sure we don't look at it again. entry = nullptr; @@ -952,6 +959,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { entry->maxLevel = maxLevel; entry->bufw = bufw; entry->cluthash = cluthash; + AddTexRange(entry); if (clutInShader) { entry->status |= TexStatus::CLUT8_INDEXED; } @@ -1160,6 +1168,7 @@ void TextureCacheCommon::Decimate(const TexCacheEntry *const exceptThisOne, bool DEBUG_LOG(Log::TexCache, "Decimating cached texture at %08x (hash: %08x)", iter->second->addr, iter->second->fullhash); cacheSizeEstimate -= iter->second->EstimateTexMemoryUsage(); ReleaseTexture(iter->second.get(), true); + RemoveTexRange(iter->second.get()); iter = cache_.erase(iter); } else { ++iter; @@ -2801,6 +2810,8 @@ void TextureCacheCommon::Clear(bool delete_them) { if (cache_.size() + secondCache_.size()) { INFO_LOG(Log::TexCache, "Texture cached cleared from %d (s: %d) textures", (int)cache_.size(), (int)secondCache_.size()); cache_.clear(); + texRanges_.clear(); + bigTexRanges_.clear(); secondCache_.clear(); } @@ -2854,23 +2865,56 @@ void TextureCacheCommon::Invalidate(u32 addr, int size, GPUInvalidationType type } } - const u64 startKey = (u64)(addr - LARGEST_TEXTURE_SIZE) << 32; - u64 endKey = (u64)(addr + size + LARGEST_TEXTURE_SIZE) << 32; - if (endKey < startKey) { - endKey = (u64)-1; - } + _dbg_assert_(texRanges_.size() + bigTexRanges_.size() == cache_.size()); - // We just loop through all textures in range, and tell them to rehash. - for (TexCache::iterator iter = cache_.lower_bound(startKey), end = cache_.upper_bound(endKey); iter != end; ++iter) { - auto &entry = iter->second; - u32 texAddr = entry->addr; - // Intentional underestimate here. - u32 texEnd = entry->addr + entry->SizeInRAM() / 2; + // Tell every texture that overlaps to rehash. Only those starting at most LARGEST_TEXTURE_SIZE before addr, like + // the walk over cache_ this replaces. + const u32 searchStart = addr - std::min(addr, (u32)LARGEST_TEXTURE_SIZE); + auto overlaps = [&](const TexRange &r) { // Quick check for overlap. Yes the check is right. - if (addr < texEnd && addr_end > texAddr) { - entry->status |= TexStatus::HASH_RECHECK; + return r.start >= searchStart && addr < r.end && addr_end > r.start; + }; + // A small texture overlapping addr starts at most BIG_TEX_RANGE before it. + const u32 smallStart = addr - std::min(addr, BIG_TEX_RANGE); + auto it = std::lower_bound(texRanges_.begin(), texRanges_.end(), smallStart, [](const TexRange &r, u32 a) { return r.start < a; }); + for (; it != texRanges_.end() && it->start < addr_end; ++it) { + if (overlaps(*it)) { + it->entry->status |= TexStatus::HASH_RECHECK; } } + for (const TexRange &r : bigTexRanges_) { + if (overlaps(r)) { + r.entry->status |= TexStatus::HASH_RECHECK; + } + } +} + +void TextureCacheCommon::AddTexRange(TexCacheEntry *entry) { + // Intentional underestimate here. + const TexRange r{ entry->addr, entry->addr + entry->SizeInRAM() / 2, entry }; + if (r.end - r.start > BIG_TEX_RANGE) { + bigTexRanges_.push_back(r); + return; + } + auto it = std::upper_bound(texRanges_.begin(), texRanges_.end(), r.start, [](u32 a, const TexRange &other) { return a < other.start; }); + texRanges_.insert(it, r); +} + +void TextureCacheCommon::RemoveTexRange(TexCacheEntry *entry) { + auto it = std::lower_bound(texRanges_.begin(), texRanges_.end(), entry->addr, [](const TexRange &r, u32 a) { return r.start < a; }); + for (; it != texRanges_.end() && it->start == entry->addr; ++it) { + if (it->entry == entry) { + texRanges_.erase(it); + return; + } + } + for (auto big = bigTexRanges_.begin(); big != bigTexRanges_.end(); ++big) { + if (big->entry == entry) { + bigTexRanges_.erase(big); + return; + } + } + _dbg_assert_msg_(false, "Texture %08x missing from texRanges_", entry->addr); } void TextureCacheCommon::InvalidateAll(GPUInvalidationType /*unused*/) { diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h index 9724c4bb37..0acd851fb3 100644 --- a/GPU/Common/TextureCacheCommon.h +++ b/GPU/Common/TextureCacheCommon.h @@ -482,6 +482,20 @@ protected: // The primary cache uses the texture address along with the clut hash as the key. TexCache cache_; + // cache_ by the range of RAM each texture covers, sorted by start, for Invalidate(), which runs for every block + // transfer (MX vs. ATV Untamed: about a thousand a second, each walking hundreds of map nodes to find a few). + // Kept in step with cache_ by AddTexRange()/RemoveTexRange(). Textures over BIG_TEX_RANGE go in a list of their + // own, so a few large ones don't widen every search. + struct TexRange { + u32 start; + u32 end; + TexCacheEntry *entry; + }; + static constexpr u32 BIG_TEX_RANGE = 0x10000; + std::vector texRanges_; + std::vector bigTexRanges_; + void AddTexRange(TexCacheEntry *entry); + void RemoveTexRange(TexCacheEntry *entry); // The secondary cache uses the texture hash and clut hash as the key. TexCache secondCache_; // Cache keys, with a zero CLUT hash, of the textures that have an indexed replacement colored by the GPU. Those From 324f608a94b41e7fd13223608a42905616f730b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Wed, 4 Nov 2026 18:59:56 +0100 Subject: [PATCH 2/2] TextureCache: Keep the range index and the map together in a TexCacheMap The map is only handed out const, and entries only come and go through Insert(), Erase(), Release(), Clear() and SetBufw(), which keep the index in step. Before, every place that changed cache_ had to update the index by hand, and one that didn't would leave a dangling pointer for the next block transfer to write through. A new entry now gets the fields its range needs before it goes in, and the slot of one going to the secondary cache is erased right away. Co-Authored-By: Claude Opus 5.5 --- GPU/Common/TextureCacheCommon.cpp | 167 ++++++++++++++++-------------- GPU/Common/TextureCacheCommon.h | 82 ++++++++++++--- 2 files changed, 157 insertions(+), 92 deletions(-) diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp index 9838521450..018a619ca4 100644 --- a/GPU/Common/TextureCacheCommon.cpp +++ b/GPU/Common/TextureCacheCommon.cpp @@ -503,7 +503,7 @@ void TextureCacheCommon::UpdateMaxSeenV(TexCacheEntry *entry, bool throughMode) if (entry->cluthash != 0 && entry->maxSeenV == 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { // They should all be the same, just make sure we take any that has already increased. // This is for a new texture. if (it->second->maxSeenV != 0) { @@ -535,7 +535,7 @@ void TextureCacheCommon::UpdateMaxSeenV(TexCacheEntry *entry, bool throughMode) if (entry->cluthash != 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { it->second->maxSeenV = entry->maxSeenV; } } @@ -608,14 +608,14 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { u8 maxLevel = gstate.getTextureMaxLevel(); const bool swizzled = gstate.isTextureSwizzled(); - TexCache::iterator entryIter = cache_.find(cachekey); + TexCache::const_iterator entryIter = cache_.Map().find(cachekey); // Note: It's necessary to reset needshadertexclamp, for otherwise DIRTY_TEXCLAMP won't get set later. // Should probably revisit how this works.. gstate_c.SetNeedShaderTexclamp(false); gstate_c.skipDrawReason &= ~SKIPDRAW_BAD_FB_TEXTURE; - if (entryIter != cache_.end()) { + if (entryIter != cache_.Map().end()) { TexCacheEntry *entry = entryIter->second.get(); // Validate the texture still matches the cache entry. @@ -731,11 +731,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { if (rehash) { // Update in case any of these changed. - if (entry->bufw != bufw) { - RemoveTexRange(entry); - entry->bufw = bufw; - AddTexRange(entry); - } + cache_.SetBufw(entry, bufw); if (entry->cluthash != cluthash) { WARN_LOG(Log::TexCache, "Mysterious cluthash miss: %08x vs %08x", entry->cluthash, cluthash); } @@ -773,16 +769,17 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { if (entry->cluthash != 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { if (it->second->cluthash != entry->cluthash) { it->second->status |= TexStatus::HASH_RECHECK; } } } - // Release the texture from the cache entry, since we're about to throw it away. - RemoveTexRange(entry); - entryIter->second.release(); + // Take it out of the cache, since we're about to throw it away. Erasing the slot too, since the + // framebuffer match below can return before it's refilled. + cache_.Release(entryIter); + entryIter = cache_.Map().end(); const u64 secondKeyOld = (u64)entry->fullhash | ((u64)cluthash << 32); @@ -830,14 +827,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // updating the address as appropriate. TexCacheEntry *secondEntry = secondIterNew->second.release(); secondCache_.erase(secondIterNew); - entryIter->second.reset(secondEntry); // Here we move it into the main cache. entry = secondEntry; // Make sure the address is correct. if (entry->addr != texaddr) { DEBUG_LOG(Log::TexCache, "%08x: Texture matched with secondary cache entry with hash %08x at other address %08x. Updating address.", texaddr, newFullHash, entry->addr); entry->addr = texaddr; } - AddTexRange(entry); + cache_.Insert(cachekey, entry); // Here we move it into the main cache. return ApplyTextureFinish(entry, doBind); } else { DEBUG_LOG(Log::TexCache, "%08x: Entry in secondary cache not suitable, ignoring and creating new: %08x (%dx%d)", texaddr, newFullHash, w, h); @@ -851,10 +847,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { DEBUG_LOG(Log::TexCache, "%08x: No entry for hash %08x in secondary cache, creating new in main cache.", texaddr, newFullHash); } } - // The slot was released above. Erase it, since the framebuffer match below can return before it's refilled. - cache_.erase(entryIter); entry = nullptr; - entryIter = cache_.end(); } else { // The texture didn't change after checking the hash, so we can just use it as-is. return ApplyTextureFinish(entry, doBind); @@ -868,14 +861,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // Wasn't a match even in format. Let's just delete it right away, since we know we need to rebuild it, // and it's unlikely that putting it in the secondary cache will do us any good. We do that for things we rehash, though. ReleaseTexture(entryIter->second.get(), true); - RemoveTexRange(entryIter->second.get()); - cache_.erase(entryIter); - entryIter = cache_.end(); // make sure we don't look at it again. + cache_.Erase(entryIter); + entryIter = cache_.Map().end(); // make sure we don't look at it again. entry = nullptr; } // If we fall out here, the iterator must be gone and entry must be nullptr. _dbg_assert_(entry == nullptr); - _dbg_assert_(entryIter == cache_.end()); + _dbg_assert_(entryIter == cache_.Map().end()); } // No texture found when looking up the key, or changed (depending on entry). @@ -922,7 +914,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { VERBOSE_LOG(Log::TexCache, "No texture in cache for %08x, decoding...", texaddr); TexCacheEntry *entry = new TexCacheEntry{}; - cache_[cachekey].reset(entry); + entry->addr = texaddr; + entry->dim = dim; + entry->format = texFormat; + entry->maxLevel = maxLevel; + entry->bufw = bufw; + entry->cluthash = cluthash; + cache_.Insert(cachekey, entry); entry->status = {}; if (isPPGE) { // It's the builtin font texture. @@ -939,12 +937,12 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { const u64 cachekeyMax = cachekeyMin + (1ULL << 32); int found = 0; - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { found++; } if (found >= TEXTURE_CLUT_VARIANTS_MIN) { - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { it->second->status |= TexStatus::MANY_CLUT_VARIANTS; } @@ -952,14 +950,6 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { } } - // We have to decode it, let's setup the cache entry first. - entry->addr = texaddr; - entry->dim = dim; - entry->format = texFormat; - entry->maxLevel = maxLevel; - entry->bufw = bufw; - entry->cluthash = cluthash; - AddTexRange(entry); if (clutInShader) { entry->status |= TexStatus::CLUT8_INDEXED; } @@ -1157,7 +1147,7 @@ void TextureCacheCommon::Decimate(const TexCacheEntry *const exceptThisOne, bool const u32 had = cacheSizeEstimate; ForgetLastTexture(); - for (TexCache::iterator iter = cache_.begin(); iter != cache_.end(); ) { + for (TexCache::const_iterator iter = cache_.Map().begin(); iter != cache_.Map().end(); ) { if (iter->second.get() == exceptThisOne) { ++iter; continue; @@ -1168,8 +1158,7 @@ void TextureCacheCommon::Decimate(const TexCacheEntry *const exceptThisOne, bool DEBUG_LOG(Log::TexCache, "Decimating cached texture at %08x (hash: %08x)", iter->second->addr, iter->second->fullhash); cacheSizeEstimate -= iter->second->EstimateTexMemoryUsage(); ReleaseTexture(iter->second.get(), true); - RemoveTexRange(iter->second.get()); - iter = cache_.erase(iter); + iter = cache_.Erase(iter); } else { ++iter; } @@ -1239,7 +1228,7 @@ void TextureCacheCommon::NotifyFramebuffer(VirtualFramebuffer *framebuffer, Fram u64 cacheKeyEnd = (u64)fb_endAddr << 32; // Color - no need to look in the mirrors. - for (auto it = cache_.lower_bound(cacheKey), end = cache_.upper_bound(cacheKeyEnd); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey), end = cache_.Map().upper_bound(cacheKeyEnd); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } @@ -1249,11 +1238,11 @@ void TextureCacheCommon::NotifyFramebuffer(VirtualFramebuffer *framebuffer, Fram // Games don't use 0x04400000 as far as I know - it has no swizzle effect so kinda useless. cacheKey = (u64)z_addr << 32; cacheKeyEnd = (u64)z_endAddr << 32; - for (auto it = cache_.lower_bound(cacheKey | 0x200000), end = cache_.upper_bound(cacheKeyEnd | 0x200000); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey | 0x200000), end = cache_.Map().upper_bound(cacheKeyEnd | 0x200000); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } - for (auto it = cache_.lower_bound(cacheKey | 0x600000), end = cache_.upper_bound(cacheKeyEnd | 0x600000); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey | 0x600000), end = cache_.Map().upper_bound(cacheKeyEnd | 0x600000); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } @@ -1800,7 +1789,7 @@ u32 TexCacheEntry::EstimatePSPSizeMemoryUsage() const { const size_t TextureCacheCommon::CacheSizeEstimate() const { s64 cacheSizeEstimate = 0; - for (auto &[_ , value] : cache_) { + for (auto &[_ , value] : cache_.Map()) { cacheSizeEstimate += value->EstimateTexMemoryUsage(); } return cacheSizeEstimate; @@ -2800,7 +2789,7 @@ void TextureCacheCommon::DeviceRestore(Draw::DrawContext *draw) { void TextureCacheCommon::Clear(bool delete_them) { textureShaderCache_.Clear(); - for (TexCache::iterator iter = cache_.begin(); iter != cache_.end(); ++iter) { + for (TexCache::const_iterator iter = cache_.Map().begin(); iter != cache_.Map().end(); ++iter) { ReleaseTexture(iter->second.get(), delete_them); } // In case the setting was changed, we ALWAYS clear the secondary cache (enabled or not.) @@ -2809,9 +2798,7 @@ void TextureCacheCommon::Clear(bool delete_them) { } if (cache_.size() + secondCache_.size()) { INFO_LOG(Log::TexCache, "Texture cached cleared from %d (s: %d) textures", (int)cache_.size(), (int)secondCache_.size()); - cache_.clear(); - texRanges_.clear(); - bigTexRanges_.clear(); + cache_.Clear(); secondCache_.clear(); } @@ -2834,7 +2821,7 @@ void TextureCacheCommon::ReleaseDynamicClut() { void TextureCacheCommon::RecheckAllAfterLoad() { // Decoding them all again stutters, while hashing finds the ones that are still the same. The // secondary cache is keyed by hash, so it's fine as it is. - for (auto &iter : cache_) { + for (auto &iter : cache_.Map()) { iter.second->status |= TexStatus::HASH_RECHECK; } ReleaseDynamicClut(); @@ -2865,56 +2852,84 @@ void TextureCacheCommon::Invalidate(u32 addr, int size, GPUInvalidationType type } } - _dbg_assert_(texRanges_.size() + bigTexRanges_.size() == cache_.size()); - // Tell every texture that overlaps to rehash. Only those starting at most LARGEST_TEXTURE_SIZE before addr, like - // the walk over cache_ this replaces. + // the walk over the map this replaces. const u32 searchStart = addr - std::min(addr, (u32)LARGEST_TEXTURE_SIZE); - auto overlaps = [&](const TexRange &r) { - // Quick check for overlap. Yes the check is right. - return r.start >= searchStart && addr < r.end && addr_end > r.start; - }; - // A small texture overlapping addr starts at most BIG_TEX_RANGE before it. - const u32 smallStart = addr - std::min(addr, BIG_TEX_RANGE); - auto it = std::lower_bound(texRanges_.begin(), texRanges_.end(), smallStart, [](const TexRange &r, u32 a) { return r.start < a; }); - for (; it != texRanges_.end() && it->start < addr_end; ++it) { - if (overlaps(*it)) { - it->entry->status |= TexStatus::HASH_RECHECK; - } + cache_.ForEachOverlapping(searchStart, addr, addr_end, [](TexCacheEntry *entry) { + entry->status |= TexStatus::HASH_RECHECK; + }); +} + +void TexCacheMap::Insert(u64 key, TexCacheEntry *entry) { + std::unique_ptr &slot = map_[key]; + _dbg_assert_msg_(!slot, "Texture %08x already in the cache", entry->addr); + if (slot) { + RemoveRange(slot.get()); } - for (const TexRange &r : bigTexRanges_) { - if (overlaps(r)) { - r.entry->status |= TexStatus::HASH_RECHECK; - } + slot.reset(entry); + AddRange(entry); +} + +TexCache::const_iterator TexCacheMap::Erase(TexCache::const_iterator it) { + RemoveRange(it->second.get()); + return map_.erase(it); +} + +TexCacheEntry *TexCacheMap::Release(TexCache::const_iterator it) { + // erase() of an empty range turns it into a non-const iterator, to release the unique_ptr. + TexCache::iterator mutableIt = map_.erase(it, it); + TexCacheEntry *entry = mutableIt->second.release(); + RemoveRange(entry); + map_.erase(mutableIt); + return entry; +} + +void TexCacheMap::Clear() { + map_.clear(); + ranges_.clear(); + bigRanges_.clear(); +} + +void TexCacheMap::SetBufw(TexCacheEntry *entry, u16 bufw) { + if (entry->bufw != bufw) { + RemoveRange(entry); + entry->bufw = bufw; + AddRange(entry); } } -void TextureCacheCommon::AddTexRange(TexCacheEntry *entry) { +void TexCacheMap::AddRange(TexCacheEntry *entry) { // Intentional underestimate here. - const TexRange r{ entry->addr, entry->addr + entry->SizeInRAM() / 2, entry }; - if (r.end - r.start > BIG_TEX_RANGE) { - bigTexRanges_.push_back(r); + const Range r{ entry->addr, entry->addr + entry->SizeInRAM() / 2, entry }; + if (r.end - r.start > BIG_RANGE) { + bigRanges_.push_back(r); return; } - auto it = std::upper_bound(texRanges_.begin(), texRanges_.end(), r.start, [](u32 a, const TexRange &other) { return a < other.start; }); - texRanges_.insert(it, r); + auto it = std::upper_bound(ranges_.begin(), ranges_.end(), r.start, [](u32 a, const Range &other) { return a < other.start; }); + ranges_.insert(it, r); } -void TextureCacheCommon::RemoveTexRange(TexCacheEntry *entry) { - auto it = std::lower_bound(texRanges_.begin(), texRanges_.end(), entry->addr, [](const TexRange &r, u32 a) { return r.start < a; }); - for (; it != texRanges_.end() && it->start == entry->addr; ++it) { +void TexCacheMap::RemoveRange(TexCacheEntry *entry) { + // Found by its start, unless the entry's addr changed while it was in here, which it mustn't. + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), entry->addr, [](const Range &r, u32 a) { return r.start < a; }); + for (; it != ranges_.end() && it->start == entry->addr; ++it) { if (it->entry == entry) { - texRanges_.erase(it); + ranges_.erase(it); return; } } - for (auto big = bigTexRanges_.begin(); big != bigTexRanges_.end(); ++big) { + for (auto big = bigRanges_.begin(); big != bigRanges_.end(); ++big) { if (big->entry == entry) { - bigTexRanges_.erase(big); + bigRanges_.erase(big); return; } } - _dbg_assert_msg_(false, "Texture %08x missing from texRanges_", entry->addr); + _dbg_assert_msg_(false, "Texture %08x not found by its address in the range index", entry->addr); + // Never leave a pointer to it behind. + auto any = std::find_if(ranges_.begin(), ranges_.end(), [entry](const Range &r) { return r.entry == entry; }); + if (any != ranges_.end()) { + ranges_.erase(any); + } } void TextureCacheCommon::InvalidateAll(GPUInvalidationType /*unused*/) { diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h index 0acd851fb3..ae4801ce37 100644 --- a/GPU/Common/TextureCacheCommon.h +++ b/GPU/Common/TextureCacheCommon.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -238,6 +239,69 @@ std::string TexStatusToString(TexStatus status); // Additionally, TexCacheEntry is not that big, maybe it's beneficial to remove the unique_ptr indirection. typedef std::map> TexCache; +// The primary cache: a TexCache, with an index of the RAM range each texture covers, sorted by start, for +// TextureCacheCommon::Invalidate(), which runs for every block transfer (MX vs. ATV Untamed: about a thousand a +// second, each walking hundreds of map nodes to find a few). Entries only come and go through here, so the two +// stay in step, and the map is only handed out const. +class TexCacheMap { +public: + const TexCache &Map() const { return map_; } + size_t size() const { return map_.size(); } + + // Takes ownership. The entry's addr, dim, format and bufw must be set, they make its range. + void Insert(u64 key, TexCacheEntry *entry); + // Deletes the entry, and returns the next. + TexCache::const_iterator Erase(TexCache::const_iterator it); + // Takes the entry out, without deleting it. + TexCacheEntry *Release(TexCache::const_iterator it); + void Clear(); + // bufw is part of the range. + void SetBufw(TexCacheEntry *entry, u16 bufw); + + // Calls func(entry) for the entries whose range overlaps [start, end), among those starting at or after + // searchStart. + template + void ForEachOverlapping(u32 searchStart, u32 start, u32 end, F func) const; + +private: + struct Range { + u32 start; + u32 end; + TexCacheEntry *entry; + }; + // Ranges longer than this go in bigRanges_, so a few large ones don't widen every search. + static constexpr u32 BIG_RANGE = 0x10000; + + void AddRange(TexCacheEntry *entry); + void RemoveRange(TexCacheEntry *entry); + + TexCache map_; + std::vector ranges_; + std::vector bigRanges_; +}; + +template +void TexCacheMap::ForEachOverlapping(u32 searchStart, u32 start, u32 end, F func) const { + _dbg_assert_(ranges_.size() + bigRanges_.size() == map_.size()); + auto overlaps = [&](const Range &r) { + // Quick check for overlap. Yes the check is right. + return r.start >= searchStart && start < r.end && end > r.start; + }; + // A small texture overlapping start begins at most BIG_RANGE before it. + const u32 smallStart = start - std::min(start, BIG_RANGE); + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), smallStart, [](const Range &r, u32 a) { return r.start < a; }); + for (; it != ranges_.end() && it->start < end; ++it) { + if (overlaps(*it)) { + func(it->entry); + } + } + for (const Range &r : bigRanges_) { + if (overlaps(r)) { + func(r.entry); + } + } +} + // Urgh. #ifdef IGNORE #undef IGNORE @@ -399,7 +463,7 @@ public: // Accessors for the debugger. virtual void *GetNativeTextureView(const TexCacheEntry *entry, bool flat) const = 0; - const TexCache &Cache() const { return cache_; } + const TexCache &Cache() const { return cache_.Map(); } const TexCache &SecondCache() const { return secondCache_; } const size_t CacheSizeEstimate() const; @@ -481,21 +545,7 @@ protected: double replacementFrameBudgetSeconds_ = 0.5 / 60.0; // The primary cache uses the texture address along with the clut hash as the key. - TexCache cache_; - // cache_ by the range of RAM each texture covers, sorted by start, for Invalidate(), which runs for every block - // transfer (MX vs. ATV Untamed: about a thousand a second, each walking hundreds of map nodes to find a few). - // Kept in step with cache_ by AddTexRange()/RemoveTexRange(). Textures over BIG_TEX_RANGE go in a list of their - // own, so a few large ones don't widen every search. - struct TexRange { - u32 start; - u32 end; - TexCacheEntry *entry; - }; - static constexpr u32 BIG_TEX_RANGE = 0x10000; - std::vector texRanges_; - std::vector bigTexRanges_; - void AddTexRange(TexCacheEntry *entry); - void RemoveTexRange(TexCacheEntry *entry); + TexCacheMap cache_; // The secondary cache uses the texture hash and clut hash as the key. TexCache secondCache_; // Cache keys, with a zero CLUT hash, of the textures that have an indexed replacement colored by the GPU. Those