diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp index 83aff9e4bf..018a619ca4 100644 --- a/GPU/Common/TextureCacheCommon.cpp +++ b/GPU/Common/TextureCacheCommon.cpp @@ -503,7 +503,7 @@ void TextureCacheCommon::UpdateMaxSeenV(TexCacheEntry *entry, bool throughMode) if (entry->cluthash != 0 && entry->maxSeenV == 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { // They should all be the same, just make sure we take any that has already increased. // This is for a new texture. if (it->second->maxSeenV != 0) { @@ -535,7 +535,7 @@ void TextureCacheCommon::UpdateMaxSeenV(TexCacheEntry *entry, bool throughMode) if (entry->cluthash != 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { it->second->maxSeenV = entry->maxSeenV; } } @@ -608,14 +608,14 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { u8 maxLevel = gstate.getTextureMaxLevel(); const bool swizzled = gstate.isTextureSwizzled(); - TexCache::iterator entryIter = cache_.find(cachekey); + TexCache::const_iterator entryIter = cache_.Map().find(cachekey); // Note: It's necessary to reset needshadertexclamp, for otherwise DIRTY_TEXCLAMP won't get set later. // Should probably revisit how this works.. gstate_c.SetNeedShaderTexclamp(false); gstate_c.skipDrawReason &= ~SKIPDRAW_BAD_FB_TEXTURE; - if (entryIter != cache_.end()) { + if (entryIter != cache_.Map().end()) { TexCacheEntry *entry = entryIter->second.get(); // Validate the texture still matches the cache entry. @@ -731,7 +731,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { if (rehash) { // Update in case any of these changed. - entry->bufw = bufw; + cache_.SetBufw(entry, bufw); if (entry->cluthash != cluthash) { WARN_LOG(Log::TexCache, "Mysterious cluthash miss: %08x vs %08x", entry->cluthash, cluthash); } @@ -769,15 +769,17 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { if (entry->cluthash != 0) { const u64 cachekeyMin = (u64)(entry->addr & 0x3FFFFFFF) << 32; const u64 cachekeyMax = cachekeyMin + (1ULL << 32); - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { if (it->second->cluthash != entry->cluthash) { it->second->status |= TexStatus::HASH_RECHECK; } } } - // Release the texture from the cache entry, since we're about to throw it away. - entryIter->second.release(); + // Take it out of the cache, since we're about to throw it away. Erasing the slot too, since the + // framebuffer match below can return before it's refilled. + cache_.Release(entryIter); + entryIter = cache_.Map().end(); const u64 secondKeyOld = (u64)entry->fullhash | ((u64)cluthash << 32); @@ -825,13 +827,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // updating the address as appropriate. TexCacheEntry *secondEntry = secondIterNew->second.release(); secondCache_.erase(secondIterNew); - entryIter->second.reset(secondEntry); // Here we move it into the main cache. entry = secondEntry; // Make sure the address is correct. if (entry->addr != texaddr) { DEBUG_LOG(Log::TexCache, "%08x: Texture matched with secondary cache entry with hash %08x at other address %08x. Updating address.", texaddr, newFullHash, entry->addr); entry->addr = texaddr; } + cache_.Insert(cachekey, entry); // Here we move it into the main cache. return ApplyTextureFinish(entry, doBind); } else { DEBUG_LOG(Log::TexCache, "%08x: Entry in secondary cache not suitable, ignoring and creating new: %08x (%dx%d)", texaddr, newFullHash, w, h); @@ -845,10 +847,7 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { DEBUG_LOG(Log::TexCache, "%08x: No entry for hash %08x in secondary cache, creating new in main cache.", texaddr, newFullHash); } } - // The slot was released above. Erase it, since the framebuffer match below can return before it's refilled. - cache_.erase(entryIter); entry = nullptr; - entryIter = cache_.end(); } else { // The texture didn't change after checking the hash, so we can just use it as-is. return ApplyTextureFinish(entry, doBind); @@ -862,13 +861,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { // Wasn't a match even in format. Let's just delete it right away, since we know we need to rebuild it, // and it's unlikely that putting it in the secondary cache will do us any good. We do that for things we rehash, though. ReleaseTexture(entryIter->second.get(), true); - cache_.erase(entryIter); - entryIter = cache_.end(); // make sure we don't look at it again. + cache_.Erase(entryIter); + entryIter = cache_.Map().end(); // make sure we don't look at it again. entry = nullptr; } // If we fall out here, the iterator must be gone and entry must be nullptr. _dbg_assert_(entry == nullptr); - _dbg_assert_(entryIter == cache_.end()); + _dbg_assert_(entryIter == cache_.Map().end()); } // No texture found when looking up the key, or changed (depending on entry). @@ -915,7 +914,13 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { VERBOSE_LOG(Log::TexCache, "No texture in cache for %08x, decoding...", texaddr); TexCacheEntry *entry = new TexCacheEntry{}; - cache_[cachekey].reset(entry); + entry->addr = texaddr; + entry->dim = dim; + entry->format = texFormat; + entry->maxLevel = maxLevel; + entry->bufw = bufw; + entry->cluthash = cluthash; + cache_.Insert(cachekey, entry); entry->status = {}; if (isPPGE) { // It's the builtin font texture. @@ -932,12 +937,12 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { const u64 cachekeyMax = cachekeyMin + (1ULL << 32); int found = 0; - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { found++; } if (found >= TEXTURE_CLUT_VARIANTS_MIN) { - for (auto it = cache_.lower_bound(cachekeyMin), end = cache_.upper_bound(cachekeyMax); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cachekeyMin), end = cache_.Map().upper_bound(cachekeyMax); it != end; ++it) { it->second->status |= TexStatus::MANY_CLUT_VARIANTS; } @@ -945,13 +950,6 @@ TextureApplyResult TextureCacheCommon::ApplyTexture(bool doBind) { } } - // We have to decode it, let's setup the cache entry first. - entry->addr = texaddr; - entry->dim = dim; - entry->format = texFormat; - entry->maxLevel = maxLevel; - entry->bufw = bufw; - entry->cluthash = cluthash; if (clutInShader) { entry->status |= TexStatus::CLUT8_INDEXED; } @@ -1149,7 +1147,7 @@ void TextureCacheCommon::Decimate(const TexCacheEntry *const exceptThisOne, bool const u32 had = cacheSizeEstimate; ForgetLastTexture(); - for (TexCache::iterator iter = cache_.begin(); iter != cache_.end(); ) { + for (TexCache::const_iterator iter = cache_.Map().begin(); iter != cache_.Map().end(); ) { if (iter->second.get() == exceptThisOne) { ++iter; continue; @@ -1160,7 +1158,7 @@ void TextureCacheCommon::Decimate(const TexCacheEntry *const exceptThisOne, bool DEBUG_LOG(Log::TexCache, "Decimating cached texture at %08x (hash: %08x)", iter->second->addr, iter->second->fullhash); cacheSizeEstimate -= iter->second->EstimateTexMemoryUsage(); ReleaseTexture(iter->second.get(), true); - iter = cache_.erase(iter); + iter = cache_.Erase(iter); } else { ++iter; } @@ -1230,7 +1228,7 @@ void TextureCacheCommon::NotifyFramebuffer(VirtualFramebuffer *framebuffer, Fram u64 cacheKeyEnd = (u64)fb_endAddr << 32; // Color - no need to look in the mirrors. - for (auto it = cache_.lower_bound(cacheKey), end = cache_.upper_bound(cacheKeyEnd); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey), end = cache_.Map().upper_bound(cacheKeyEnd); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } @@ -1240,11 +1238,11 @@ void TextureCacheCommon::NotifyFramebuffer(VirtualFramebuffer *framebuffer, Fram // Games don't use 0x04400000 as far as I know - it has no swizzle effect so kinda useless. cacheKey = (u64)z_addr << 32; cacheKeyEnd = (u64)z_endAddr << 32; - for (auto it = cache_.lower_bound(cacheKey | 0x200000), end = cache_.upper_bound(cacheKeyEnd | 0x200000); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey | 0x200000), end = cache_.Map().upper_bound(cacheKeyEnd | 0x200000); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } - for (auto it = cache_.lower_bound(cacheKey | 0x600000), end = cache_.upper_bound(cacheKeyEnd | 0x600000); it != end; ++it) { + for (auto it = cache_.Map().lower_bound(cacheKey | 0x600000), end = cache_.Map().upper_bound(cacheKeyEnd | 0x600000); it != end; ++it) { it->second->status |= TexStatus::FRAMEBUFFER_OVERLAP; gpuStats.perFrame.numTextureInvalidationsByFramebuffer++; } @@ -1791,7 +1789,7 @@ u32 TexCacheEntry::EstimatePSPSizeMemoryUsage() const { const size_t TextureCacheCommon::CacheSizeEstimate() const { s64 cacheSizeEstimate = 0; - for (auto &[_ , value] : cache_) { + for (auto &[_ , value] : cache_.Map()) { cacheSizeEstimate += value->EstimateTexMemoryUsage(); } return cacheSizeEstimate; @@ -2791,7 +2789,7 @@ void TextureCacheCommon::DeviceRestore(Draw::DrawContext *draw) { void TextureCacheCommon::Clear(bool delete_them) { textureShaderCache_.Clear(); - for (TexCache::iterator iter = cache_.begin(); iter != cache_.end(); ++iter) { + for (TexCache::const_iterator iter = cache_.Map().begin(); iter != cache_.Map().end(); ++iter) { ReleaseTexture(iter->second.get(), delete_them); } // In case the setting was changed, we ALWAYS clear the secondary cache (enabled or not.) @@ -2800,7 +2798,7 @@ void TextureCacheCommon::Clear(bool delete_them) { } if (cache_.size() + secondCache_.size()) { INFO_LOG(Log::TexCache, "Texture cached cleared from %d (s: %d) textures", (int)cache_.size(), (int)secondCache_.size()); - cache_.clear(); + cache_.Clear(); secondCache_.clear(); } @@ -2823,7 +2821,7 @@ void TextureCacheCommon::ReleaseDynamicClut() { void TextureCacheCommon::RecheckAllAfterLoad() { // Decoding them all again stutters, while hashing finds the ones that are still the same. The // secondary cache is keyed by hash, so it's fine as it is. - for (auto &iter : cache_) { + for (auto &iter : cache_.Map()) { iter.second->status |= TexStatus::HASH_RECHECK; } ReleaseDynamicClut(); @@ -2854,23 +2852,84 @@ void TextureCacheCommon::Invalidate(u32 addr, int size, GPUInvalidationType type } } - const u64 startKey = (u64)(addr - LARGEST_TEXTURE_SIZE) << 32; - u64 endKey = (u64)(addr + size + LARGEST_TEXTURE_SIZE) << 32; - if (endKey < startKey) { - endKey = (u64)-1; - } + // Tell every texture that overlaps to rehash. Only those starting at most LARGEST_TEXTURE_SIZE before addr, like + // the walk over the map this replaces. + const u32 searchStart = addr - std::min(addr, (u32)LARGEST_TEXTURE_SIZE); + cache_.ForEachOverlapping(searchStart, addr, addr_end, [](TexCacheEntry *entry) { + entry->status |= TexStatus::HASH_RECHECK; + }); +} - // We just loop through all textures in range, and tell them to rehash. - for (TexCache::iterator iter = cache_.lower_bound(startKey), end = cache_.upper_bound(endKey); iter != end; ++iter) { - auto &entry = iter->second; - u32 texAddr = entry->addr; - // Intentional underestimate here. - u32 texEnd = entry->addr + entry->SizeInRAM() / 2; - // Quick check for overlap. Yes the check is right. - if (addr < texEnd && addr_end > texAddr) { - entry->status |= TexStatus::HASH_RECHECK; +void TexCacheMap::Insert(u64 key, TexCacheEntry *entry) { + std::unique_ptr &slot = map_[key]; + _dbg_assert_msg_(!slot, "Texture %08x already in the cache", entry->addr); + if (slot) { + RemoveRange(slot.get()); + } + slot.reset(entry); + AddRange(entry); +} + +TexCache::const_iterator TexCacheMap::Erase(TexCache::const_iterator it) { + RemoveRange(it->second.get()); + return map_.erase(it); +} + +TexCacheEntry *TexCacheMap::Release(TexCache::const_iterator it) { + // erase() of an empty range turns it into a non-const iterator, to release the unique_ptr. + TexCache::iterator mutableIt = map_.erase(it, it); + TexCacheEntry *entry = mutableIt->second.release(); + RemoveRange(entry); + map_.erase(mutableIt); + return entry; +} + +void TexCacheMap::Clear() { + map_.clear(); + ranges_.clear(); + bigRanges_.clear(); +} + +void TexCacheMap::SetBufw(TexCacheEntry *entry, u16 bufw) { + if (entry->bufw != bufw) { + RemoveRange(entry); + entry->bufw = bufw; + AddRange(entry); + } +} + +void TexCacheMap::AddRange(TexCacheEntry *entry) { + // Intentional underestimate here. + const Range r{ entry->addr, entry->addr + entry->SizeInRAM() / 2, entry }; + if (r.end - r.start > BIG_RANGE) { + bigRanges_.push_back(r); + return; + } + auto it = std::upper_bound(ranges_.begin(), ranges_.end(), r.start, [](u32 a, const Range &other) { return a < other.start; }); + ranges_.insert(it, r); +} + +void TexCacheMap::RemoveRange(TexCacheEntry *entry) { + // Found by its start, unless the entry's addr changed while it was in here, which it mustn't. + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), entry->addr, [](const Range &r, u32 a) { return r.start < a; }); + for (; it != ranges_.end() && it->start == entry->addr; ++it) { + if (it->entry == entry) { + ranges_.erase(it); + return; } } + for (auto big = bigRanges_.begin(); big != bigRanges_.end(); ++big) { + if (big->entry == entry) { + bigRanges_.erase(big); + return; + } + } + _dbg_assert_msg_(false, "Texture %08x not found by its address in the range index", entry->addr); + // Never leave a pointer to it behind. + auto any = std::find_if(ranges_.begin(), ranges_.end(), [entry](const Range &r) { return r.entry == entry; }); + if (any != ranges_.end()) { + ranges_.erase(any); + } } void TextureCacheCommon::InvalidateAll(GPUInvalidationType /*unused*/) { diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h index 9724c4bb37..ae4801ce37 100644 --- a/GPU/Common/TextureCacheCommon.h +++ b/GPU/Common/TextureCacheCommon.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -238,6 +239,69 @@ std::string TexStatusToString(TexStatus status); // Additionally, TexCacheEntry is not that big, maybe it's beneficial to remove the unique_ptr indirection. typedef std::map> TexCache; +// The primary cache: a TexCache, with an index of the RAM range each texture covers, sorted by start, for +// TextureCacheCommon::Invalidate(), which runs for every block transfer (MX vs. ATV Untamed: about a thousand a +// second, each walking hundreds of map nodes to find a few). Entries only come and go through here, so the two +// stay in step, and the map is only handed out const. +class TexCacheMap { +public: + const TexCache &Map() const { return map_; } + size_t size() const { return map_.size(); } + + // Takes ownership. The entry's addr, dim, format and bufw must be set, they make its range. + void Insert(u64 key, TexCacheEntry *entry); + // Deletes the entry, and returns the next. + TexCache::const_iterator Erase(TexCache::const_iterator it); + // Takes the entry out, without deleting it. + TexCacheEntry *Release(TexCache::const_iterator it); + void Clear(); + // bufw is part of the range. + void SetBufw(TexCacheEntry *entry, u16 bufw); + + // Calls func(entry) for the entries whose range overlaps [start, end), among those starting at or after + // searchStart. + template + void ForEachOverlapping(u32 searchStart, u32 start, u32 end, F func) const; + +private: + struct Range { + u32 start; + u32 end; + TexCacheEntry *entry; + }; + // Ranges longer than this go in bigRanges_, so a few large ones don't widen every search. + static constexpr u32 BIG_RANGE = 0x10000; + + void AddRange(TexCacheEntry *entry); + void RemoveRange(TexCacheEntry *entry); + + TexCache map_; + std::vector ranges_; + std::vector bigRanges_; +}; + +template +void TexCacheMap::ForEachOverlapping(u32 searchStart, u32 start, u32 end, F func) const { + _dbg_assert_(ranges_.size() + bigRanges_.size() == map_.size()); + auto overlaps = [&](const Range &r) { + // Quick check for overlap. Yes the check is right. + return r.start >= searchStart && start < r.end && end > r.start; + }; + // A small texture overlapping start begins at most BIG_RANGE before it. + const u32 smallStart = start - std::min(start, BIG_RANGE); + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), smallStart, [](const Range &r, u32 a) { return r.start < a; }); + for (; it != ranges_.end() && it->start < end; ++it) { + if (overlaps(*it)) { + func(it->entry); + } + } + for (const Range &r : bigRanges_) { + if (overlaps(r)) { + func(r.entry); + } + } +} + // Urgh. #ifdef IGNORE #undef IGNORE @@ -399,7 +463,7 @@ public: // Accessors for the debugger. virtual void *GetNativeTextureView(const TexCacheEntry *entry, bool flat) const = 0; - const TexCache &Cache() const { return cache_; } + const TexCache &Cache() const { return cache_.Map(); } const TexCache &SecondCache() const { return secondCache_; } const size_t CacheSizeEstimate() const; @@ -481,7 +545,7 @@ protected: double replacementFrameBudgetSeconds_ = 0.5 / 60.0; // The primary cache uses the texture address along with the clut hash as the key. - TexCache cache_; + TexCacheMap cache_; // The secondary cache uses the texture hash and clut hash as the key. TexCache secondCache_; // Cache keys, with a zero CLUT hash, of the textures that have an indexed replacement colored by the GPU. Those