From dad7777b6f6d5554cd3a59acfc33e115dc9d5903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 20:13:10 +0100 Subject: [PATCH 1/7] Heap-allocate the desc --- GPU/Common/ReplacedTexture.cpp | 20 +++++++++-------- GPU/Common/ReplacedTexture.h | 4 ++-- GPU/Common/TextureReplacer.cpp | 41 +++++++++++++++++----------------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index 8265f19e82..4206734dfb 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -101,21 +101,21 @@ bool ReplacedTexture::IsReady(double budget) { return false; } -void ReplacedTexture::FinishPopulate(const ReplacementDesc &desc) { - logId_ = desc.logId; - levelData_ = desc.cache; +void ReplacedTexture::FinishPopulate(ReplacementDesc *desc) { + logId_ = desc->logId; + levelData_ = desc->cache; // TODO: The rest can be done on the thread. - for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc.filenames.size()); ++i) { - if (desc.filenames[i].empty()) { + for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc->filenames.size()); ++i) { + if (desc->filenames[i].empty()) { // Out of valid mip levels. Bail out. break; } - const Path filename = desc.basePath / desc.filenames[i]; + const Path filename = desc->basePath / desc->filenames[i]; - VFSFileReference *fileRef = vfs_->GetFile(desc.filenames[i].c_str()); + VFSFileReference *fileRef = vfs_->GetFile(desc->filenames[i].c_str()); if (!fileRef) { // If the file doesn't exist, let's just bail immediately here. break; @@ -136,8 +136,8 @@ void ReplacedTexture::FinishPopulate(const ReplacementDesc &desc) { good = PopulateLevel(level, false); // We pad files that have been hashrange'd so they are the same texture size. - level.w = (level.w * desc.w) / desc.newW; - level.h = (level.h * desc.h) / desc.newH; + level.w = (level.w * desc->w) / desc->newW; + level.h = (level.h * desc->h) / desc->newH; if (good && i != 0) { // Check that the mipmap size is correct. Can't load mips of the wrong size. @@ -154,6 +154,8 @@ void ReplacedTexture::FinishPopulate(const ReplacementDesc &desc) { break; } + delete desc; + if (levels_.empty()) { // Bad. SetState(ReplacementState::NOT_FOUND); diff --git a/GPU/Common/ReplacedTexture.h b/GPU/Common/ReplacedTexture.h index 2911c5fef1..5ec3950b72 100644 --- a/GPU/Common/ReplacedTexture.h +++ b/GPU/Common/ReplacedTexture.h @@ -129,7 +129,7 @@ struct ReplacedTexture { bool IsReady(double budget); bool CopyLevelTo(int level, void *out, int rowPitch); - void FinishPopulate(const ReplacementDesc &desc); + void FinishPopulate(ReplacementDesc *desc); std::string logId_; private: @@ -148,7 +148,7 @@ private: std::mutex mutex_; Draw::DataFormat fmt = Draw::DataFormat::UNDEFINED; // NOTE: Right now, the only supported format is Draw::DataFormat::R8G8B8A8_UNORM. - ReplacementState state_ = ReplacementState::UNINITIALIZED; + std::atomic state_ = ReplacementState::UNINITIALIZED; VFSBackend *vfs_ = nullptr; diff --git a/GPU/Common/TextureReplacer.cpp b/GPU/Common/TextureReplacer.cpp index ebe7bb4f25..1ce1fd62ca 100644 --- a/GPU/Common/TextureReplacer.cpp +++ b/GPU/Common/TextureReplacer.cpp @@ -498,23 +498,24 @@ ReplacedTexture *TextureReplacer::FindReplacement(u64 cachekey, u32 hash, int w, } void TextureReplacer::PopulateReplacement(ReplacedTexture *texture, u64 cachekey, u32 hash, int w, int h) { - ReplacementDesc desc; - desc.newW = w; - desc.newH = h; - desc.w = w; - desc.h = h; - desc.cachekey = cachekey; - desc.hash = hash; - desc.basePath = basePath_; - LookupHashRange(cachekey >> 32, desc.newW, desc.newH); + // We pass this to a thread, so can't keep it on the stack. + ReplacementDesc *desc = new ReplacementDesc(); + desc->newW = w; + desc->newH = h; + desc->w = w; + desc->h = h; + desc->cachekey = cachekey; + desc->hash = hash; + desc->basePath = basePath_; + LookupHashRange(cachekey >> 32, desc->newW, desc->newH); if (ignoreAddress_) { cachekey = cachekey & 0xFFFFFFFFULL; } - desc.foundAlias = false; + desc->foundAlias = false; bool ignored = false; - desc.hashfiles = LookupHashFile(cachekey, hash, &desc.foundAlias, &ignored); + desc->hashfiles = LookupHashFile(cachekey, hash, &desc->foundAlias, &ignored); // Early-out for ignored textures, let's not bother even starting a thread task. if (ignored) { @@ -524,22 +525,22 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *texture, u64 cachekey return; } - if (!desc.foundAlias) { + if (!desc->foundAlias) { // We'll just need to generate the names for each level. // By default, we look for png since that's also what's dumped. // For other file formats, use the ini to create aliases. - desc.filenames.resize(MAX_REPLACEMENT_MIP_LEVELS); - for (int level = 0; level < desc.filenames.size(); level++) { - desc.filenames[level] = TextureReplacer::HashName(desc.cachekey, desc.hash, level) + ".png"; + desc->filenames.resize(MAX_REPLACEMENT_MIP_LEVELS); + for (int level = 0; level < desc->filenames.size(); level++) { + desc->filenames[level] = TextureReplacer::HashName(desc->cachekey, desc->hash, level) + ".png"; } - desc.logId = desc.filenames[0]; - desc.hashfiles = desc.filenames[0]; // This is used as the key in the data cache. + desc->logId = desc->filenames[0]; + desc->hashfiles = desc->filenames[0]; // This is used as the key in the data cache. } else { - desc.logId = desc.hashfiles; - SplitString(desc.hashfiles, '|', desc.filenames); + desc->logId = desc->hashfiles; + SplitString(desc->hashfiles, '|', desc->filenames); } - desc.cache = &levelCache_[desc.hashfiles]; + desc->cache = &levelCache_[desc->hashfiles]; texture->FinishPopulate(desc); } From 9b7a1a2ddcd5e0a90036bd7eee258647ac337ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 20:31:37 +0100 Subject: [PATCH 2/7] Move the "Populate" phase onto the thread --- GPU/Common/ReplacedTexture.cpp | 127 +++++++++++++++++---------------- GPU/Common/ReplacedTexture.h | 1 + 2 files changed, 66 insertions(+), 62 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index 4206734dfb..1061abd940 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -90,13 +90,13 @@ bool ReplacedTexture::IsReady(double budget) { _assert_(!threadWaitable_); threadWaitable_ = new LimitedWaitable(); + SetState(ReplacementState::PENDING); g_threadManager.EnqueueTask(new ReplacedTextureTask(vfs_, *this, threadWaitable_)); if (threadWaitable_->WaitFor(budget)) { // If we successfully wait here, we're done. The thread will set state accordingly. _assert_(State() == ReplacementState::ACTIVE || State() == ReplacementState::NOT_FOUND || State() == ReplacementState::CANCEL_INIT); return true; } - SetState(ReplacementState::PENDING); // Still pending on thread. return false; } @@ -104,67 +104,10 @@ bool ReplacedTexture::IsReady(double budget) { void ReplacedTexture::FinishPopulate(ReplacementDesc *desc) { logId_ = desc->logId; levelData_ = desc->cache; - - // TODO: The rest can be done on the thread. - - for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc->filenames.size()); ++i) { - if (desc->filenames[i].empty()) { - // Out of valid mip levels. Bail out. - break; - } - - const Path filename = desc->basePath / desc->filenames[i]; - - VFSFileReference *fileRef = vfs_->GetFile(desc->filenames[i].c_str()); - if (!fileRef) { - // If the file doesn't exist, let's just bail immediately here. - break; - } - - // TODO: Here, if we find a file with multiple built-in mipmap levels, - // we'll have to change a bit how things work... - ReplacedTextureLevel level; - level.file = filename; - - if (i == 0) { - fmt = Draw::DataFormat::R8G8B8A8_UNORM; - } - - bool good; - - level.fileRef = fileRef; - good = PopulateLevel(level, false); - - // We pad files that have been hashrange'd so they are the same texture size. - level.w = (level.w * desc->w) / desc->newW; - level.h = (level.h * desc->h) / desc->newH; - - if (good && i != 0) { - // Check that the mipmap size is correct. Can't load mips of the wrong size. - if (level.w != (levels_[0].w >> i) || level.h != (levels_[0].h >> i)) { - WARN_LOG(G3D, "Replacement mipmap invalid: size=%dx%d, expected=%dx%d (level %d, '%s')", level.w, level.h, levels_[0].w >> i, levels_[0].h >> i, i, filename.c_str()); - good = false; - } - } - - if (good) - levels_.push_back(level); - // Otherwise, we're done loading mips (bad PNG or bad size, either way.) - else - break; - } - - delete desc; - - if (levels_.empty()) { - // Bad. - SetState(ReplacementState::NOT_FOUND); - levelData_ = nullptr; - return; - } - - // Populate the data pointer. + desc_ = desc; SetState(ReplacementState::POPULATED); + + // TODO: What used to be here is now done on the thread task. } bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreError) { @@ -217,9 +160,69 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreErro } void ReplacedTexture::Prepare(VFSBackend *vfs) { - std::unique_lock lock(mutex_); this->vfs_ = vfs; + for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc_->filenames.size()); ++i) { + if (desc_->filenames[i].empty()) { + // Out of valid mip levels. Bail out. + break; + } + + const Path filename = desc_->basePath / desc_->filenames[i]; + + VFSFileReference *fileRef = vfs_->GetFile(desc_->filenames[i].c_str()); + if (!fileRef) { + // If the file doesn't exist, let's just bail immediately here. + break; + } + + // TODO: Here, if we find a file with multiple built-in mipmap levels, + // we'll have to change a bit how things work... + ReplacedTextureLevel level; + level.file = filename; + + if (i == 0) { + fmt = Draw::DataFormat::R8G8B8A8_UNORM; + } + + bool good; + + level.fileRef = fileRef; + good = PopulateLevel(level, false); + + // We pad files that have been hashrange'd so they are the same texture size. + level.w = (level.w * desc_->w) / desc_->newW; + level.h = (level.h * desc_->h) / desc_->newH; + + if (good && i != 0) { + // Check that the mipmap size is correct. Can't load mips of the wrong size. + if (level.w != (levels_[0].w >> i) || level.h != (levels_[0].h >> i)) { + WARN_LOG(G3D, "Replacement mipmap invalid: size=%dx%d, expected=%dx%d (level %d, '%s')", level.w, level.h, levels_[0].w >> i, levels_[0].h >> i, i, filename.c_str()); + good = false; + } + } + + if (good) + levels_.push_back(level); + // Otherwise, we're done loading mips (bad PNG or bad size, either way.) + else + break; + } + + delete desc_; + desc_ = nullptr; + + if (levels_.empty()) { + // Bad. + SetState(ReplacementState::NOT_FOUND); + levelData_ = nullptr; + return; + } + + // Populate the data pointer. + + std::unique_lock lock(mutex_); + if (State() == ReplacementState::CANCEL_INIT) { return; } diff --git a/GPU/Common/ReplacedTexture.h b/GPU/Common/ReplacedTexture.h index 5ec3950b72..93ff79b613 100644 --- a/GPU/Common/ReplacedTexture.h +++ b/GPU/Common/ReplacedTexture.h @@ -151,6 +151,7 @@ private: std::atomic state_ = ReplacementState::UNINITIALIZED; VFSBackend *vfs_ = nullptr; + ReplacementDesc *desc_ = nullptr; friend class TextureReplacer; friend class ReplacedTextureTask; From 84c8fee1f23ff29c201fe246d8980eb6f7dd30f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 20:41:43 +0100 Subject: [PATCH 3/7] Interleave calls to Populate / PrepareData --- GPU/Common/ReplacedTexture.cpp | 51 +++++++++++++++------------------- GPU/Common/ReplacedTexture.h | 2 +- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index 1061abd940..ac09a4d05e 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -162,7 +162,13 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreErro void ReplacedTexture::Prepare(VFSBackend *vfs) { this->vfs_ = vfs; + std::unique_lock lock(mutex_); + for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc_->filenames.size()); ++i) { + if (State() == ReplacementState::CANCEL_INIT) { + break; + } + if (desc_->filenames[i].empty()) { // Out of valid mip levels. Bail out. break; @@ -202,8 +208,13 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { } } - if (good) - levels_.push_back(level); + if (good) { + if (PrepareData(level, i)) { + levels_.push_back(level); + } else { + break; + } + } // Otherwise, we're done loading mips (bad PNG or bad size, either way.) else break; @@ -219,33 +230,18 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { return; } - // Populate the data pointer. - - std::unique_lock lock(mutex_); - - if (State() == ReplacementState::CANCEL_INIT) { - return; - } - - for (int i = 0; i < (int)levels_.size(); ++i) { - if (State() == ReplacementState::CANCEL_INIT) - break; - PrepareData(i); - } + SetState(ReplacementState::ACTIVE); if (threadWaitable_) threadWaitable_->Notify(); } -void ReplacedTexture::PrepareData(int level) { - _assert_msg_((size_t)level < levels_.size(), "Invalid miplevel"); +bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { _assert_msg_(levelData_ != nullptr, "Level cache not set"); // We must lock around access to levelData_ in case two textures try to load it at once. std::lock_guard guard(levelData_->lock); - const ReplacedTextureLevel &info = levels_[level]; - if (levelData_->data.size() <= level) { levelData_->data.resize(level + 1); } @@ -254,8 +250,7 @@ void ReplacedTexture::PrepareData(int level) { // Already populated from cache. if (!out.empty()) { - SetState(ReplacementState::ACTIVE); - return; + return true; } ReplacedImageType imageType; @@ -276,14 +271,14 @@ void ReplacedTexture::PrepareData(int level) { ERROR_LOG(G3D, "Failed to allocate memory for texture replacement"); SetState(ReplacementState::NOT_FOUND); cleanup(); - return; + return false; } if (vfs_->Read(openFile, &zim[0], fileSize) != fileSize) { ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM", info.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); - return; + return false; } int w, h, f; @@ -293,7 +288,7 @@ void ReplacedTexture::PrepareData(int level) { ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); - return; + return false; } out.resize(info.w * info.h * 4); @@ -322,13 +317,13 @@ void ReplacedTexture::PrepareData(int level) { ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s (zip)", info.file.c_str(), png.message); SetState(ReplacementState::NOT_FOUND); cleanup(); - return; + return false; } if (png.width > (uint32_t)info.w || png.height > (uint32_t)info.h) { ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); - return; + return false; } bool checkedAlpha = false; @@ -347,7 +342,7 @@ void ReplacedTexture::PrepareData(int level) { SetState(ReplacementState::NOT_FOUND); cleanup(); out.resize(0); - return; + return false; } png_image_free(&png); @@ -360,8 +355,8 @@ void ReplacedTexture::PrepareData(int level) { } } - SetState(ReplacementState::ACTIVE); cleanup(); + return true; } void ReplacedTexture::PurgeIfOlder(double t) { diff --git a/GPU/Common/ReplacedTexture.h b/GPU/Common/ReplacedTexture.h index 93ff79b613..ebd4a323d6 100644 --- a/GPU/Common/ReplacedTexture.h +++ b/GPU/Common/ReplacedTexture.h @@ -134,7 +134,7 @@ struct ReplacedTexture { private: void Prepare(VFSBackend *vfs); - void PrepareData(int level); + bool PrepareData(const ReplacedTextureLevel &info, int level); void PurgeIfOlder(double t); bool PopulateLevel(ReplacedTextureLevel & level, bool ignoreError); From 6ece79c3fae7dbb5f267862141a13a02872c5083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 21:06:33 +0100 Subject: [PATCH 4/7] Move PopulateLevel and PrepareData next to each other, for later merging --- GPU/Common/ReplacedTexture.cpp | 98 +++++++++++++++++----------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index ac09a4d05e..e913534b94 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -110,55 +110,6 @@ void ReplacedTexture::FinishPopulate(ReplacementDesc *desc) { // TODO: What used to be here is now done on the thread task. } -bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreError) { - bool good = false; - - if (!level.fileRef) { - if (!ignoreError) - ERROR_LOG(G3D, "Error opening replacement texture file '%s' in textures.zip", level.file.c_str()); - return false; - } - - size_t fileSize; - VFSOpenFile *file = vfs_->OpenFileForRead(level.fileRef, &fileSize); - if (!file) { - return false; - } - - std::string magic; - auto imageType = Identify(vfs_, file, &magic); - - if (imageType == ReplacedImageType::ZIM) { - uint32_t ignore = 0; - struct ZimHeader { - uint32_t magic; - uint32_t w; - uint32_t h; - uint32_t flags; - } header; - good = vfs_->Read(file, &header, sizeof(header)) == sizeof(header); - level.w = header.w; - level.h = header.h; - good = (header.flags & ZIM_FORMAT_MASK) == ZIM_RGBA8888; - } else if (imageType == ReplacedImageType::PNG) { - PNGHeaderPeek headerPeek; - good = vfs_->Read(file, &headerPeek, sizeof(headerPeek)) == sizeof(headerPeek); - if (good && headerPeek.IsValidPNGHeader()) { - level.w = headerPeek.Width(); - level.h = headerPeek.Height(); - good = true; - } else { - ERROR_LOG(G3D, "Could not get PNG dimensions: %s (zip)", level.file.ToVisualString().c_str()); - good = false; - } - } else { - ERROR_LOG(G3D, "Could not load texture replacement info: %s - unsupported format %s", level.file.ToVisualString().c_str(), magic.c_str()); - } - vfs_->CloseFile(file); - - return good; -} - void ReplacedTexture::Prepare(VFSBackend *vfs) { this->vfs_ = vfs; @@ -236,6 +187,55 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { threadWaitable_->Notify(); } +bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreError) { + bool good = false; + + if (!level.fileRef) { + if (!ignoreError) + ERROR_LOG(G3D, "Error opening replacement texture file '%s' in textures.zip", level.file.c_str()); + return false; + } + + size_t fileSize; + VFSOpenFile *file = vfs_->OpenFileForRead(level.fileRef, &fileSize); + if (!file) { + return false; + } + + std::string magic; + auto imageType = Identify(vfs_, file, &magic); + + if (imageType == ReplacedImageType::ZIM) { + uint32_t ignore = 0; + struct ZimHeader { + uint32_t magic; + uint32_t w; + uint32_t h; + uint32_t flags; + } header; + good = vfs_->Read(file, &header, sizeof(header)) == sizeof(header); + level.w = header.w; + level.h = header.h; + good = (header.flags & ZIM_FORMAT_MASK) == ZIM_RGBA8888; + } else if (imageType == ReplacedImageType::PNG) { + PNGHeaderPeek headerPeek; + good = vfs_->Read(file, &headerPeek, sizeof(headerPeek)) == sizeof(headerPeek); + if (good && headerPeek.IsValidPNGHeader()) { + level.w = headerPeek.Width(); + level.h = headerPeek.Height(); + good = true; + } else { + ERROR_LOG(G3D, "Could not get PNG dimensions: %s (zip)", level.file.ToVisualString().c_str()); + good = false; + } + } else { + ERROR_LOG(G3D, "Could not load texture replacement info: %s - unsupported format %s", level.file.ToVisualString().c_str(), magic.c_str()); + } + vfs_->CloseFile(file); + + return good; +} + bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { _assert_msg_(levelData_ != nullptr, "Level cache not set"); From 265ae5d0e91b4d7246ad28773a645575f4999b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 23:28:11 +0100 Subject: [PATCH 5/7] Make the two methods more compatible --- GPU/Common/ReplacedTexture.cpp | 85 +++++++++++++++++----------------- GPU/Common/ReplacedTexture.h | 2 +- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index e913534b94..e296f821b6 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -115,6 +115,11 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { std::unique_lock lock(mutex_); + _assert_msg_(levelData_ != nullptr, "Level cache not set"); + + // We must lock around access to levelData_ in case two textures try to load it at once. + std::lock_guard guard(levelData_->lock); + for (int i = 0; i < std::min(MAX_REPLACEMENT_MIP_LEVELS, (int)desc_->filenames.size()); ++i) { if (State() == ReplacementState::CANCEL_INIT) { break; @@ -145,19 +150,7 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { bool good; level.fileRef = fileRef; - good = PopulateLevel(level, false); - - // We pad files that have been hashrange'd so they are the same texture size. - level.w = (level.w * desc_->w) / desc_->newW; - level.h = (level.h * desc_->h) / desc_->newH; - - if (good && i != 0) { - // Check that the mipmap size is correct. Can't load mips of the wrong size. - if (level.w != (levels_[0].w >> i) || level.h != (levels_[0].h >> i)) { - WARN_LOG(G3D, "Replacement mipmap invalid: size=%dx%d, expected=%dx%d (level %d, '%s')", level.w, level.h, levels_[0].w >> i, levels_[0].h >> i, i, filename.c_str()); - good = false; - } - } + good = PopulateLevel(level, i, false); if (good) { if (PrepareData(level, i)) { @@ -187,7 +180,7 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { threadWaitable_->Notify(); } -bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreError) { +bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, int mipLevel, bool ignoreError) { bool good = false; if (!level.fileRef) { @@ -233,20 +226,28 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, bool ignoreErro } vfs_->CloseFile(file); + // We pad files that have been hashrange'd so they are the same texture size. + level.w = (level.w * desc_->w) / desc_->newW; + level.h = (level.h * desc_->h) / desc_->newH; + + if (good && mipLevel != 0) { + // Check that the mipmap size is correct. Can't load mips of the wrong size. + if (level.w != (levels_[0].w >> mipLevel) || level.h != (levels_[0].h >> mipLevel)) { + WARN_LOG(G3D, "Replacement mipmap invalid: size=%dx%d, expected=%dx%d (level %d)", + level.w, level.h, levels_[0].w >> mipLevel, levels_[0].h >> mipLevel, mipLevel); + good = false; + } + } + return good; } -bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { - _assert_msg_(levelData_ != nullptr, "Level cache not set"); - - // We must lock around access to levelData_ in case two textures try to load it at once. - std::lock_guard guard(levelData_->lock); - - if (levelData_->data.size() <= level) { - levelData_->data.resize(level + 1); +bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &level, int mipLevel) { + if (levelData_->data.size() <= mipLevel) { + levelData_->data.resize(mipLevel + 1); } - std::vector &out = levelData_->data[level]; + std::vector &out = levelData_->data[mipLevel]; // Already populated from cache. if (!out.empty()) { @@ -256,7 +257,7 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { ReplacedImageType imageType; size_t fileSize; - VFSOpenFile *openFile = vfs_->OpenFileForRead(info.fileRef, &fileSize); + VFSOpenFile *openFile = vfs_->OpenFileForRead(level.fileRef, &fileSize); std::string magic; imageType = Identify(vfs_, openFile, &magic); @@ -275,7 +276,7 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { } if (vfs_->Read(openFile, &zim[0], fileSize) != fileSize) { - ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM", info.file.c_str()); + ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM", level.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); return false; @@ -284,26 +285,26 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { int w, h, f; uint8_t *image; if (LoadZIMPtr(&zim[0], fileSize, &w, &h, &f, &image)) { - if (w > info.w || h > info.h) { - ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str()); + if (w > level.w || h > level.h) { + ERROR_LOG(G3D, "Texture replacement changed since header read: %s", level.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); return false; } - out.resize(info.w * info.h * 4); - if (w == info.w) { - memcpy(&out[0], image, info.w * 4 * info.h); + out.resize(level.w * level.h * 4); + if (w == level.w) { + memcpy(&out[0], image, level.w * 4 * level.h); } else { for (int y = 0; y < h; ++y) { - memcpy(&out[info.w * 4 * y], image + w * 4 * y, w * 4); + memcpy(&out[level.w * 4 * y], image + w * 4 * y, w * 4); } } free(image); } - CheckAlphaResult res = CheckAlpha32Rect((u32 *)&out[0], info.w, w, h, 0xFF000000); - if (res == CHECKALPHA_ANY || level == 0) { + CheckAlphaResult res = CheckAlpha32Rect((u32 *)&out[0], level.w, w, h, 0xFF000000); + if (res == CHECKALPHA_ANY || mipLevel == 0) { alphaStatus_ = ReplacedTextureAlpha(res); } } else if (imageType == ReplacedImageType::PNG) { @@ -314,13 +315,13 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { pngdata.resize(fileSize); pngdata.resize(vfs_->Read(openFile, &pngdata[0], fileSize)); if (!png_image_begin_read_from_memory(&png, &pngdata[0], pngdata.size())) { - ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s (zip)", info.file.c_str(), png.message); + ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s (zip)", level.file.c_str(), png.message); SetState(ReplacementState::NOT_FOUND); cleanup(); return false; } - if (png.width > (uint32_t)info.w || png.height > (uint32_t)info.h) { - ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str()); + if (png.width > (uint32_t)level.w || png.height > (uint32_t)level.h) { + ERROR_LOG(G3D, "Texture replacement changed since header read: %s", level.file.c_str()); SetState(ReplacementState::NOT_FOUND); cleanup(); return false; @@ -329,16 +330,16 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { bool checkedAlpha = false; if ((png.format & PNG_FORMAT_FLAG_ALPHA) == 0) { // Well, we know for sure it doesn't have alpha. - if (level == 0) { + if (mipLevel == 0) { alphaStatus_ = ReplacedTextureAlpha::FULL; } checkedAlpha = true; } png.format = PNG_FORMAT_RGBA; - out.resize(info.w * info.h * 4); - if (!png_image_finish_read(&png, nullptr, &out[0], info.w * 4, nullptr)) { - ERROR_LOG(G3D, "Could not load texture replacement: %s - %s", info.file.c_str(), png.message); + out.resize(level.w * level.h * 4); + if (!png_image_finish_read(&png, nullptr, &out[0], level.w * 4, nullptr)) { + ERROR_LOG(G3D, "Could not load texture replacement: %s - %s", level.file.c_str(), png.message); SetState(ReplacementState::NOT_FOUND); cleanup(); out.resize(0); @@ -348,8 +349,8 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &info, int level) { if (!checkedAlpha) { // This will only check the hashed bits. - CheckAlphaResult res = CheckAlpha32Rect((u32 *)&out[0], info.w, png.width, png.height, 0xFF000000); - if (res == CHECKALPHA_ANY || level == 0) { + CheckAlphaResult res = CheckAlpha32Rect((u32 *)&out[0], level.w, png.width, png.height, 0xFF000000); + if (res == CHECKALPHA_ANY || mipLevel == 0) { alphaStatus_ = ReplacedTextureAlpha(res); } } diff --git a/GPU/Common/ReplacedTexture.h b/GPU/Common/ReplacedTexture.h index ebd4a323d6..98b4a8c941 100644 --- a/GPU/Common/ReplacedTexture.h +++ b/GPU/Common/ReplacedTexture.h @@ -137,7 +137,7 @@ private: bool PrepareData(const ReplacedTextureLevel &info, int level); void PurgeIfOlder(double t); - bool PopulateLevel(ReplacedTextureLevel & level, bool ignoreError); + bool PopulateLevel(ReplacedTextureLevel &level, int mipLevel, bool ignoreError); std::vector levels_; ReplacedLevelsCache *levelData_ = nullptr; From 3dc2463b5529c2f24ec659b3ba54e584611b519f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 23:36:00 +0100 Subject: [PATCH 6/7] Simplify, avoid opening file twice (still rewinding!) --- GPU/Common/ReplacedTexture.cpp | 54 +++++++++++----------------------- GPU/Common/ReplacedTexture.h | 4 +-- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/GPU/Common/ReplacedTexture.cpp b/GPU/Common/ReplacedTexture.cpp index e296f821b6..ac5e54f599 100644 --- a/GPU/Common/ReplacedTexture.cpp +++ b/GPU/Common/ReplacedTexture.cpp @@ -147,21 +147,14 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { fmt = Draw::DataFormat::R8G8B8A8_UNORM; } - bool good; - level.fileRef = fileRef; - good = PopulateLevel(level, i, false); - if (good) { - if (PrepareData(level, i)) { - levels_.push_back(level); - } else { - break; - } - } - // Otherwise, we're done loading mips (bad PNG or bad size, either way.) - else + if (LoadLevelData(level, i)) { + levels_.push_back(level); + } else { + // Otherwise, we're done loading mips (bad PNG or bad size, either way.) break; + } } delete desc_; @@ -180,23 +173,17 @@ void ReplacedTexture::Prepare(VFSBackend *vfs) { threadWaitable_->Notify(); } -bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, int mipLevel, bool ignoreError) { +bool ReplacedTexture::LoadLevelData(ReplacedTextureLevel &level, int mipLevel) { bool good = false; - if (!level.fileRef) { - if (!ignoreError) - ERROR_LOG(G3D, "Error opening replacement texture file '%s' in textures.zip", level.file.c_str()); - return false; - } - size_t fileSize; - VFSOpenFile *file = vfs_->OpenFileForRead(level.fileRef, &fileSize); - if (!file) { + VFSOpenFile *openFile = vfs_->OpenFileForRead(level.fileRef, &fileSize); + if (!openFile) { return false; } std::string magic; - auto imageType = Identify(vfs_, file, &magic); + ReplacedImageType imageType = Identify(vfs_, openFile, &magic); if (imageType == ReplacedImageType::ZIM) { uint32_t ignore = 0; @@ -206,13 +193,13 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, int mipLevel, b uint32_t h; uint32_t flags; } header; - good = vfs_->Read(file, &header, sizeof(header)) == sizeof(header); + good = vfs_->Read(openFile, &header, sizeof(header)) == sizeof(header); level.w = header.w; level.h = header.h; good = (header.flags & ZIM_FORMAT_MASK) == ZIM_RGBA8888; } else if (imageType == ReplacedImageType::PNG) { PNGHeaderPeek headerPeek; - good = vfs_->Read(file, &headerPeek, sizeof(headerPeek)) == sizeof(headerPeek); + good = vfs_->Read(openFile, &headerPeek, sizeof(headerPeek)) == sizeof(headerPeek); if (good && headerPeek.IsValidPNGHeader()) { level.w = headerPeek.Width(); level.h = headerPeek.Height(); @@ -224,9 +211,8 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, int mipLevel, b } else { ERROR_LOG(G3D, "Could not load texture replacement info: %s - unsupported format %s", level.file.ToVisualString().c_str(), magic.c_str()); } - vfs_->CloseFile(file); - // We pad files that have been hashrange'd so they are the same texture size. + // Is this really the right place to do it? level.w = (level.w * desc_->w) / desc_->newW; level.h = (level.h * desc_->h) / desc_->newH; @@ -239,10 +225,10 @@ bool ReplacedTexture::PopulateLevel(ReplacedTextureLevel &level, int mipLevel, b } } - return good; -} + if (!good) { + return false; + } -bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &level, int mipLevel) { if (levelData_->data.size() <= mipLevel) { levelData_->data.resize(mipLevel + 1); } @@ -254,18 +240,12 @@ bool ReplacedTexture::PrepareData(const ReplacedTextureLevel &level, int mipLeve return true; } - ReplacedImageType imageType; - - size_t fileSize; - VFSOpenFile *openFile = vfs_->OpenFileForRead(level.fileRef, &fileSize); - - std::string magic; - imageType = Identify(vfs_, openFile, &magic); - auto cleanup = [&] { vfs_->CloseFile(openFile); }; + vfs_->Rewind(openFile); + if (imageType == ReplacedImageType::ZIM) { std::unique_ptr zim(new uint8_t[fileSize]); if (!zim) { diff --git a/GPU/Common/ReplacedTexture.h b/GPU/Common/ReplacedTexture.h index 98b4a8c941..56868d5689 100644 --- a/GPU/Common/ReplacedTexture.h +++ b/GPU/Common/ReplacedTexture.h @@ -134,11 +134,9 @@ struct ReplacedTexture { private: void Prepare(VFSBackend *vfs); - bool PrepareData(const ReplacedTextureLevel &info, int level); + bool LoadLevelData(ReplacedTextureLevel &info, int level); void PurgeIfOlder(double t); - bool PopulateLevel(ReplacedTextureLevel &level, int mipLevel, bool ignoreError); - std::vector levels_; ReplacedLevelsCache *levelData_ = nullptr; From 8e3428438ecd59da127c66ef83fd552aeae28ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 10 Mar 2023 23:58:15 +0100 Subject: [PATCH 7/7] Replacement: Fix mip level loading. Fix bug if LoadIni gets rerun for some reason. --- GPU/Common/TextureCacheCommon.cpp | 3 ++- GPU/Common/TextureReplacer.cpp | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp index 5c056eb87f..d852267f5c 100644 --- a/GPU/Common/TextureCacheCommon.cpp +++ b/GPU/Common/TextureCacheCommon.cpp @@ -2829,8 +2829,9 @@ bool TextureCacheCommon::PrepareBuildTexture(BuildTexturePlan &plan, TexCacheEnt if (plan.replaceValid) { // We're replacing, so we won't scale. plan.scaleFactor = 1; + // We're ignoring how many levels were specified - instead we just load all available from the replacer. plan.levelsToLoad = plan.replaced->NumLevels(); - plan.levelsToCreate = std::min(plan.levelsToLoad, plan.levelsToCreate); + plan.levelsToCreate = plan.levelsToLoad; // Or more, if we wanted to generate. plan.badMipSizes = false; // But, we still need to create the texture at a larger size. plan.replaced->GetSize(0, &plan.createW, &plan.createH); diff --git a/GPU/Common/TextureReplacer.cpp b/GPU/Common/TextureReplacer.cpp index 1ce1fd62ca..0a3889c062 100644 --- a/GPU/Common/TextureReplacer.cpp +++ b/GPU/Common/TextureReplacer.cpp @@ -191,6 +191,13 @@ bool TextureReplacer::LoadIni() { } vfs_ = dir; + + // If we have stuff loaded from before, need to update the vfs pointers to avoid + // crash on exit. The actual problem is that we tend to call LoadIni a little too much... + for (auto &repl : cache_) { + repl.second->vfs_ = vfs_; + } + INFO_LOG(G3D, "Texture pack activated from '%s'", basePath_.c_str()); // The ini doesn't have to exist for the texture directory or zip to be valid.