From 111f01481cdb862e59bc00b7ecb4a831073aca00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 3 Sep 2026 11:34:53 -0600 Subject: [PATCH] PSAR: don't decrypt entries a filter is going to reject Decrypting an entry's contents is by far the most expensive part of walking an archive, and it happened for every entry before the filter was even consulted. Now it waits until entryData()/entryCompression() asks, so pulling just the fonts out of an updater no longer costs a full firmware decrypt. Records are decrypted independently of each other, so deferring one is safe. Unpacking fonts from a 3.11 updater goes 0.365s -> 0.133s; a full unpack is unchanged and produces identical output. The compression counts now describe the entries we actually decoded rather than everything in the archive. Also adds --unpack-updater-filter to headless, which is how the above was measured. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JvJR8oJNSCimCM9KXVLjfq --- Core/CmdLine.cpp | 1 + Core/CmdLine.h | 1 + Core/Util/PSARUnpack.cpp | 69 ++++++++++++++++++++++++++++------------ headless/Headless.cpp | 7 ++-- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 9f5a388265..46b9742c08 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -209,6 +209,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, {POFF(unpackUpdater), CmdParamType::String, "unpack-updater", '\0', "Unpack the firmware in an updater EBOOT.PBP into DIR and exit", CmdLineMode::Headless}, {POFF(unpackUpdaterModel), CmdParamType::String, "unpack-updater-model", '\0', "PSP model to unpack for (01g..12g, default any)", CmdLineMode::Headless}, + {POFF(unpackUpdaterFilter), CmdParamType::String, "unpack-updater-filter", '\0', "Only unpack entries under this path, e.g. flash0:/font/", CmdLineMode::Headless}, {POFF(odsLog), CmdParamType::Bool, "odslog", 'o', "Also log through OutputDebugString (Windows)", CmdLineMode::Headless}, {POFF(generateInterpreterDispatch), CmdParamType::Bool, "generate-interpreter-dispatch", '\0', "Generate C++ interpreter dispatch code (ExecInstruction) to stdout and exit", CmdLineMode::Headless}, {POFF(resolutionScale), CmdParamType::Int, "resolution-scale", '\0', "Set the resolution scale factor"}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 817262b804..cb45d43042 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -75,6 +75,7 @@ struct CommandLineOptions { // Headless: which PSP model the unpacker resolves names against - "01g".."12g", or "any" // (the default) to take whatever file list names each file first. std::optional unpackUpdaterModel; + std::optional unpackUpdaterFilter; std::optional memReadAction; std::optional memWriteAction; diff --git a/Core/Util/PSARUnpack.cpp b/Core/Util/PSARUnpack.cpp index 7028cc31da..e7aff71097 100644 --- a/Core/Util/PSARUnpack.cpp +++ b/Core/Util/PSARUnpack.cpp @@ -491,9 +491,12 @@ public: const std::string &firmwareVersion() const { return firmwareVersion_; } const std::string &entryName() const { return entryName_; } bool entryIsDirectory() const { return entryIsDirectory_; } - PSARCompression entryCompression() const { return entryCompression_; } + // Decrypting an entry's contents is by far the most expensive thing here, and an archive holds + // well over a thousand of them - so it doesn't happen until one of these two asks for it. An + // entry a filter rejects never gets decrypted at all. + PSARCompression entryCompression() { EnsureContents(); return entryCompression_; } // Empty for a directory, or for an entry we couldn't decompress. - const std::vector &entryData() const { return entryData_; } + const std::vector &entryData() { EnsureContents(); return entryData_; } // How far into the archive the next record starts, and where the records stop - i.e. progress. u32 position() const { return pos_; } size_t limit() const { return limit_; } @@ -501,6 +504,8 @@ public: private: // Decrypts one record into 'out'. Returns the decrypted size, or <= 0 on failure. int DecodeBlock(u32 offset, u32 cbIn, std::vector &out); + // Decodes the current entry's contents, if that hasn't happened yet. + void EnsureContents(); const u8 *psar_; size_t size_; @@ -519,6 +524,13 @@ private: PSARCompression entryCompression_ = PSARCompression::None; std::vector entryData_; + // Where the current entry's contents live, for decoding them later. Records are decrypted + // independently of each other, so it doesn't matter that we've moved past them by then. + bool contentsDecoded_ = true; + u32 contentPos_ = 0; + u32 contentChunkSize_ = 0; + u32 contentExpandedSize_ = 0; + std::vector block_; std::vector block2_; }; @@ -639,6 +651,7 @@ int PSARReader::NextEntry(std::string *error) { entryData_.clear(); entryIsDirectory_ = false; entryCompression_ = PSARCompression::None; + contentsDecoded_ = true; // Nothing to decode until we find out there's a payload. // Stop when what's left can't hold another whole record. Both archives I've looked at end with // a few bytes of padding that would otherwise be decoded as a truncated entry and reported as @@ -673,29 +686,42 @@ int PSARReader::NextEntry(std::string *error) { *error = StringFromFormat("Entry '%s' claims an unreasonable size (%u)", entryName_.c_str(), expandedSize); return -1; } else { - decoded = DecodeBlock(pos_, chunkSize, block2_); - if (decoded <= 0) { - WARN_LOG(Log::Loader, "PSAR: couldn't decrypt the contents of '%s'", entryName_.c_str()); - entryCompression_ = PSARCompression::Unknown; - } else { - entryCompression_ = DetectCompression(block2_.data(), decoded); - if (entryCompression_ == PSARCompression::Zlib) { - entryData_.resize(expandedSize); - uLongf destLen = expandedSize; - const int zResult = uncompress(entryData_.data(), &destLen, block2_.data(), decoded); - if (zResult != Z_OK || destLen != expandedSize) { - WARN_LOG(Log::Loader, "PSAR: inflate failed for '%s' (%d)", entryName_.c_str(), zResult); - entryData_.clear(); - } - } - // Anything else we leave empty - the caller reports it through the stats. - } + contentPos_ = pos_; + contentChunkSize_ = chunkSize; + contentExpandedSize_ = expandedSize; + contentsDecoded_ = false; } pos_ += chunkSize; return 1; } +void PSARReader::EnsureContents() { + if (contentsDecoded_) { + return; + } + contentsDecoded_ = true; + + const int decoded = DecodeBlock(contentPos_, contentChunkSize_, block2_); + if (decoded <= 0) { + WARN_LOG(Log::Loader, "PSAR: couldn't decrypt the contents of '%s'", entryName_.c_str()); + entryCompression_ = PSARCompression::Unknown; + return; + } + + entryCompression_ = DetectCompression(block2_.data(), decoded); + if (entryCompression_ == PSARCompression::Zlib) { + entryData_.resize(contentExpandedSize_); + uLongf destLen = contentExpandedSize_; + const int zResult = uncompress(entryData_.data(), &destLen, block2_.data(), decoded); + if (zResult != Z_OK || destLen != contentExpandedSize_) { + WARN_LOG(Log::Loader, "PSAR: inflate failed for '%s' (%d)", entryName_.c_str(), zResult); + entryData_.clear(); + } + } + // Anything else we leave empty - the caller reports it through the stats. +} + bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error) { PSARUnpackStats localStats; if (!stats) { @@ -732,7 +758,6 @@ bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PS } stats->entries++; - stats->compressionCounts[(int)reader.entryCompression()]++; if (options.progress && reader.limit() > 0) { options.progress(std::min(1.0f, (float)reader.position() / (float)reader.limit())); @@ -835,6 +860,10 @@ bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PS continue; } + // Past the filter, so this one's contents are worth decrypting. The compression counts + // only cover the entries we got this far with, not everything in the archive. + stats->compressionCounts[(int)reader.entryCompression()]++; + if (reader.entryData().empty()) { ERROR_LOG(Log::Loader, "PSAR: no usable contents for '%s' (%s)", reader.entryName().c_str(), PSARCompressionToString(reader.entryCompression())); diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 3a78043746..7ad1641799 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -630,6 +630,9 @@ int main(int argc, const char* argv[]) { PSARUnpackOptions unpackOptions; unpackOptions.verbose = testOptions.verbose; + if (cmdLineOptions.unpackUpdaterFilter.has_value()) { + unpackOptions.prefixFilter = cmdLineOptions.unpackUpdaterFilter.value(); + } if (cmdLineOptions.unpackUpdaterModel.has_value() && !PSPModelGenerationFromString(cmdLineOptions.unpackUpdaterModel.value(), &unpackOptions.model)) { fprintf(stderr, "Unknown PSP model '%s' - expected 01g..12g or any\n", cmdLineOptions.unpackUpdaterModel.value().c_str()); @@ -641,9 +644,9 @@ int main(int argc, const char* argv[]) { if (!ok) { fprintf(stderr, "Unpacking failed: %s\n", unpackError.c_str()); } - printf("Firmware %s (model %s): %d entries, %d files written, %d directories, %d unresolved names, %d for other models, %d failed\n", + printf("Firmware %s (model %s): %d entries, %d files written, %d directories, %d skipped by filter, %d unresolved names, %d for other models, %d failed\n", stats.firmwareVersion.c_str(), PSPModelGenerationToString(unpackOptions.model), stats.entries, stats.written, - stats.directories, stats.unnamed, stats.otherModel, stats.failed); + stats.directories, stats.skippedByFilter, stats.unnamed, stats.otherModel, stats.failed); printf("Compression: none=%d zlib=%d KL4E=%d KL3E=%d LZR=%d unknown=%d\n", stats.compressionCounts[(int)PSARCompression::None], stats.compressionCounts[(int)PSARCompression::Zlib],