diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 1e77dd9bcd..de3e4d83e1 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -210,6 +210,7 @@ static const CommandLineParam g_autoParams[] = { {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, {POFF(unpackUpdater), CmdParamType::String, "unpack-updater", '\0', "Unpack the firmware in an updater EBOOT.PBP into DIR and exit", CmdLineMode::Headless}, {POFF(unpackUpdaterModel), CmdParamType::String, "unpack-updater-model", '\0', "PSP model to unpack for (01g..12g, default any)", CmdLineMode::Headless}, + {POFF(unpackUpdaterFilter), CmdParamType::String, "unpack-updater-filter", '\0', "Only unpack entries under this path, e.g. flash0:/font/", CmdLineMode::Headless}, {POFF(odsLog), CmdParamType::Bool, "odslog", 'o', "Also log through OutputDebugString (Windows)", CmdLineMode::Headless}, {POFF(generateInterpreterDispatch), CmdParamType::Bool, "generate-interpreter-dispatch", '\0', "Generate C++ interpreter dispatch code (ExecInstruction) to stdout and exit", CmdLineMode::Headless}, {POFF(resolutionScale), CmdParamType::Int, "resolution-scale", '\0', "Set the resolution scale factor"}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index 01331301a4..d8beec6bf9 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -76,6 +76,7 @@ struct CommandLineOptions { // Headless: which PSP model the unpacker resolves names against - "01g".."12g", or "any" // (the default) to take whatever file list names each file first. std::optional unpackUpdaterModel; + std::optional unpackUpdaterFilter; std::optional memReadAction; std::optional memWriteAction; diff --git a/Core/ELF/ParamSFO.cpp b/Core/ELF/ParamSFO.cpp index 24a910f8a9..74191a6816 100644 --- a/Core/ELF/ParamSFO.cpp +++ b/Core/ELF/ParamSFO.cpp @@ -119,6 +119,18 @@ std::string ParamSFOData::GetDiscID() { return discID; } +int ParamSFOData::VersionToInt(std::string_view version) { + int major = 0, minor = 0; + if (sscanf(std::string(version).c_str(), "%d.%d", &major, &minor) != 2) { + return 0; + } + return major * 100 + minor; +} + +int ParamSFOData::GetSystemVersion() const { + return VersionToInt(GetValueString("PSP_SYSTEM_VER")); +} + // I'm so sorry Ced but this is highly endian unsafe :( bool ParamSFOData::ReadSFO(const u8 *paramsfo, size_t size) { if (size < sizeof(Header)) diff --git a/Core/ELF/ParamSFO.h b/Core/ELF/ParamSFO.h index fd3e6e8823..060a61261a 100644 --- a/Core/ELF/ParamSFO.h +++ b/Core/ELF/ParamSFO.h @@ -42,6 +42,14 @@ public: std::string GetDiscID(); + // PSP_SYSTEM_VER, the firmware a disc says it needs, as major * 100 + minor - "2.60" gives + // 260, which sorts the way you'd want. Zero if the key is missing or unparseable; homebrew + // often leaves it out, but retail discs practically always set it. + int GetSystemVersion() const; + + // The same conversion on its own, for versions that come from somewhere other than this SFO. + static int VersionToInt(std::string_view version); + // This allocates a buffer (*paramsfo) using new[], whose size is zero-filled up to a multiple of 16 bytes. // This is required for SavedataParam::BuildHash. void WriteSFO(u8 **paramsfo, size_t *size) const; diff --git a/Core/FileSystems/ISOFileSystem.cpp b/Core/FileSystems/ISOFileSystem.cpp index e3a6ad68b4..b01837d058 100644 --- a/Core/FileSystems/ISOFileSystem.cpp +++ b/Core/FileSystems/ISOFileSystem.cpp @@ -349,6 +349,11 @@ const ISOFileSystem::TreeEntry *ISOFileSystem::GetFromPath(std::string_view path if (pathLength <= pathIndex) return treeroot; + if (!treeroot) { + // The constructor gave up - no ISO9660 volume descriptor, or it wouldn't read. + return nullptr; + } + TreeEntry *entry = treeroot; while (true) { if (!entry->valid) { diff --git a/Core/HLE/sceFont.cpp b/Core/HLE/sceFont.cpp index ade70d0f64..c6c89fdc0c 100644 --- a/Core/HLE/sceFont.cpp +++ b/Core/HLE/sceFont.cpp @@ -8,6 +8,7 @@ #include "Common/Serialize/Serializer.h" #include "Common/Serialize/SerializeFuncs.h" #include "Common/Serialize/SerializeMap.h" +#include "Common/StringUtils.h" #include "Core/HLE/HLE.h" #include "Core/HLE/ErrorCodes.h" #include "Core/HLE/FunctionWrappers.h" @@ -21,7 +22,12 @@ #include "Core/Reporting.h" #include "Core/Core.h" #include "Core/System.h" +#include "Core/ELF/ParamSFO.h" #include "Core/Font/PGF.h" +#include "Core/Util/PathUtil.h" +#include "Core/Util/PSARUnpack.h" +#include "Common/Data/Text/I18n.h" +#include "Common/System/OSD.h" constexpr int MAX_FONT_REFS = 4; @@ -81,8 +87,17 @@ struct FontRegistryEntry { u32 fontFileSize; u32 stingySize; // for the FONT_OPEN_INTERNAL_STINGY mode, from pspautotests. bool ignoreIfMissing; + // The earliest firmware known to ship this font, as major * 100 + minor, or 0 if every + // firmware has it. A game can't want a font that didn't exist when it was made, so this is + // what keeps us from forever chasing a file its own disc could never provide. + int minFirmware; }; +// The minFirmware values come from unpacking flash0:/font out of the updater on every disc in a +// large library, covering firmware 1.50 through 6.60. jpn0 and ltn0..ltn15 are in all of them; +// kr0.pgf is the only one that ever appeared later, absent in 1.50 and present from 1.52 on. +// (The updaters carry more than we register: shadow.pgf in 2.00-2.60, gb3s1518.bwfon from 2.60, +// arib.pgf and imagefont.bin from 3.71.) static const FontRegistryEntry fontRegistry[] = { // This was added for Chinese translations and is not normally loaded on a PSP. { 0x288, 0x288, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SANS_SERIF, FONT_STYLE_DB, 0, FONT_LANGUAGE_CHINESE, 0, 1, "zh_gb.pgf", "FTT-NewRodin Pro DB", 0, 0, 1581700, 145844, true }, @@ -103,7 +118,7 @@ static const FontRegistryEntry fontRegistry[] = { { 0x1c0, 0x1c0, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SERIF, FONT_STYLE_BOLD, 0, FONT_LANGUAGE_LATIN, 0, 1, "ltn13.pgf", "FTT-Matisse Pro Latin", 0, 0, 41772, 16436 }, { 0x1c0, 0x1c0, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SANS_SERIF, FONT_STYLE_BOLD_ITALIC, 0, FONT_LANGUAGE_LATIN, 0, 1, "ltn14.pgf", "FTT-NewRodin Pro Latin", 0, 0, 45184, 16272 }, { 0x1c0, 0x1c0, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SERIF, FONT_STYLE_BOLD_ITALIC, 0, FONT_LANGUAGE_LATIN, 0, 1, "ltn15.pgf", "FTT-Matisse Pro Latin", 0, 0, 43044, 16704 }, - { 0x288, 0x288, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SANS_SERIF, FONT_STYLE_REGULAR, 0, FONT_LANGUAGE_KOREAN, 0, 3, "kr0.pgf", "AsiaNHH(512Johab)", 0, 0, 394192, 51856 }, + { 0x288, 0x288, 0x2000, 0x2000, 0, 0, FONT_FAMILY_SANS_SERIF, FONT_STYLE_REGULAR, 0, FONT_LANGUAGE_KOREAN, 0, 3, "kr0.pgf", "AsiaNHH(512Johab)", 0, 0, 394192, 51856, false, 152 }, }; static const float pointDPI = 72.f; @@ -872,6 +887,98 @@ static LoadedFont *GetLoadedFont(u32 handle, bool allowClosed) { } } +// The real PSP fonts live in flash0, which we only have if the user installed a firmware. +static const char *const g_nandFontPath = "flash0:/font/"; + +// The firmware this game was built against, which bounds the fonts it can ask for. Practically +// all discs declare it in PARAM.SFO, and the version of the updater they carry stands in for the +// rest. Zero when we can't tell, which asks for nothing beyond the fonts every firmware has. +static int GameFirmwareVersion() { + int version = g_paramSFO.GetSystemVersion(); + if (version == 0) { + version = ParamSFOData::VersionToInt(ReadMountedDiscUpdaterVersion()); + } + return version; +} + +// Whether NAND has the fonts this game could actually ask for. Not every font we know of - an +// older game's firmware never had the later ones, so requiring those could never be satisfied and +// we'd unpack the same updater on every launch. +// +// One directory listing rather than a stat per font, since on Android's scoped storage the +// individual checks are slow. +static bool NandFontsComplete() { + const int firmware = GameFirmwareVersion(); + + std::string_view fontDir(g_nandFontPath); + fontDir.remove_suffix(1); // GetDirListing doesn't want the trailing slash. + + bool dirExists = false; + const std::vector listing = pspFileSystem.GetDirListing(fontDir, &dirExists); + if (!dirExists) { + return false; + } + + for (const FontRegistryEntry &entry : fontRegistry) { + if (entry.ignoreIfMissing) { + // zh_gb.pgf, which we added for Chinese translations. No firmware ships it, so + // waiting for one to show up would mean never being satisfied. + continue; + } + if (entry.minFirmware > firmware) { + // Didn't exist yet when this game was made. + continue; + } + bool found = false; + for (const PSPFileInfo &file : listing) { + if (equalsNoCase(file.name, entry.fileName)) { + found = true; + break; + } + } + if (!found) { + DEBUG_LOG(Log::sceFont, "%s missing from NAND (game wants firmware %d)", entry.fileName, firmware); + return false; + } + } + return true; +} + +// Most UMDs carry a firmware updater, so if we're missing real fonts we can pull them out of +// whatever game is running rather than making the user find an updater themselves. Our bundled +// substitutes are a good deal worse - some homebrew even trips over them. +static bool InstallFontsFromDiscUpdater() { + if (!MountedDiscHasUpdater()) { + return false; + } + + INFO_LOG(Log::sceFont, "No fonts in NAND, trying the firmware updater on the disc (%s)", + ReadMountedDiscUpdaterVersion().c_str()); + + PSARUnpackOptions options; + options.model = EmulatedModelGeneration(); + // Just the fonts - the rest of a firmware is none of our business here. + options.prefixFilter = g_nandFontPath; + + PSARUnpackStats stats; + std::string error; + if (!UnpackUpdaterFromMountedDisc(GetSysDirectory(DIRECTORY_NAND), options, &stats, &error)) { + WARN_LOG(Log::sceFont, "Couldn't unpack fonts from the disc's updater: %s", error.c_str()); + return false; + } + if (stats.written == 0) { + // The updater has no font list for the model we're emulating. + WARN_LOG(Log::sceFont, "The disc's updater had no fonts for this PSP model"); + return false; + } + + INFO_LOG(Log::sceFont, "Installed %d font files from the disc's %s updater", stats.written, + stats.firmwareVersion.c_str()); + auto sy = GetI18NCategory(I18NCat::SYSTEM); + g_OSD.Show(OSDType::MESSAGE_SUCCESS, sy->T("Installed fonts"), stats.firmwareVersion, 3.0f); + return true; +} + static void __LoadInternalFonts() { if (internalFonts.size()) { // Fonts already loaded. @@ -883,6 +990,8 @@ static void __LoadInternalFonts() { const bool checkClassicOverrides = pspFileSystem.GetFileInfo(fontOverridePath).exists; if (checkClassicOverrides) { WARN_LOG(Log::sceFont, "Classic font overrides active, ignoring NAND: %s", fontOverridePath.c_str()); + } else if (!NandFontsComplete()) { + InstallFontsFromDiscUpdater(); } if ((pspFileSystem.GetFileInfo("disc0:/PSP_GAME/USRDIR/zh_gb.pgf").exists) && (pspFileSystem.GetFileInfo("disc0:/PSP_GAME/USRDIR/oldfont.prx").exists)) { @@ -913,6 +1022,10 @@ static void __LoadInternalFonts() { // No game font, let's try classic override path. NOTE: This is not recommended - use flash0. fontFilename = fontOverridePath + entry.fileName; bufferRead = pspFileSystem.ReadEntireFile(fontFilename, buffer, true) >= 0; + } else if (!bufferRead) { + // The real fonts, from a firmware the user installed or that we took off a disc. + fontFilename = std::string(g_nandFontPath) + entry.fileName; + bufferRead = pspFileSystem.ReadEntireFile(fontFilename, buffer, true) >= 0; } if (!bufferRead) { diff --git a/Core/Util/PSARUnpack.cpp b/Core/Util/PSARUnpack.cpp index ca3b27299d..e7aff71097 100644 --- a/Core/Util/PSARUnpack.cpp +++ b/Core/Util/PSARUnpack.cpp @@ -26,6 +26,7 @@ #include "Common/File/Path.h" #include "Common/Log.h" #include "Common/StringUtils.h" +#include "Core/Config.h" #include "Core/ELF/ParamSFO.h" #include "Core/ELF/PBPReader.h" #include "Core/ELF/PrxDecrypter.h" @@ -334,6 +335,10 @@ const char *PSPModelGenerationToString(PSPModelGeneration generation) { } } +PSPModelGeneration EmulatedModelGeneration() { + return g_Config.iPSPModel == PSP_MODEL_FAT ? PSPModelGeneration::PSP_1000 : PSPModelGeneration::PSP_2000; +} + bool PSPModelGenerationFromString(std::string_view name, PSPModelGeneration *generation) { if (equalsNoCase(name, "any")) { *generation = PSPModelGeneration::Any; @@ -486,9 +491,12 @@ public: const std::string &firmwareVersion() const { return firmwareVersion_; } const std::string &entryName() const { return entryName_; } bool entryIsDirectory() const { return entryIsDirectory_; } - PSARCompression entryCompression() const { return entryCompression_; } + // Decrypting an entry's contents is by far the most expensive thing here, and an archive holds + // well over a thousand of them - so it doesn't happen until one of these two asks for it. An + // entry a filter rejects never gets decrypted at all. + PSARCompression entryCompression() { EnsureContents(); return entryCompression_; } // Empty for a directory, or for an entry we couldn't decompress. - const std::vector &entryData() const { return entryData_; } + const std::vector &entryData() { EnsureContents(); return entryData_; } // How far into the archive the next record starts, and where the records stop - i.e. progress. u32 position() const { return pos_; } size_t limit() const { return limit_; } @@ -496,6 +504,8 @@ public: private: // Decrypts one record into 'out'. Returns the decrypted size, or <= 0 on failure. int DecodeBlock(u32 offset, u32 cbIn, std::vector &out); + // Decodes the current entry's contents, if that hasn't happened yet. + void EnsureContents(); const u8 *psar_; size_t size_; @@ -514,6 +524,13 @@ private: PSARCompression entryCompression_ = PSARCompression::None; std::vector entryData_; + // Where the current entry's contents live, for decoding them later. Records are decrypted + // independently of each other, so it doesn't matter that we've moved past them by then. + bool contentsDecoded_ = true; + u32 contentPos_ = 0; + u32 contentChunkSize_ = 0; + u32 contentExpandedSize_ = 0; + std::vector block_; std::vector block2_; }; @@ -634,6 +651,7 @@ int PSARReader::NextEntry(std::string *error) { entryData_.clear(); entryIsDirectory_ = false; entryCompression_ = PSARCompression::None; + contentsDecoded_ = true; // Nothing to decode until we find out there's a payload. // Stop when what's left can't hold another whole record. Both archives I've looked at end with // a few bytes of padding that would otherwise be decoded as a truncated entry and reported as @@ -668,29 +686,42 @@ int PSARReader::NextEntry(std::string *error) { *error = StringFromFormat("Entry '%s' claims an unreasonable size (%u)", entryName_.c_str(), expandedSize); return -1; } else { - decoded = DecodeBlock(pos_, chunkSize, block2_); - if (decoded <= 0) { - WARN_LOG(Log::Loader, "PSAR: couldn't decrypt the contents of '%s'", entryName_.c_str()); - entryCompression_ = PSARCompression::Unknown; - } else { - entryCompression_ = DetectCompression(block2_.data(), decoded); - if (entryCompression_ == PSARCompression::Zlib) { - entryData_.resize(expandedSize); - uLongf destLen = expandedSize; - const int zResult = uncompress(entryData_.data(), &destLen, block2_.data(), decoded); - if (zResult != Z_OK || destLen != expandedSize) { - WARN_LOG(Log::Loader, "PSAR: inflate failed for '%s' (%d)", entryName_.c_str(), zResult); - entryData_.clear(); - } - } - // Anything else we leave empty - the caller reports it through the stats. - } + contentPos_ = pos_; + contentChunkSize_ = chunkSize; + contentExpandedSize_ = expandedSize; + contentsDecoded_ = false; } pos_ += chunkSize; return 1; } +void PSARReader::EnsureContents() { + if (contentsDecoded_) { + return; + } + contentsDecoded_ = true; + + const int decoded = DecodeBlock(contentPos_, contentChunkSize_, block2_); + if (decoded <= 0) { + WARN_LOG(Log::Loader, "PSAR: couldn't decrypt the contents of '%s'", entryName_.c_str()); + entryCompression_ = PSARCompression::Unknown; + return; + } + + entryCompression_ = DetectCompression(block2_.data(), decoded); + if (entryCompression_ == PSARCompression::Zlib) { + entryData_.resize(contentExpandedSize_); + uLongf destLen = contentExpandedSize_; + const int zResult = uncompress(entryData_.data(), &destLen, block2_.data(), decoded); + if (zResult != Z_OK || destLen != contentExpandedSize_) { + WARN_LOG(Log::Loader, "PSAR: inflate failed for '%s' (%d)", entryName_.c_str(), zResult); + entryData_.clear(); + } + } + // Anything else we leave empty - the caller reports it through the stats. +} + bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error) { PSARUnpackStats localStats; if (!stats) { @@ -727,7 +758,6 @@ bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PS } stats->entries++; - stats->compressionCounts[(int)reader.entryCompression()]++; if (options.progress && reader.limit() > 0) { options.progress(std::min(1.0f, (float)reader.position() / (float)reader.limit())); @@ -830,6 +860,10 @@ bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PS continue; } + // Past the filter, so this one's contents are worth decrypting. The compression counts + // only cover the entries we got this far with, not everything in the archive. + stats->compressionCounts[(int)reader.entryCompression()]++; + if (reader.entryData().empty()) { ERROR_LOG(Log::Loader, "PSAR: no usable contents for '%s' (%s)", reader.entryName().c_str(), PSARCompressionToString(reader.entryCompression())); diff --git a/Core/Util/PSARUnpack.h b/Core/Util/PSARUnpack.h index 620932c5d9..f3ef8a5d18 100644 --- a/Core/Util/PSARUnpack.h +++ b/Core/Util/PSARUnpack.h @@ -67,6 +67,10 @@ enum class PSPModelGeneration { }; const char *PSPModelGenerationToString(PSPModelGeneration generation); +// The model we're claiming to be. An updater carries one file list per hardware revision, and +// anything the chosen model's list doesn't name isn't part of its firmware, so unpacking this +// one keeps flash0 consistent with what the emulator reports to games. +PSPModelGeneration EmulatedModelGeneration(); // Accepts "01g".."12g", a bare number, or "any". Returns false if it's none of those. bool PSPModelGenerationFromString(std::string_view name, PSPModelGeneration *generation); diff --git a/UI/InstallUpdateScreen.cpp b/UI/InstallUpdateScreen.cpp index 5af068c453..abe8a4de2b 100644 --- a/UI/InstallUpdateScreen.cpp +++ b/UI/InstallUpdateScreen.cpp @@ -36,13 +36,6 @@ #include "UI/MiscViews.h" #include "UI/EmuScreen.h" -// An updater carries one file list per hardware revision, and anything the chosen model's list -// doesn't name isn't part of its firmware. Unpacking the model we claim to be keeps flash0 -// consistent with what the emulator reports to games. -static PSPModelGeneration EmulatedModelGeneration() { - return g_Config.iPSPModel == PSP_MODEL_FAT ? PSPModelGeneration::PSP_1000 : PSPModelGeneration::PSP_2000; -} - InstallUpdateScreen::InstallUpdateScreen(const Path &path, std::string_view title) : UISimpleBaseDialogScreen(Path(), SimpleDialogFlags::ContentsCanScroll), path_(path), title_(title) { destination_ = GetSysDirectory(DIRECTORY_NAND); diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 30d38546cf..e7e7ef2dc7 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -644,6 +644,9 @@ int main(int argc, const char* argv[]) { PSARUnpackOptions unpackOptions; unpackOptions.verbose = testOptions.verbose; + if (cmdLineOptions.unpackUpdaterFilter.has_value()) { + unpackOptions.prefixFilter = cmdLineOptions.unpackUpdaterFilter.value(); + } if (cmdLineOptions.unpackUpdaterModel.has_value() && !PSPModelGenerationFromString(cmdLineOptions.unpackUpdaterModel.value(), &unpackOptions.model)) { fprintf(stderr, "Unknown PSP model '%s' - expected 01g..12g or any\n", cmdLineOptions.unpackUpdaterModel.value().c_str()); @@ -655,9 +658,9 @@ int main(int argc, const char* argv[]) { if (!ok) { fprintf(stderr, "Unpacking failed: %s\n", unpackError.c_str()); } - printf("Firmware %s (model %s): %d entries, %d files written, %d directories, %d unresolved names, %d for other models, %d failed\n", + printf("Firmware %s (model %s): %d entries, %d files written, %d directories, %d skipped by filter, %d unresolved names, %d for other models, %d failed\n", stats.firmwareVersion.c_str(), PSPModelGenerationToString(unpackOptions.model), stats.entries, stats.written, - stats.directories, stats.unnamed, stats.otherModel, stats.failed); + stats.directories, stats.skippedByFilter, stats.unnamed, stats.otherModel, stats.failed); printf("Compression: none=%d zlib=%d KL4E=%d KL3E=%d LZR=%d unknown=%d\n", stats.compressionCounts[(int)PSARCompression::None], stats.compressionCounts[(int)PSARCompression::Zlib],