diff --git a/AGENTS.md b/AGENTS.md index 4969747446..29f4671eef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,6 +169,8 @@ codebase first - PPSSPP already has implementations of many formats (CSO, LZRC, handlers, PBP, SevenZip, etc.), possibly in several places. Reuse or extend an existing one instead of writing a new one (e.g. there is an LZRC decompressor in Core/FileSystems/tlzrc.cpp). +For string sanitation, we already have SanitizeString in StringUtils.cpp - add new modes if needed. + ## Headless and unittest builds We have additional PPSSPPHeadless and unit test builds (/headless and /unittest), that have their own separate @@ -537,6 +539,12 @@ Concretely, as measured against the headless build (2026-08-16), per CPU backend | `memory.breakpoint.*` (memchecks) | works | **only for constant addresses** - see below | | `cpu.regBreakpoint.*` | works | never trips (as documented) | +## Commit message style + +Keep commit messages focused, not overly long (although sometimes it's motivated if a single commit +is super complex). Do not report things like 100/100 tests passed - that's a given, if tests break +you aren't supposed to make a commit. + ## Quick rebuild on Linux You don't need to do ./b.sh --debug to verify every single little change, instead use this shortcut: diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index aadb324560..9307aacead 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -705,6 +705,8 @@ add_library(Core STATIC Util/BlockAllocator.h Util/PPGeDraw.cpp Util/PPGeDraw.h + Util/PSARUnpack.cpp + Util/PSARUnpack.h Util/RecentFiles.cpp Util/RecentFiles.h ${CMAKE_SOURCE_DIR}/ext/disarm.cpp diff --git a/Core/CmdLine.cpp b/Core/CmdLine.cpp index 45186ce934..fcb8519a9c 100644 --- a/Core/CmdLine.cpp +++ b/Core/CmdLine.cpp @@ -192,6 +192,8 @@ static const CommandLineParam g_autoParams[] = { {POFF(timeout), CmdParamType::Double, "timeout", '\0', "Set the timeout value", CmdLineMode::Headless}, {POFF(maxScreenshotError), CmdParamType::Double, "max-mse", '\0', "Maximum allowed MSE error for screenshot comparison", CmdLineMode::Headless}, {POFF(mountIso), CmdParamType::String, "mount", 'm', "Mount ISO/CSO on umd1:", CmdLineMode::Headless}, + {POFF(unpackUpdater), CmdParamType::String, "unpack-updater", '\0', "Unpack the firmware in an updater EBOOT.PBP into DIR and exit", CmdLineMode::Headless}, + {POFF(unpackUpdaterModel), CmdParamType::String, "unpack-updater-model", '\0', "PSP model to unpack for (01g..12g, default any)", CmdLineMode::Headless}, {POFF(odsLog), CmdParamType::Bool, "odslog", 'o', "Also log through OutputDebugString (Windows)", CmdLineMode::Headless}, {POFF(generateInterpreterDispatch), CmdParamType::Bool, "generate-interpreter-dispatch", '\0', "Generate C++ interpreter dispatch code (ExecInstruction) to stdout and exit", CmdLineMode::Headless}, {POFF(resolutionScale), CmdParamType::Int, "resolution-scale", '\0', "Set the resolution scale factor"}, diff --git a/Core/CmdLine.h b/Core/CmdLine.h index f5db9f1001..02ae8c9105 100644 --- a/Core/CmdLine.h +++ b/Core/CmdLine.h @@ -64,6 +64,15 @@ struct CommandLineOptions { std::optional memStick; std::optional stateToLoad; + // Headless: unpack the firmware inside an official updater EBOOT.PBP (given as the boot + // filename) into this directory, then exit without booting anything. + // See Core/Util/PSARUnpack.h - the API can also filter to a subfolder, which the command line + // deliberately doesn't expose since the in-app use is specifically flash0:/font. + std::optional unpackUpdater; + // Headless: which PSP model the unpacker resolves names against - "01g".."12g", or "any" + // (the default) to take whatever file list names each file first. + std::optional unpackUpdaterModel; + std::optional memReadAction; std::optional memWriteAction; std::optional breakAction; diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index c9694ce3fb..5b8fa83a39 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -917,6 +917,7 @@ + MaxSpeed true @@ -1280,6 +1281,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 94946d503c..f8c593bd94 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -363,6 +363,9 @@ Util + + Util + HLE\Libraries @@ -1674,6 +1677,9 @@ Util + + Util + HLE\Libraries diff --git a/Core/ELF/PrxDecrypter.cpp b/Core/ELF/PrxDecrypter.cpp index 27ae004e44..795dab1865 100644 --- a/Core/ELF/PrxDecrypter.cpp +++ b/Core/ELF/PrxDecrypter.cpp @@ -284,12 +284,24 @@ static const u32 g_key_INDEXDAT1xx[] = { 0xA34D8C80, 0x962B235D, 0x3E420548, 0x09CF9FFE, 0xD4883F5C, 0xD90E9CB5, 0x00AEF4E9, 0xF0886DE9, 0x62A58A5B, 0x52A55546, 0x971941B5, 0xF5B79FAC}; +// The blocks inside an official updater's DATA.PSAR carry this tag. See Core/Util/PSARUnpack.cpp. +static const u32 g_keyUPDATER_PSAR[] = { + 0x77B757DE, 0xEE62DD17, 0x5D03787B, 0x59CA8644, 0x93F68D20, 0x21819328, + 0x86A74E71, 0x1B2482CA, 0x5F74AE58, 0x568D016C, 0x9A4D8832, 0x2EA24372, + 0x820CF484, 0xFCFC06B9, 0x8A5BFB6A, 0xBF9F9CD7, 0x15850D01, 0x39ED5FBA, + 0x4CC38393, 0xED3ADEAF, 0x1AA768BF, 0x89BD8A77, 0x46564165, 0x7333DBD9, + 0x62E86C81, 0x03299B96, 0x73AFAE5A, 0x40A05320, 0x10664BE8, 0xE5B76A99, + 0x29E0DD70, 0xEA602428, 0x2042AE30, 0x946F8D32, 0xA29E5F71, 0x7C0C7FD5}; + struct TAG_INFO { u32 tag; // 4 byte value at offset 0xD0 in the PRX file const u32 *key; // "step1_result" use for XOR step u8 code; u8 codeExtra; + // Most keys here are stored already scrambled, so the kirk7 pass the hardware would do is + // skipped. Set this for a key that's stored raw and needs that pass applied first. + bool scrambleKey; }; static const TAG_INFO g_tagInfo[] = @@ -310,7 +322,8 @@ static const TAG_INFO g_tagInfo[] = { 0x862648D1, g_keyMEIMG260, 0x52, 0x52 }, { 0x207BBF2F, g_keyUNK1, 0x5A, 0x5A }, { 0x09000000, g_key_GAMESHARE1xx, 0x4C }, - { 0xBB67C59F, g_key_GAMESHARE2xx, 0x5E, 0x5E } + { 0xBB67C59F, g_key_GAMESHARE2xx, 0x5E, 0x5E }, + { 0x0E000000, g_keyUPDATER_PSAR, 0x51, 0x00, true } }; static const TAG_INFO *GetTagInfo(u32 tagFind) @@ -726,6 +739,11 @@ static int pspDecryptType0(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz // normally this would be a kirk7 op, but we have the seed pre-decrypted std::array xorbuf; memcpy(xorbuf.data(), reinterpret_cast(pti->key), xorbuf.size()); + if (pti->scrambleKey) + { + // ...except for keys we have in their raw form, which need that pass after all. + kirk7(xorbuf.data(), xorbuf.data(), xorbuf.size(), pti->code); + } // construct the header format for a type 0 prx PRXType0 type0(inbuf); diff --git a/Core/Loaders.cpp b/Core/Loaders.cpp index cfb997dd52..0d5b138598 100644 --- a/Core/Loaders.cpp +++ b/Core/Loaders.cpp @@ -221,7 +221,10 @@ IdentifiedFileType Identify_File(FileLoader *fileLoader, std::string *errorStrin return IdentifiedFileType::ARCHIVE_7Z; } - if (id == 'FLE\x7F') { + // "~PSP" is an encrypted PRX. The module loader decrypts those on the way in, so as far as + // identification goes it's the same thing as a plain ELF - which is how the modules in a + // firmware unpacked straight from an updater arrive (see Core/Util/PSARUnpack.cpp). + if (id == 'FLE\x7F' || id == 'PSP~') { Path filename = fileLoader->GetPath(); // There are a few elfs misnamed as pbp (like Trig Wars), accept that. Also accept extension-less paths. if (extension == ".plf" || strstr(filename.GetFilename().c_str(), "BOOT.BIN") || diff --git a/Core/Util/PSARUnpack.cpp b/Core/Util/PSARUnpack.cpp new file mode 100644 index 0000000000..131859d658 --- /dev/null +++ b/Core/Util/PSARUnpack.cpp @@ -0,0 +1,1053 @@ +// Copyright (c) 2026- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#include +#include +#include +#include + +#include "zlib.h" + +#include "Common/File/FileUtil.h" +#include "Common/File/Path.h" +#include "Common/Log.h" +#include "Common/StringUtils.h" +#include "Core/ELF/ParamSFO.h" +#include "Core/ELF/PBPReader.h" +#include "Core/ELF/PrxDecrypter.h" +#include "Core/FileSystems/BlockDevices.h" +#include "Core/FileSystems/ISOFileSystem.h" +#include "Core/FileSystems/MetaFileSystem.h" +#include "Core/Loaders.h" +#include "Core/System.h" +#include "Core/Util/PSARUnpack.h" + +extern "C" { +#include "ext/libkirk/kirk_engine.h" +} + +// A PSAR record is [header][entry], where the header is 0x150 bytes of PRX-style encryption +// metadata and the entry is 0x110 bytes describing one file. Pre-decrypted archives (rare, and +// only produced by other tools) have no header at all. +static const u32 PSAR_MAGIC = 0x52415350; // "PSAR" +static const u32 PSAR_DECRYPTED_MARKER = 0x2C333333; +static const u32 PSAR_ENTRY_SIZE = 0x110; +static const u32 PSAR_HEADER_SIZE = 0x150; +// The KIRK CMD7 key the "demangle" step below uses. Same for every firmware. +static const int PSAR_DEMANGLE_KEYSEED = 0x55; +// Sanity bound on the sizes we take from the archive, so a corrupt one can't ask for a huge +// allocation. No file inside an updater comes close. +static const u32 PSAR_MAX_ENTRY_BYTES = 64 * 1024 * 1024; +// Same idea for the archive itself. The biggest official updater is a few tens of MB. +static const s64 PSAR_MAX_DISC_FILE_BYTES = 256 * 1024 * 1024; + +const char *PSARCompressionToString(PSARCompression c) { + switch (c) { + case PSARCompression::None: return "none"; + case PSARCompression::Zlib: return "zlib"; + case PSARCompression::KL4E: return "KL4E"; + case PSARCompression::KL3E: return "KL3E"; + case PSARCompression::LZR: return "LZR"; + default: return "unknown"; + } +} + +static u32 ReadU32(const u8 *p) { + u32 value; + memcpy(&value, p, sizeof(value)); + return value; +} + +static u16 ReadU16(const u8 *p) { + u16 value; + memcpy(&value, p, sizeof(value)); + return value; +} + +static PSARCompression DetectCompression(const u8 *data, size_t size) { + if (size < 4) { + return PSARCompression::Unknown; + } + // Standard zlib stream: CM=8, CINFO=7, then the check byte. + if (data[0] == 0x78 && data[1] == 0x9C) { + return PSARCompression::Zlib; + } + if (!memcmp(data, "KL4E", 4)) { + return PSARCompression::KL4E; + } + if (!memcmp(data, "KL3E", 4)) { + return PSARCompression::KL3E; + } + if (!memcmp(data, "2RLZ", 4)) { + return PSARCompression::LZR; + } + return PSARCompression::Unknown; +} + +// Turns a name out of the archive into a path relative to the output directory: +// "flash0:/font/ltn0.pgf" becomes "flash0/font/ltn0.pgf". Returns false for anything that could +// escape the output directory - these names come from a file we didn't write. +static bool RelativePathFromEntryName(std::string_view name, std::string *out) { + std::string path(name); + if (path.empty()) { + return false; + } + for (size_t i = 0; i < path.size(); i++) { + if (path[i] == ':' || path[i] == '\\') { + path[i] = '/'; + } + } + // Collapse the "flash0:/" -> "flash0//" the above just produced, and any other doubles. + std::string cleaned; + cleaned.reserve(path.size()); + for (size_t i = 0; i < path.size(); i++) { + if (path[i] == '/' && (cleaned.empty() || cleaned.back() == '/')) { + continue; + } + cleaned.push_back(path[i]); + } + if (cleaned.empty()) { + return false; + } + // No traversal, no absolute paths, no drive letters - all of which would land outside outputDir. + if (cleaned.find("..") != std::string::npos) { + return false; + } + *out = cleaned; + return true; +} + +// An entry name is either a real path or a short token that only the archive's own file lists can +// resolve. 1.x and 2.x archives skip the lists and name entries outright: "flash0:/...", but also +// "flash1:/..." and "ipl:/psp_nandipl.bin", so rather than keep a list of devices, take the ":/" +// shape as the tell. 3.x's grouped short names ("com:00123") have a colon but no slash after it, +// which is what keeps them out. +static bool EntryNameIsRealPath(std::string_view name) { + return name.find(":/") != std::string_view::npos; +} + +// 6.x file lists write "flash0:/font/x.pgf", 3.x writes "flash0/font/x.pgf" for the same thing. +// Settle on the first, so callers only have to know one form - a prefix filter of "flash0:/font/" +// has to work whatever the firmware. +static std::string CanonicalFlashPath(std::string_view name) { + if (name.size() > 6 && startsWithNoCase(name, "flash") && name[5] >= '0' && name[5] <= '9' && name[6] == '/') { + return std::string(name.substr(0, 6)) + ":" + std::string(name.substr(6)); + } + return std::string(name); +} + +// The name tables inside the archive are DES-CBC encrypted, with a PRX blob underneath. Nothing +// here is PSP-specific - it's plain DES with the standard tables - so it lives at file scope +// rather than pretending to be part of the PSAR format. +static const u8 kIP[64] = { + 58,50,42,34,26,18,10,2, 60,52,44,36,28,20,12,4, 62,54,46,38,30,22,14,6, 64,56,48,40,32,24,16,8, + 57,49,41,33,25,17, 9,1, 59,51,43,35,27,19,11,3, 61,53,45,37,29,21,13,5, 63,55,47,39,31,23,15,7 }; +static const u8 kFP[64] = { + 40,8,48,16,56,24,64,32, 39,7,47,15,55,23,63,31, 38,6,46,14,54,22,62,30, 37,5,45,13,53,21,61,29, + 36,4,44,12,52,20,60,28, 35,3,43,11,51,19,59,27, 34,2,42,10,50,18,58,26, 33,1,41, 9,49,17,57,25 }; +static const u8 kE[48] = { + 32,1,2,3,4,5, 4,5,6,7,8,9, 8,9,10,11,12,13, 12,13,14,15,16,17, + 16,17,18,19,20,21, 20,21,22,23,24,25, 24,25,26,27,28,29, 28,29,30,31,32,1 }; +static const u8 kP[32] = { + 16,7,20,21, 29,12,28,17, 1,15,23,26, 5,18,31,10, 2,8,24,14, 32,27,3,9, 19,13,30,6, 22,11,4,25 }; +static const u8 kPC1[56] = { + 57,49,41,33,25,17, 9, 1,58,50,42,34,26,18, 10, 2,59,51,43,35,27, 19,11, 3,60,52,44,36, + 63,55,47,39,31,23,15, 7,62,54,46,38,30,22, 14, 6,61,53,45,37,29, 21,13, 5,28,20,12, 4 }; +static const u8 kPC2[48] = { + 14,17,11,24, 1, 5, 3,28,15, 6,21,10, 23,19,12, 4,26, 8, 16, 7,27,20,13, 2, + 41,52,31,37,47,55, 30,40,51,45,33,48, 44,49,39,56,34,53, 46,42,50,36,29,32 }; +static const u8 kShifts[16] = { 1,1,2,2,2,2,2,2,1,2,2,2,2,2,2,1 }; +static const u8 kSBox[8][64] = { + { 14,4,13,1,2,15,11,8,3,10,6,12,5,9,0,7, 0,15,7,4,14,2,13,1,10,6,12,11,9,5,3,8, + 4,1,14,8,13,6,2,11,15,12,9,7,3,10,5,0, 15,12,8,2,4,9,1,7,5,11,3,14,10,0,6,13 }, + { 15,1,8,14,6,11,3,4,9,7,2,13,12,0,5,10, 3,13,4,7,15,2,8,14,12,0,1,10,6,9,11,5, + 0,14,7,11,10,4,13,1,5,8,12,6,9,3,2,15, 13,8,10,1,3,15,4,2,11,6,7,12,0,5,14,9 }, + { 10,0,9,14,6,3,15,5,1,13,12,7,11,4,2,8, 13,7,0,9,3,4,6,10,2,8,5,14,12,11,15,1, + 13,6,4,9,8,15,3,0,11,1,2,12,5,10,14,7, 1,10,13,0,6,9,8,7,4,15,14,3,11,5,2,12 }, + { 7,13,14,3,0,6,9,10,1,2,8,5,11,12,4,15, 13,8,11,5,6,15,0,3,4,7,2,12,1,10,14,9, + 10,6,9,0,12,11,7,13,15,1,3,14,5,2,8,4, 3,15,0,6,10,1,13,8,9,4,5,11,12,7,2,14 }, + { 2,12,4,1,7,10,11,6,8,5,3,15,13,0,14,9, 14,11,2,12,4,7,13,1,5,0,15,10,3,9,8,6, + 4,2,1,11,10,13,7,8,15,9,12,5,6,3,0,14, 11,8,12,7,1,14,2,13,6,15,0,9,10,4,5,3 }, + { 12,1,10,15,9,2,6,8,0,13,3,4,14,7,5,11, 10,15,4,2,7,12,9,5,6,1,13,14,0,11,3,8, + 9,14,15,5,2,8,12,3,7,0,4,10,1,13,11,6, 4,3,2,12,9,5,15,10,11,14,1,7,6,0,8,13 }, + { 4,11,2,14,15,0,8,13,3,12,9,7,5,10,6,1, 13,0,11,7,4,9,1,10,14,3,5,12,2,15,8,6, + 1,4,11,13,12,3,7,14,10,15,6,8,0,5,9,2, 6,11,13,8,1,4,10,7,9,5,0,15,14,2,3,12 }, + { 13,2,8,4,6,15,11,1,10,9,3,14,5,0,12,7, 1,15,13,8,10,3,7,4,12,5,6,11,0,14,9,2, + 7,11,4,1,9,12,14,2,0,6,10,13,15,3,5,8, 2,1,14,7,4,10,8,13,15,12,9,0,3,5,6,11 } }; + +// One bit per byte throughout - the tables are a few tens of KB in total, so this doesn't need +// to be a fast DES, and the permutations are much easier to read this way. +static void BytesToBits(const u8 *bytes, int byteCount, u8 *bits) { + for (int i = 0; i < byteCount; i++) { + for (int bit = 0; bit < 8; bit++) { + bits[i * 8 + bit] = (bytes[i] >> (7 - bit)) & 1; + } + } +} + +static void BitsToBytes(const u8 *bits, int byteCount, u8 *bytes) { + for (int i = 0; i < byteCount; i++) { + u8 value = 0; + for (int bit = 0; bit < 8; bit++) { + value = (u8)((value << 1) | bits[i * 8 + bit]); + } + bytes[i] = value; + } +} + +// The tables are 1-based, as they're always written. +static void Permute(const u8 *in, u8 *out, const u8 *table, int count) { + for (int i = 0; i < count; i++) { + out[i] = in[table[i] - 1]; + } +} + +class DESContext { +public: + explicit DESContext(const u8 key[8]) { + u8 keyBits[64]; + BytesToBits(key, 8, keyBits); + u8 cd[56]; + Permute(keyBits, cd, kPC1, 56); + u8 *c = cd; + u8 *d = cd + 28; + for (int round = 0; round < 16; round++) { + for (int shift = 0; shift < kShifts[round]; shift++) { + const u8 cFirst = c[0]; + const u8 dFirst = d[0]; + memmove(c, c + 1, 27); + memmove(d, d + 1, 27); + c[27] = cFirst; + d[27] = dFirst; + } + Permute(cd, subkeys_[round], kPC2, 48); + } + } + + void DecryptBlock(const u8 in[8], u8 out[8]) const { + u8 bits[64]; + u8 permuted[64]; + BytesToBits(in, 8, bits); + Permute(bits, permuted, kIP, 64); + + u8 left[32], right[32]; + memcpy(left, permuted, 32); + memcpy(right, permuted + 32, 32); + + // Subkeys in reverse order is what makes this a decrypt. + for (int round = 15; round >= 0; round--) { + u8 expanded[48]; + Permute(right, expanded, kE, 48); + for (int i = 0; i < 48; i++) { + expanded[i] ^= subkeys_[round][i]; + } + + u8 substituted[32]; + for (int box = 0; box < 8; box++) { + const u8 *six = expanded + box * 6; + const int row = (six[0] << 1) | six[5]; + const int column = (six[1] << 3) | (six[2] << 2) | (six[3] << 1) | six[4]; + const u8 value = kSBox[box][row * 16 + column]; + for (int bit = 0; bit < 4; bit++) { + substituted[box * 4 + bit] = (value >> (3 - bit)) & 1; + } + } + + u8 mixed[32]; + Permute(substituted, mixed, kP, 32); + u8 next[32]; + for (int i = 0; i < 32; i++) { + next[i] = left[i] ^ mixed[i]; + } + memcpy(left, right, 32); + memcpy(right, next, 32); + } + + u8 combined[64]; + memcpy(combined, right, 32); + memcpy(combined + 32, left, 32); + Permute(combined, permuted, kFP, 64); + BitsToBytes(permuted, 8, out); + } + +private: + u8 subkeys_[16][48]; +}; + +// One DES key and IV per firmware series. +struct PSARTableKey { + u32 keyLow; + u32 keyHigh; + u8 iv[8]; +}; + +static const PSARTableKey kTableKeys[] = { + { 0xB730E5C7, 0x95620B49, { 0x9E, 0xA4, 0x33, 0x81, 0x86, 0x0C, 0x52, 0x85 } }, + { 0x45C9DC95, 0x5A7B3D9D, { 0xB2, 0xFE, 0xD9, 0x79, 0x8A, 0x02, 0xB1, 0x87 } }, + { 0x6F20585A, 0x4CCE495B, { 0x81, 0x08, 0xC1, 0xF2, 0x35, 0x98, 0x69, 0xB0 } }, + { 0x620BF15A, 0x73F45262, { 0x6D, 0x52, 0x1B, 0xA3, 0xC2, 0x36, 0xF9, 0x2B } }, + { 0xFD9D4498, 0xA664C8F8, { 0xDB, 0x4E, 0x79, 0x41, 0xF5, 0x97, 0x30, 0xAD } }, + { 0x3D6426E7, 0xD7BD7481, { 0xA6, 0x83, 0x0C, 0x2F, 0x63, 0x0B, 0x96, 0x29 } }, +}; + +// Which of those a firmware uses, from the version string in the archive's first block. +static int TableKeyIndexForVersion(std::string_view version) { + if (startsWith(version, "3.8") || startsWith(version, "3.9")) { + return 1; + } else if (startsWith(version, "4.")) { + return 2; + } else if (startsWith(version, "5.")) { + return 3; + } else if (startsWith(version, "6.")) { + return 4; + } + return 0; +} + +const char *PSPModelGenerationToString(PSPModelGeneration generation) { + switch (generation) { + case PSPModelGeneration::Any: return "any"; + case PSPModelGeneration::PSP_1000: return "01g"; + case PSPModelGeneration::PSP_2000: return "02g"; + case PSPModelGeneration::PSP_3000: return "03g"; + case PSPModelGeneration::PSP_4000: return "04g"; + case PSPModelGeneration::PSP_N1000: return "05g"; + case PSPModelGeneration::PSP_6000: return "06g"; + case PSPModelGeneration::PSP_7000: return "07g"; + case PSPModelGeneration::PSP_9000: return "09g"; + case PSPModelGeneration::PSP_11000: return "11g"; + default: return "unknown"; + } +} + +bool PSPModelGenerationFromString(std::string_view name, PSPModelGeneration *generation) { + if (equalsNoCase(name, "any")) { + *generation = PSPModelGeneration::Any; + return true; + } + // "03g" and a bare "3" both work. + std::string_view digits = name; + if (digits.size() > 1 && (digits.back() == 'g' || digits.back() == 'G')) { + digits.remove_suffix(1); + } + if (digits.empty() || digits.size() > 2) { + return false; + } + int value = 0; + for (char c : digits) { + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + if (value < 0 || value > (int)PSPModelGeneration::MAX) { + return false; + } + *generation = (PSPModelGeneration)value; + return true; +} + +// Entries are named one of two ways, depending on the firmware's age. +// +// 6.x archives use a flat five-digit token: "00001".."00012" are the per-model file lists, and +// everything above that is a file which could be named by any of them. +// +// 3.x archives group by model instead - "com:00123", "01g:00005" - where the group is either +// "com" for files every model gets or "NNg" for one model's, and ":00000" is that group's +// file list. Files there only ever need their own group's list. +// +// Returns the group's generation (0 for "com" and for the flat form) and the numeric index. +static bool SplitEntryName(std::string_view name, int *generation, int *index, bool *grouped) { + std::string_view digits = name; + *generation = 0; + *grouped = false; + + const size_t colon = name.find(':'); + if (colon != std::string_view::npos) { + const std::string_view group = name.substr(0, colon); + digits = name.substr(colon + 1); + *grouped = true; + if (group != "com") { + // "01g".."12g". + if (group.size() != 3 || group[2] != 'g' || group[0] < '0' || group[0] > '9' || group[1] < '0' || group[1] > '9') { + return false; + } + *generation = (group[0] - '0') * 10 + (group[1] - '0'); + } + } + + if (digits.size() != 5) { + return false; + } + int value = 0; + for (char c : digits) { + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + } + *index = value; + return true; +} + +// True if this entry is a file list rather than a file, and if so which model's. +static bool IsNameTableEntry(std::string_view name, int *generation) { + int index = 0; + bool grouped = false; + int group = 0; + if (!SplitEntryName(name, &group, &index, &grouped)) { + return false; + } + if (grouped) { + // ":00000" lists that group. + *generation = group; + return index == 0; + } + *generation = index; + return index >= 1 && index <= (int)PSPModelGeneration::MAX; +} + +// Decrypts a name table in place and returns the length of the text in it, or <= 0 on failure. +static int DecryptNameTable(std::vector &table, int keyIndex) { + if (keyIndex < 0 || keyIndex >= (int)ARRAY_SIZE(kTableKeys) || table.size() < 8) { + return -1; + } + const PSARTableKey &tableKey = kTableKeys[keyIndex]; + + u8 key[8]; + const u64 combined = ((u64)tableKey.keyHigh << 32) | tableKey.keyLow; + for (int i = 0; i < 8; i++) { + key[i] = (u8)(combined >> (56 - i * 8)); + } + const DESContext des(key); + + u8 previous[8]; + memcpy(previous, tableKey.iv, sizeof(previous)); + for (size_t offset = 0; offset + 8 <= table.size(); offset += 8) { + u8 *at = table.data() + offset; + u8 cipher[8], plain[8]; + memcpy(cipher, at, sizeof(cipher)); + des.DecryptBlock(cipher, plain); + for (int i = 0; i < 8; i++) { + at[i] = plain[i] ^ previous[i]; + } + memcpy(previous, cipher, sizeof(previous)); + } + + // What's underneath is an ordinary PRX blob. + return pspDecryptPRX(table.data(), table.data(), (u32)table.size()); +} + +// Table text is one "shortnamerealpath" per line. 6.x separates with a comma and writes the +// path as "flash0:/font/x.pgf"; 3.x separates with a bar and writes "flash0/font/x.pgf". The path +// difference doesn't matter - RelativePathFromEntryName normalizes both. +static void ParseNameTable(const char *text, size_t length, std::map *names) { + size_t start = 0; + while (start < length) { + size_t end = start; + while (end < length && text[end] != '\r' && text[end] != '\n') { + end++; + } + const std::string_view line(text + start, end - start); + const size_t separator = line.find_first_of(",|"); + if (separator != std::string_view::npos && separator > 0) { + names->emplace(std::string(line.substr(0, separator)), std::string(line.substr(separator + 1))); + } + while (end < length && (text[end] == '\r' || text[end] == '\n')) { + end++; + } + start = end; + } +} + +class PSARReader { +public: + PSARReader(const u8 *psar, size_t size) : psar_(psar), size_(size) {} + + bool Init(std::string *error); + + // 1: got an entry. 0: no more entries. -1: failed. + int NextEntry(std::string *error); + + const std::string &firmwareVersion() const { return firmwareVersion_; } + const std::string &entryName() const { return entryName_; } + bool entryIsDirectory() const { return entryIsDirectory_; } + PSARCompression entryCompression() const { return entryCompression_; } + // Empty for a directory, or for an entry we couldn't decompress. + const std::vector &entryData() const { return entryData_; } + +private: + // Decrypts one record into 'out'. Returns the decrypted size, or <= 0 on failure. + int DecodeBlock(u32 offset, u32 cbIn, std::vector &out); + + const u8 *psar_; + size_t size_; + + bool decrypted_ = false; + bool oldschool_ = false; + u32 overhead_ = PSAR_HEADER_SIZE; + u32 pos_ = 0; + // Where the records stop. The header says how long the archive really is, and both the ones + // I've looked at have a few bytes of padding after that which don't decode as a record. + size_t limit_ = 0; + std::string firmwareVersion_; + + std::string entryName_; + bool entryIsDirectory_ = false; + PSARCompression entryCompression_ = PSARCompression::None; + std::vector entryData_; + + std::vector block_; + std::vector block2_; +}; + +int PSARReader::DecodeBlock(u32 offset, u32 cbIn, std::vector &out) { + if (cbIn == 0 || cbIn > PSAR_MAX_ENTRY_BYTES || offset >= size_) { + return -1; + } + const u8 *in = psar_ + offset; + const size_t avail = size_ - offset; + + if (decrypted_) { + if (cbIn > avail) { + return -1; + } + out.assign(in, in + cbIn); + return (int)cbIn; + } + + if (cbIn > avail) { + return -1; + } + // The decrypter reads a little past the block and does use what it finds there, so hand it the + // real bytes whenever the archive has them. It hasn't got them for the very last entry, which + // ends flush with the last record - zero-fill only then, rather than refusing the entry, which + // is what used to lose every 1.x and 2.x archive's final file. + const size_t slack = std::min(0x10, avail - cbIn); + out.assign(in, in + cbIn + slack); + out.resize((size_t)cbIn + 0x10, 0); + + if (!oldschool_) { + // "Demangle": the 0x130 bytes at +0x20 are AES-128-CBC encrypted on top of everything + // else, and hide the PRX tag at +0xD0 that says how to decrypt the rest. + if (cbIn < 0x150) { + return -1; + } + kirk7(out.data() + 0x20, in + 0x20, 0x130, PSAR_DEMANGLE_KEYSEED); + } + + const int decrypted = pspDecryptPRX(out.data(), out.data(), cbIn); + if (decrypted <= 0) { + WARN_LOG(Log::Loader, "PSAR: block at %u (%u bytes) failed to decrypt: %d, tag %08x", offset, cbIn, decrypted, ReadU32(out.data() + 0xD0)); + } + return decrypted; +} + +bool PSARReader::Init(std::string *error) { + if (size_ < 0x40 || ReadU32(psar_) != PSAR_MAGIC) { + *error = "Not a PSAR archive"; + return false; + } + + const u8 version = psar_[4]; + oldschool_ = version == 1; + decrypted_ = ReadU32(psar_ + 0x20) == PSAR_DECRYPTED_MARKER; + overhead_ = decrypted_ ? 0 : PSAR_HEADER_SIZE; + + limit_ = size_; + const u32 declaredSize = ReadU32(psar_ + 8); + if (declaredSize >= 0x40 && declaredSize <= size_) { + limit_ = declaredSize; + } + + INFO_LOG(Log::Loader, "PSAR version %d, %s, %d bytes (%d of records)", version, + decrypted_ ? "already decrypted" : "encrypted", (int)size_, (int)limit_); + + int decoded = DecodeBlock(0x10, overhead_ + PSAR_ENTRY_SIZE, block_); + if (decoded != (int)PSAR_ENTRY_SIZE) { + *error = StringFromFormat("Couldn't decrypt the PSAR's first block (got %d)", decoded); + return false; + } + + // The first block is a text record ending in the firmware version, e.g. "...,6.61". + const char *text = (const char *)block_.data() + 0x10; + const size_t textLen = strnlen(text, PSAR_ENTRY_SIZE - 0x10); + const std::string_view firstLine(text, textLen); + const size_t comma = firstLine.rfind(','); + firmwareVersion_ = comma == std::string_view::npos ? std::string(firstLine) : std::string(firstLine.substr(comma + 1)); + + pos_ = 0x10 + overhead_ + PSAR_ENTRY_SIZE; + + if (decrypted_) { + decoded = DecodeBlock(pos_, ReadU32(block_.data() + 0x90), block2_); + if (decoded <= 0) { + *error = "Couldn't read the PSAR's second block"; + return false; + } + pos_ += overhead_ + decoded; + return true; + } + + if (!oldschool_) { + // The second block's size isn't recorded anywhere, so try the ones real updaters use. + // 100 covers most, 2.7x is bigger, and some store it in the first block. + const u32 candidates[] = { 100, 144, ReadU16(block_.data() + 0x90) }; + decoded = -1; + for (u32 candidate : candidates) { + if (candidate == 0) { + continue; + } + decoded = DecodeBlock(pos_, overhead_ + candidate, block2_); + if (decoded > 0) { + break; + } + } + if (decoded <= 0) { + *error = "Couldn't read the PSAR's second block"; + return false; + } + pos_ += overhead_ + ((decoded + 15) & ~15); + } + + return true; +} + +int PSARReader::NextEntry(std::string *error) { + entryName_.clear(); + entryData_.clear(); + entryIsDirectory_ = false; + entryCompression_ = PSARCompression::None; + + // Stop when what's left can't hold another whole record. Both archives I've looked at end with + // a few bytes of padding that would otherwise be decoded as a truncated entry and reported as + // a failure right at the finish line. + if ((size_t)pos_ + overhead_ + PSAR_ENTRY_SIZE > limit_) { + return 0; + } + + int decoded = DecodeBlock(pos_, overhead_ + PSAR_ENTRY_SIZE, block_); + if (decoded != (int)PSAR_ENTRY_SIZE) { + *error = StringFromFormat("Couldn't decrypt the entry at %u (got %d)", pos_, decoded); + return -1; + } + + // A well-formed entry has a zero here. If it doesn't, we've lost our place in the archive. + if (ReadU32(block_.data() + 0x100) != 0) { + *error = StringFromFormat("Malformed entry at %u", pos_); + return -1; + } + + const char *name = (const char *)block_.data() + 4; + entryName_.assign(name, strnlen(name, PSAR_ENTRY_SIZE - 4)); + + pos_ += overhead_ + PSAR_ENTRY_SIZE; + + const u32 chunkSize = ReadU32(block_.data() + 0x104); + const u32 expandedSize = ReadU32(block_.data() + 0x108); + + if (expandedSize == 0) { + entryIsDirectory_ = true; + } else if (expandedSize > PSAR_MAX_ENTRY_BYTES) { + *error = StringFromFormat("Entry '%s' claims an unreasonable size (%u)", entryName_.c_str(), expandedSize); + return -1; + } else { + decoded = DecodeBlock(pos_, chunkSize, block2_); + if (decoded <= 0) { + WARN_LOG(Log::Loader, "PSAR: couldn't decrypt the contents of '%s'", entryName_.c_str()); + entryCompression_ = PSARCompression::Unknown; + } else { + entryCompression_ = DetectCompression(block2_.data(), decoded); + if (entryCompression_ == PSARCompression::Zlib) { + entryData_.resize(expandedSize); + uLongf destLen = expandedSize; + const int zResult = uncompress(entryData_.data(), &destLen, block2_.data(), decoded); + if (zResult != Z_OK || destLen != expandedSize) { + WARN_LOG(Log::Loader, "PSAR: inflate failed for '%s' (%d)", entryName_.c_str(), zResult); + entryData_.clear(); + } + } + // Anything else we leave empty - the caller reports it through the stats. + } + } + + pos_ += chunkSize; + return 1; +} + +bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error) { + PSARUnpackStats localStats; + if (!stats) { + stats = &localStats; + } + std::string localError; + if (!error) { + error = &localError; + } + + PSARReader reader(psar, psarSize); + if (!reader.Init(error)) { + return false; + } + stats->firmwareVersion = reader.firmwareVersion(); + const int tableKeyIndex = TableKeyIndexForVersion(stats->firmwareVersion); + INFO_LOG(Log::Loader, "Unpacking firmware %s (name table key %d)", stats->firmwareVersion.c_str(), tableKeyIndex); + + // One file list per model, filled in as we go - they come before the files they name. + std::map> namesByModel; + INFO_LOG(Log::Loader, "Resolving names against model %s", PSPModelGenerationToString(options.model)); + + while (true) { + const int result = reader.NextEntry(error); + if (result < 0) { + // Losing our place means the rest of the archive is unreadable, so stop rather than + // spraying garbage - but keep whatever we already extracted. + ERROR_LOG(Log::Loader, "PSAR: %s", error->c_str()); + stats->failed++; + return false; + } + if (result == 0) { + break; + } + + stats->entries++; + stats->compressionCounts[(int)reader.entryCompression()]++; + + if (options.verbose) { + INFO_LOG(Log::Loader, "PSAR entry '%s' (%s, %d bytes)", reader.entryName().c_str(), + PSARCompressionToString(reader.entryCompression()), (int)reader.entryData().size()); + } + + if (reader.entryIsDirectory()) { + stats->directories++; + continue; + } + + // The name tables have to be read before anything they name shows up, which the archive's + // own ordering takes care of. + int tableGeneration = 0; + if (IsNameTableEntry(reader.entryName(), &tableGeneration)) { + std::vector table = reader.entryData(); + const int textLength = DecryptNameTable(table, tableKeyIndex); + if (textLength > 0 && (size_t)textLength <= table.size()) { + stats->nameTables++; + std::map &names = namesByModel[tableGeneration]; + ParseNameTable((const char *)table.data(), textLength, &names); + INFO_LOG(Log::Loader, "PSAR: file list '%s' names %d files", reader.entryName().c_str(), (int)names.size()); + continue; + } + // Which numbers are file lists varies between firmwares - 6.61 uses 1-11, 6.00 stops + // earlier and has real files in that range. A list always decrypts (the PRX layer + // underneath validates a hash), so failing here means this is just a file. + DEBUG_LOG(Log::Loader, "PSAR: '%s' isn't a file list, treating it as a file", reader.entryName().c_str()); + } + + int entryGeneration = 0; + int entryIndex = 0; + bool entryGrouped = false; + SplitEntryName(reader.entryName(), &entryGeneration, &entryIndex, &entryGrouped); + + std::string realName; + bool wrongModel = false; + if (EntryNameIsRealPath(reader.entryName())) { + realName = reader.entryName(); + } else if (entryGrouped) { + // The name says which list to look in, and the lists key on just the number - entry + // "com:00004" is "00004" in the com list. "com" entries are common to every model, so + // they're wanted whichever one was asked for. + if (options.model != PSPModelGeneration::Any && entryGeneration != 0 && entryGeneration != (int)options.model) { + wrongModel = true; + } else { + const auto model = namesByModel.find(entryGeneration); + if (model != namesByModel.end()) { + const std::string key = reader.entryName().substr(reader.entryName().find(':') + 1); + const auto found = model->second.find(key); + if (found != model->second.end()) { + realName = found->second; + } + } + } + } else if (options.model != PSPModelGeneration::Any) { + // Only this model's list counts. A file it doesn't name belongs to some other model. + const auto model = namesByModel.find((int)options.model); + if (model != namesByModel.end()) { + const auto found = model->second.find(reader.entryName()); + if (found != model->second.end()) { + realName = found->second; + } else { + wrongModel = true; + } + } + } else { + // A short name that several lists claim is the same file, so take the first name for it. + for (const auto &[generation, names] : namesByModel) { + const auto found = names.find(reader.entryName()); + if (found != names.end()) { + realName = found->second; + break; + } + } + } + + if (wrongModel) { + stats->otherModel++; + continue; + } + + realName = CanonicalFlashPath(realName); + + if (realName.empty()) { + // No list claimed this one. Still worth writing out under its short name, but a + // filter has nothing to match it against. + stats->unnamed++; + realName = reader.entryName(); + if (!options.prefixFilter.empty()) { + stats->skippedByFilter++; + continue; + } + } else if (!options.prefixFilter.empty() && !startsWithNoCase(realName, options.prefixFilter)) { + stats->skippedByFilter++; + continue; + } + + if (reader.entryData().empty()) { + ERROR_LOG(Log::Loader, "PSAR: no usable contents for '%s' (%s)", reader.entryName().c_str(), + PSARCompressionToString(reader.entryCompression())); + stats->failed++; + continue; + } + + if (options.listOnly) { + stats->written++; + continue; + } + + std::string relative; + if (!RelativePathFromEntryName(realName, &relative)) { + ERROR_LOG(Log::Loader, "PSAR: refusing to write '%s'", realName.c_str()); + stats->failed++; + continue; + } + + const Path destination = outputDir / relative; + if (!File::CreateFullPath(destination.NavigateUp())) { + ERROR_LOG(Log::Loader, "PSAR: couldn't create a directory for '%s'", relative.c_str()); + stats->failed++; + continue; + } + if (!File::WriteDataToFile(false, reader.entryData().data(), reader.entryData().size(), destination)) { + ERROR_LOG(Log::Loader, "PSAR: couldn't write '%s'", destination.c_str()); + stats->failed++; + continue; + } + stats->written++; + } + + return true; +} + +// Where a disc keeps its updater, relative to the root. Both files are unwrapped: DATA.BIN is the +// bare archive, with no PBP around it, and EBOOT.BIN next to it is the updater executable we have +// no use for. Reachable either by opening the image ourselves or, while a game is running, as +// disc0: - hence the prefixes rather than whole paths. +static const char *UPDATE_DIR_SUFFIX = "PSP_GAME/SYSDIR/UPDATE"; +static const char *UPDATE_PSAR_SUFFIX = "PSP_GAME/SYSDIR/UPDATE/DATA.BIN"; +static const char *UPDATE_SFO_SUFFIX = "PSP_GAME/SYSDIR/UPDATE/PARAM.SFO"; + +// A disc updater's PARAM.SFO titles itself "PSP(tm) Update ver 3.95"; we want the number. +static std::string VersionFromUpdaterTitle(std::string_view title) { + const size_t space = title.rfind(' '); + if (space != std::string_view::npos) { + return std::string(title.substr(space + 1)); + } + return std::string(title); +} + +static std::string VersionFromSFO(const std::vector &sfo) { + ParamSFOData paramSFO; + if (sfo.empty() || !paramSFO.ReadSFO(sfo)) { + return std::string(); + } + return VersionFromUpdaterTitle(paramSFO.GetValueString("TITLE")); +} + +// Opens filename as a disc image, if it is one. Returns null otherwise, which is the normal +// outcome for a PBP or a loose DATA.BIN. +static ISOFileSystem *OpenDisc(const Path &filename, FileLoader *loader, IHandleAllocator *hAlloc) { + std::string blockError; + std::shared_ptr device(ConstructBlockDevice(loader, &blockError)); + if (!device) { + return nullptr; + } + ISOFileSystem *iso = new ISOFileSystem(hAlloc, device); + if (!iso->GetFileInfo("/" + std::string(UPDATE_DIR_SUFFIX)).exists) { + delete iso; + return nullptr; + } + return iso; +} + +static bool ReadWholeFile(IFileSystem *fs, const char *path, std::vector *out) { + const PSPFileInfo info = fs->GetFileInfo(path); + if (!info.exists || info.size == 0 || info.size > PSAR_MAX_DISC_FILE_BYTES) { + return false; + } + const int handle = fs->OpenFile(path, FILEACCESS_READ); + if (handle < 0) { + return false; + } + out->resize((size_t)info.size); + const size_t read = fs->ReadFile(handle, out->data(), info.size); + fs->CloseFile(handle); + if (read != (size_t)info.size) { + out->clear(); + return false; + } + return true; +} + +// Pulls the archive out of whatever this is - see the header for the shapes we accept. +static bool ReadUpdaterPSAR(const Path &filename, std::vector *psar, std::string *sfoVersion, std::string *error) { + FileLoader *loader = ConstructFileLoader(filename); + if (!loader || !loader->Exists()) { + *error = "Couldn't open " + filename.ToString(); + delete loader; + return false; + } + + SequentialHandleAllocator hAlloc; + if (ISOFileSystem *disc = OpenDisc(filename, loader, &hAlloc)) { + // A game disc with an updater on it. + std::vector sfo; + if (sfoVersion && ReadWholeFile(disc, ("/" + std::string(UPDATE_SFO_SUFFIX)).c_str(), &sfo)) { + *sfoVersion = VersionFromSFO(sfo); + } + const bool ok = ReadWholeFile(disc, ("/" + std::string(UPDATE_PSAR_SUFFIX)).c_str(), psar); + delete disc; + delete loader; + if (!ok) { + *error = "No updater on the disc " + filename.ToString(); + return false; + } + INFO_LOG(Log::Loader, "Found a %d byte updater on the disc %s", (int)psar->size(), filename.c_str()); + return true; + } + + u8 magic[4]{}; + loader->ReadAt(0, sizeof(magic), magic); + + if (!memcmp(magic, "\0PBP", 4)) { + // A downloaded updater. + PBPReader pbp(loader); + std::vector sfo; + if (sfoVersion && pbp.IsValid() && pbp.GetSubFile(PBP_PARAM_SFO, &sfo)) { + *sfoVersion = VersionFromSFO(sfo); + } + const bool ok = pbp.IsValid() && pbp.GetSubFile(PBP_UNKNOWN_PSAR, psar); + delete loader; + if (!ok || psar->size() < 0x40) { + *error = "No DATA.PSAR in " + filename.ToString(); + return false; + } + INFO_LOG(Log::Loader, "Found a %d byte DATA.PSAR in %s", (int)psar->size(), filename.c_str()); + return true; + } + + if (ReadU32(magic) == PSAR_MAGIC) { + // A disc updater's DATA.BIN, or an archive someone already pulled out. + const s64 size = loader->FileSize(); + if (size < 0x40 || size > PSAR_MAX_DISC_FILE_BYTES) { + *error = "Bad PSAR size in " + filename.ToString(); + delete loader; + return false; + } + psar->resize((size_t)size); + const size_t read = loader->ReadAt(0, psar->size(), psar->data()); + delete loader; + if (read != psar->size()) { + *error = "Couldn't read " + filename.ToString(); + return false; + } + INFO_LOG(Log::Loader, "Found a %d byte bare PSAR: %s", (int)psar->size(), filename.c_str()); + return true; + } + + delete loader; + *error = filename.ToString() + " isn't an updater, a PSAR or a disc with one on it"; + return false; +} + +std::string ReadUpdaterVersion(const Path &filename) { + std::vector psar; + std::string version; + std::string error; + if (!ReadUpdaterPSAR(filename, &psar, &version, &error)) { + return std::string(); + } + return version; +} + +// The mounted disc, i.e. the game that's running. Everything here goes through pspFileSystem +// rather than opening the image again - it's already open, and for a folder-based or otherwise +// unusual "disc" there may be no image to open. +static bool ReadFromMountedDisc(const char *suffix, std::vector *out) { + if (pspFileSystem.ReadEntireFile(std::string("disc0:/") + suffix, *out, true) < 0) { + out->clear(); + return false; + } + return !out->empty(); +} + +bool MountedDiscHasUpdater() { + return pspFileSystem.GetFileInfo(std::string("disc0:/") + UPDATE_PSAR_SUFFIX).exists; +} + +std::string ReadMountedDiscUpdaterVersion() { + std::vector sfo; + if (!ReadFromMountedDisc(UPDATE_SFO_SUFFIX, &sfo)) { + return std::string(); + } + return VersionFromSFO(sfo); +} + +bool UnpackUpdaterFromMountedDisc(const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error) { + std::string localError; + if (!error) { + error = &localError; + } + + std::vector psar; + if (!ReadFromMountedDisc(UPDATE_PSAR_SUFFIX, &psar) || psar.size() < 0x40) { + *error = "The mounted disc has no updater on it"; + return false; + } + + INFO_LOG(Log::Loader, "Found a %d byte updater on the mounted disc", (int)psar.size()); + return UnpackPSAR(psar.data(), psar.size(), outputDir, options, stats, error); +} + +bool UnpackUpdater(const Path &filename, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error) { + std::string localError; + if (!error) { + error = &localError; + } + + std::vector psar; + if (!ReadUpdaterPSAR(filename, &psar, nullptr, error)) { + return false; + } + return UnpackPSAR(psar.data(), psar.size(), outputDir, options, stats, error); +} diff --git a/Core/Util/PSARUnpack.h b/Core/Util/PSARUnpack.h new file mode 100644 index 0000000000..42f3afe3b3 --- /dev/null +++ b/Core/Util/PSARUnpack.h @@ -0,0 +1,120 @@ +// Copyright (c) 2026- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/CommonTypes.h" + +class Path; + +// Unpacks the firmware image inside an official PSP updater (PSP/GAME/UPDATE/EBOOT.PBP). +// +// The updater's DATA.PSAR is a flat sequence of encrypted, compressed file records. The real +// updater installs them by executing on the PSP, but the archive can be walked directly - each +// record carries its own name, sizes and a standard PRX-style encryption header, so all it needs +// is the KIRK engine and the PRX decrypter we already have. +// +// The main use is pulling flash0:/font out of an updater the user supplies, which is why a +// prefix filter is part of the API rather than something the caller filters afterwards. + +enum class PSARCompression { + None, + Zlib, + KL4E, + KL3E, + LZR, + Unknown, +}; + +const char *PSARCompressionToString(PSARCompression c); + +// PSP hardware revisions, as the updater numbers them. An updater carries one file list per +// model, so which one you resolve names against decides both what a file is called and whether +// it's part of that model's firmware at all. +enum class PSPModelGeneration { + Any = 0, // Whichever list names a file first. Use this to extract everything. + PSP_1000 = 1, + PSP_2000 = 2, + PSP_3000 = 3, + PSP_4000 = 4, + PSP_N1000 = 5, // PSP Go + PSP_6000 = 6, + PSP_7000 = 7, + PSP_9000 = 9, + PSP_11000 = 11, + MAX = 12, +}; + +const char *PSPModelGenerationToString(PSPModelGeneration generation); +// Accepts "01g".."12g", a bare number, or "any". Returns false if it's none of those. +bool PSPModelGenerationFromString(std::string_view name, PSPModelGeneration *generation); + +struct PSARUnpackOptions { + // Only unpack entries whose name starts with this, e.g. "flash0:/font/". Case insensitive. + // Empty means everything. Entries whose real name we can't recover are never matched by a + // non-empty filter. + std::string prefixFilter; + // Which model's file list to resolve names against. Anything that model's list doesn't name + // isn't part of its firmware, and is skipped. + PSPModelGeneration model = PSPModelGeneration::Any; + // Walk and report, but don't write any files. + bool listOnly = false; + // Log a line per entry. Off by default - an updater holds well over a thousand of them. + bool verbose = false; +}; + +struct PSARUnpackStats { + std::string firmwareVersion; + int entries = 0; + int directories = 0; + int written = 0; + int skippedByFilter = 0; + int nameTables = 0; // Entries that were file lists rather than files. + int unnamed = 0; // Short names no file list claimed. + int otherModel = 0; // Files that belong to a model other than the requested one. + int failed = 0; + // How many entries used each compression, indexed by PSARCompression. + int compressionCounts[6]{}; +}; + +// psar points at the DATA.PSAR contents, starting with the "PSAR" magic. +bool UnpackPSAR(const u8 *psar, size_t psarSize, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error); + +// Finds the firmware image in whatever an updater arrives as and unpacks it: +// - a downloaded updater EBOOT.PBP, where the archive is its DATA.PSAR +// - a disc updater's DATA.BIN, which is the same archive without the PBP wrapper +// - a game ISO/CSO/CHD, which is searched for PSP_GAME/SYSDIR/UPDATE/DATA.BIN +// The last one is the interesting case: most UMDs carry a firmware updater, so the fonts can come +// from whatever game the user already has rather than a separate download. +bool UnpackUpdater(const Path &filename, const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error); + +// The version string an updater advertises ("6.61"), read from the PARAM.SFO next to it - no +// decryption needed, so it's cheap enough to check every disc with. Empty if there's no updater. +std::string ReadUpdaterVersion(const Path &filename); + +// The same three, for the disc mounted as disc0: - i.e. the game that's running. These read +// through the mounted filesystem instead of opening the image a second time, which also means +// they work for the shapes that aren't an image at all, like a folder-based "disc". +// +// This is the path the font extraction is meant to take: while a game is running, ask whether its +// disc carries an updater, and if so unpack just flash0:/font out of it. +bool MountedDiscHasUpdater(); +std::string ReadMountedDiscUpdaterVersion(); +bool UnpackUpdaterFromMountedDisc(const Path &outputDir, const PSARUnpackOptions &options, PSARUnpackStats *stats, std::string *error); diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index da01877e66..fc9adcd92a 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -332,6 +332,7 @@ + @@ -646,6 +647,7 @@ + diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index 5834076ba8..40dec34a6e 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -277,6 +277,7 @@ + @@ -662,6 +663,7 @@ + diff --git a/android/jni/Android.mk b/android/jni/Android.mk index b3d9294826..ddbc8c79f2 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -805,6 +805,7 @@ EXEC_AND_LIB_FILES := \ $(SRC)/Core/Util/GameManager.cpp \ $(SRC)/Core/Util/BlockAllocator.cpp \ $(SRC)/Core/Util/PPGeDraw.cpp \ + $(SRC)/Core/Util/PSARUnpack.cpp \ $(SRC)/Core/Util/RecentFiles.cpp \ $(SRC)/Core/Util/VideoPlayer.cpp \ $(SRC)/git-version.cpp diff --git a/docs/PsarFileFormat.md b/docs/PsarFileFormat.md new file mode 100644 index 0000000000..ce00ffc4f5 --- /dev/null +++ b/docs/PsarFileFormat.md @@ -0,0 +1,222 @@ +# The PSAR archive format (PSP firmware updaters) + +An official PSP firmware updater carries the whole firmware image in a PSAR archive. PPSSPP can +walk one and extract the files out of it without emulating the updater, which is what +`Core/Util/PSARUnpack.cpp` does. The main use is getting `flash0:/font` - the PGF system fonts - +out of an updater the user already has, since most UMDs carry one. + +This document is what was learned reading 43 different firmware versions, from 1.50 to 6.61. It's +a description of the format rather than of the code; the code is the same shape but has the error +handling. + +## Where an updater lives + +The archive turns up in three shapes, all of them the same bytes once you find them: + +| Shape | Where | Notes | +| --- | --- | --- | +| Downloaded updater | `EBOOT.PBP`, subfile 7 (`DATA.PSAR`) | The one you get from Sony's site | +| Disc updater | `PSP_GAME/SYSDIR/UPDATE/DATA.BIN` | Not wrapped in a PBP - the bare archive | +| Loose | any file starting with `PSAR` | Whatever someone already pulled out | + +A disc updater directory also holds `EBOOT.BIN` (the updater executable, a bare `~PSP` PRX, of no +use to us) and `PARAM.SFO`. That SFO is worth knowing about: its `TITLE` reads +`"PSP™ Update ver 3.95"`, so **the firmware version can be read without decrypting anything**, +which is cheap enough to check a whole game library with. `DISC_ID` is `MSTKUPDATE` and `CATEGORY` +is `MG`. + +Of 589 discs scanned across two libraries, 473 carried an updater. + +## Archive header + +``` ++0x00 u32 "PSAR" magic (0x52415350) ++0x04 u8 version. 1 = "oldschool" (see below), 2 and 3 are the common ones ++0x08 u32 total length of the records ++0x20 u32 0x2C333333 if this archive is already decrypted, otherwise part of the ciphertext +``` + +The length at +0x08 matters: archives have a few bytes of padding after the last record (16 and 32 +bytes in the two checked), and a walk that goes by file size instead will try to decode that +padding as a record and report a spurious failure at the very end. + +The marker at +0x20 only appears in archives some other tool has already decrypted. In an original +updater those bytes are ciphertext and will not match. + +## Records + +The archive is a flat sequence of records starting at +0x10. There is no index - you find entry +*n+1* by decoding entry *n* and adding up its sizes. Each record is: + +``` +[ 0x150 bytes encryption header ] <- absent if the archive is pre-decrypted +[ 0x110 bytes entry description ] +[ variable file contents ] <- has its own 0x150 header, same as above +``` + +Call the header size *overhead*: 0x150 normally, 0 for a pre-decrypted archive. + +### Decoding a block + +Every encrypted block - entry descriptions and file contents alike - is decoded the same way: + +1. **Demangle.** The 0x130 bytes at block+0x20 are AES-128-CBC encrypted on top of everything + else. Decrypt them with KIRK command 7 (`KIRK_CMD_DECRYPT_IV_0`), key seed **0x55**, IV zero, + writing the result back over block+0x20. This step is skipped for version 1 archives. + + The point of it is that it hides the PRX tag: you cannot tell how to decrypt the block until + you have done this. + +2. **PRX decrypt.** The block is now an ordinary PRX blob, with its tag at block+0xD0. Run it + through the normal PRX decrypter (`pspDecryptPRX`). The tag official updaters use is + **0x0E000000**; `0x06000000` also appears in the wild. + + Note that `0x0E000000`'s key is stored *unscrambled*, unlike almost every other key in + PPSSPP's tag table, so it needs the `kirk7` pass applied before use - that's what the + `scrambleKey` flag on `TAG_INFO` is for. + +One wrinkle worth knowing: the decrypter reads a little past the end of the block and **uses what +it finds there**, so a decoder should hand it about 16 bytes of slack from the archive. The last +record ends flush with the end of the archive and has no slack to give; zero-filling it there is +fine, but zero-filling it everywhere breaks every archive that does have those bytes. + +### Entry description + +0x110 bytes once decoded: + +``` ++0x004 char[] entry name, NUL padded (see "Naming" below) ++0x100 u32 always zero. If it isn't, you've lost your place in the archive ++0x104 u32 size of the contents block that follows, including its 0x150 header ++0x108 u32 uncompressed size of the file. 0 means this entry is a directory +``` + +A directory entry still has a contents block to skip over - advance by `+0x104` either way. + +### The first two records + +The first record (at +0x10) is not a file. Decoded, it holds a text string at +0x10 whose last +comma-separated field is the firmware version, e.g. `...,6.61`. That's the authoritative version +for choosing decryption keys further down. + +There is a second record after it whose length isn't recorded anywhere. In practice it is +`overhead + 100`, or `overhead + 144` on 2.7x, or `overhead + u16 at +0x90 of the first record`. +Try them in that order and take the first that decodes. Round the result up to a multiple of 16 +before advancing. Version 1 archives don't have this record at all. + +## Contents + +The contents block decodes to a compressed stream. Sniff the first bytes: + +| Magic | Format | +| --- | --- | +| `78 9C` | zlib | +| `KL4E` | KL4E - Sony's own, not implemented in PPSSPP | +| `KL3E` | KL3E - likewise | +| `2RLZ` | LZR | + +Across all 43 firmware versions examined, **every file in every archive was zlib**. KL4E does turn +up on the PSP, but a layer further in: two kernel modules (`memlmd_01g.prx` and `loadexec_01g.prx`) +are themselves KL4E-compressed *inside* their own PRX, which is a separate problem from this one. + +## Naming + +This is the part that changed most across firmware generations, and the part most likely to catch +out an implementation tested against only one updater. + +### 1.x and 2.x - real paths + +Entries are named outright: `flash0:/vsh/resource/opening_plugin.rco`, `flash1:/registry`, +`ipl:/psp_nandipl.bin`. Nothing else is needed. Note `ipl:` - it is not enough to recognise +`flash0:` and `flash1:`; better to treat any `:/` shape as a real path. + +### 3.x - grouped short names + +Entries are named `:<5 digits>`, where the group is `com` for files every model gets, or +`01g`, `02g`, ... for one model's. `:00000` is that group's **file list**, and the rest of +that group's entries are looked up in it by their number alone (entry `com:00004` is key `00004`). + +### 5.x and 6.x - flat short names with per-model lists + +Entries are a bare 5-digit number. The low numbers are file lists, one per PSP model +(`00001` = 01g, `00002` = 02g, ...) and the rest are files that any of the lists may name. + +**Which numbers are lists is not fixed.** 6.61 uses 1-11; 6.00 has real files at `00010`-`00012`. +Since a list always decrypts successfully (the PRX layer inside validates a hash), the robust rule +is: try to decrypt a candidate as a list, and if that fails, treat it as an ordinary file. + +### File lists + +A list decodes to plain text, one `shortnamepath` per line. Both the separator and the +path style depend on the generation: + +| Generation | Separator | Path written as | +| --- | --- | --- | +| 3.x | `\|` | `flash0/font/ltn0.pgf` | +| 5.x, 6.x | `,` | `flash0:/font/ltn0.pgf` | + +Worth normalising to one form early - otherwise a filter like `flash0:/font/` silently matches +nothing on a 3.x archive. + +### List encryption + +The lists are encrypted on top of everything else, with **DES** - the tables in the reference +implementations are DES's own: a 56-entry PC-1, a 48-entry PC-2, eight 64-entry 4-bit S-boxes, a +32-entry P permutation, and textbook IP/FP bit-shuffling constants. + +It is DES-CBC decrypt, with the key assembled from two 32-bit words as `(high << 32) | low` in +big-endian byte order, and an IV alongside them. There is one such set per firmware series, and +which one applies is chosen from the version string in the archive's first record: + +| Firmware | Key set | +| --- | --- | +| 1.x, 2.x, 3.0 - 3.7 | 0 | +| 3.8, 3.9 | 1 | +| 4.x | 2 | +| 5.x | 3 | +| 6.x | 4 | + +A sixth set exists that nothing selects by version. The values themselves are in `kTableKeys` in +`Core/Util/PSARUnpack.cpp` and aren't repeated here. + +Underneath the DES layer is an ordinary PRX blob, so run the result through `pspDecryptPRX` as +well; what comes out of *that* is the text. + +A good check while implementing: after the DES pass but before the PRX pass, the u32 at +0xD0 +should be a recognisable PRX tag. A wrong key gives a random value there, so that single word +tells you whether the cipher is right before anything else has to work. + +## What the archives look like in practice + +Three structural eras, visible in the shape of the output: + +| Firmware | Naming | Directory entries | Fonts | +| --- | --- | --- | --- | +| 1.50 - 3.52 | real paths | 0-16 | 17-20 | +| 3.71 - 4.05 | `com:`/`NNg:` groups | 50 | 21 | +| 5.01 - 6.61 | flat + per-model lists | 16-17 | 21 | + +A 6.61 archive holds 435 records: 411 files, 17 directories and 7 file lists. Among the files are +295 `~PSP` modules, 61 PRF files, 7 encrypted XMB indices (`PSPsysGP`), and 21 files in +`flash0:/font` - 19 of them PGF, plus `gb3s1518.bwfon` and `imagefont.bin`. + +The "Fonts" column above counts files in `flash0:/font`, not PGFs specifically. + +## Gotchas, collected + +- Walk by the length at +0x08, not the file size, or the trailing padding decodes as a bad record. +- Stop when fewer than `overhead + 0x110` bytes remain rather than when the position passes the + end - the two differ by exactly one spurious record. +- Give the block decrypter its 16 bytes of slack, but don't require them for the last record. +- Don't assume which entry numbers are file lists; find out by trying. +- Don't assume there are file lists at all - 1.x and 2.x have none. +- `ipl:` is a device too. +- A 3.x path has no colon; a 6.x path does. + +## See also + +- `Core/Util/PSARUnpack.{cpp,h}` - the implementation, and the only consumer of most of this. +- `Core/ELF/PrxDecrypter.cpp` - the PRX layer every block goes through. +- `ext/libkirk` - `kirk7()` is the demangle step. +- `PPSSPPHeadless --unpack-updater=DIR ` unpacks one from the command line; + `--unpack-updater-model=01g..12g` picks a model. diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 578342f23b..0c6a927b25 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -57,6 +57,7 @@ #include "Core/EmuThread.h" #include "Core/MIPS/MIPSTables.h" #include "Core/System.h" +#include "Core/Util/PSARUnpack.h" #include "Core/WebServer.h" #include "Core/HLE/sceUtility.h" #include "Core/SaveState.h" @@ -612,6 +613,40 @@ int main(int argc, const char* argv[]) { g_logManager.EnableOutput(LogOutput::Printf); } + // Unpacking an updater is a standalone action - no emulation involved, so do it here and exit + // before any of the setup below. + if (cmdLineOptions.unpackUpdater.has_value()) { + if (cmdLineOptions.bootFilenames.size() != 1) { + fprintf(stderr, "--unpack-updater takes exactly one updater, disc image or PSAR\n"); + return 1; + } + + PSARUnpackOptions unpackOptions; + unpackOptions.verbose = testOptions.verbose; + if (cmdLineOptions.unpackUpdaterModel.has_value() && + !PSPModelGenerationFromString(cmdLineOptions.unpackUpdaterModel.value(), &unpackOptions.model)) { + fprintf(stderr, "Unknown PSP model '%s' - expected 01g..12g or any\n", cmdLineOptions.unpackUpdaterModel.value().c_str()); + return 1; + } + PSARUnpackStats stats; + std::string unpackError; + const bool ok = UnpackUpdater(Path(cmdLineOptions.bootFilenames[0]), Path(cmdLineOptions.unpackUpdater.value()), unpackOptions, &stats, &unpackError); + if (!ok) { + fprintf(stderr, "Unpacking failed: %s\n", unpackError.c_str()); + } + printf("Firmware %s (model %s): %d entries, %d files written, %d directories, %d unresolved names, %d for other models, %d failed\n", + stats.firmwareVersion.c_str(), PSPModelGenerationToString(unpackOptions.model), stats.entries, stats.written, + stats.directories, stats.unnamed, stats.otherModel, stats.failed); + printf("Compression: none=%d zlib=%d KL4E=%d KL3E=%d LZR=%d unknown=%d\n", + stats.compressionCounts[(int)PSARCompression::None], + stats.compressionCounts[(int)PSARCompression::Zlib], + stats.compressionCounts[(int)PSARCompression::KL4E], + stats.compressionCounts[(int)PSARCompression::KL3E], + stats.compressionCounts[(int)PSARCompression::LZR], + stats.compressionCounts[(int)PSARCompression::Unknown]); + return ok ? 0 : 1; + } + g_Config.RestoreDefaults(RestoreSettingsBits::SETTINGS | RestoreSettingsBits::CONTROLS | RestoreSettingsBits::RECENT, false); Core_RegisterDebugOutputListeners(&SendDebugOutput, &SendDebugScreenshot); diff --git a/libretro/Makefile.common b/libretro/Makefile.common index 6c49b5bc10..9a83bd6c8c 100644 --- a/libretro/Makefile.common +++ b/libretro/Makefile.common @@ -895,6 +895,7 @@ SOURCES_CXX += \ $(COREDIR)/Util/BlockAllocator.cpp \ $(COREDIR)/Util/MemStick.cpp \ $(COREDIR)/Util/PPGeDraw.cpp \ + $(COREDIR)/Util/PSARUnpack.cpp \ $(COREDIR)/Util/RecentFiles.cpp \ $(COREDIR)/Util/AudioFormat.cpp \ $(COREDIR)/Util/PathUtil.cpp \