Fix a heap overflow decrypting a PRX whose size we had to guess

KIRK CMD1 writes header + data_offset + align16(data_size) bytes into outbuf,
and all three come out of the header the decrypter just decrypted, not from the
caller. The SHA1 check doesn't bound them - it only covers the header, so it
passes just as happily for a block that's been cut short.

The PSAR walker has to guess how long an updater's second block is (nothing
records it, so it tries the sizes real updaters use), and a wrong guess sent
KIRK off the end of the buffer: unpacking a firmware crashed roughly half the
time, on every version and disc I tried, depending on the heap layout.

Bound the write against the size the caller gave us, in all six decrypt types.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Henrik Rydgård
2026-09-09 12:56:17 -06:00
co-authored by Claude Opus 5
parent 57e96b2d81
commit bb96802f72
+51
View File
@@ -780,6 +780,27 @@ struct PRXType9
};
static_assert(sizeof(PRXType9) == 0x150, "inconsistent size of PRX Type 9");
// KIRK CMD1 writes the header plus data_offset plus data_size (rounded up to 16) bytes into
// outbuf, and all three of those come out of the header we just decrypted rather than from the
// caller. The SHA1 check above doesn't bound them - it only covers the header, so it passes just
// as happily for a block that's been cut short as for a whole one. A caller that has to guess how
// long a block is (the PSAR walker does: an updater doesn't record the length of its second
// block, so it tries the sizes real updaters use) then hands us a size that's too small and KIRK
// runs off the end of the buffer. That showed up as an intermittent crash unpacking any official
// updater, since whether the overrun lands on an unmapped page depends on the heap layout.
static bool KirkOutputFits(u32 headerSize, u32 dataOffset, u32 dataSize, u32 size) {
const u64 alignedDataSize = ((u64)dataSize + 15) & ~(u64)15;
return (u64)headerSize + (u64)dataOffset + alignedDataSize <= (u64)size;
}
static bool KirkOutputFits(const KIRK_CMD1_HEADER *header, u32 size) {
return KirkOutputFits(sizeof(KIRK_CMD1_HEADER), header->data_offset, header->data_size, size);
}
static bool KirkOutputFits(const KIRK_CMD1_ECDSA_HEADER *header, u32 size) {
return KirkOutputFits(sizeof(KIRK_CMD1_ECDSA_HEADER), header->data_offset, header->data_size, size);
}
static int pspDecryptType0(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 size) {
DEBUG_LOG(Log::Loader, "Decrypting tag %02X", (u32)*(u32_le *)&inbuf[0xD0]);
const auto decryptSize = *(s32_le*)&inbuf[0xB0];
@@ -830,6 +851,11 @@ static int pspDecryptType0(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
memcpy(reinterpret_cast<u8*>(header)+sizeof(KIRK_CMD1_HEADER), type0.prxHeader, sizeof(type0.prxHeader));
decryptKirkHeaderType0(reinterpret_cast<u8*>(header), type0.kirkBlock, xorbuf, pti->code);
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;
@@ -885,6 +911,11 @@ static int pspDecryptType1(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
memcpy(reinterpret_cast<u8*>(header)+sizeof(KIRK_CMD1_HEADER), type1.prxHeader, sizeof(type1.prxHeader));
decryptKirkHeaderType0(reinterpret_cast<u8*>(header), type1.kirkBlock, xorbuf, pti->code);
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;
@@ -949,6 +980,11 @@ static int pspDecryptType2(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
decryptKirkHeader(reinterpret_cast<u8*>(header), type2.kirkHeader, xorbuf.cbegin()+0x10, pti->code);
header->mode = 1;
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;
@@ -1017,6 +1053,11 @@ static int pspDecryptType5(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
decryptKirkHeader(reinterpret_cast<u8*>(header), type5.kirkHeader, xorbuf.cbegin()+0x10, pti->code);
header->mode = 1;
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;
@@ -1084,6 +1125,11 @@ static int pspDecryptType6(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
header->mode = 1;
header->ecdsa_hash = 1;
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;
@@ -1156,6 +1202,11 @@ static int pspDecryptType9(KirkState *kirk, const u8 *inbuf, u8 *outbuf, u32 siz
// branch only writes the mode word and zeroes the rest of that region.
header->ecdsa_hash = 0;
if (!KirkOutputFits(header, size))
{
return -5;
}
if (kirk_sceUtilsBufferCopyWithRange(kirk, outbuf, size, reinterpret_cast<u8*>(header), size - offset, KIRK_CMD_DECRYPT_PRIVATE) != 0)
{
return -4;