From a42ed5e312a2a3b409c1f03cd11e60577a84b10d Mon Sep 17 00:00:00 2001 From: mariobob <13618350+mariobob@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:44:40 +0200 Subject: [PATCH] Encoding: Report bad UTF-16 surrogates and Shift-JIS lead bytes as invalid UTF16_Type::next() assumed every surrogate came in a valid pair, so a lone high surrogate swallowed the unit after it, and one right before the terminator stepped over it and kept reading. It now returns INVALID and consumes only the bad unit, which sceCccDecodeUTF16() already checks for. ShiftJIS::next() returned 0xA0 and 0xFD-0xFF as single-byte characters. Like 0x80, which it already rejected, they encode nothing (the PSP's JIS to UCS table in pspautotests/tests/ccc/jis2ucs.bin has no entry for any of them), so they're INVALID too. Only sceCcc uses these decoders. --- Common/Data/Encoding/Shiftjis.h | 10 +++++-- Common/Data/Encoding/Utf16.h | 15 ++++++---- unittest/UnitTest.cpp | 52 +++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/Common/Data/Encoding/Shiftjis.h b/Common/Data/Encoding/Shiftjis.h index 35ffb8e752..a1c2b42c01 100644 --- a/Common/Data/Encoding/Shiftjis.h +++ b/Common/Data/Encoding/Shiftjis.h @@ -33,11 +33,17 @@ struct ShiftJIS { } else if (j < 0xFD) { row = ((j & 0x7F) << 1) - 0x1B; } else { - return j; + return INVALID; } break; - // Anything else (i.e. <= 0x7x, 0xAx, 0xBx, 0xCx, and 0xDx) is JIS X 0201, return directly. + case 0xA: + if (j == 0xA0) { + return INVALID; + } + return j; + + // Anything else (i.e. <= 0x7x, 0xBx, 0xCx, and 0xDx) is JIS X 0201, return directly. default: return j; } diff --git a/Common/Data/Encoding/Utf16.h b/Common/Data/Encoding/Utf16.h index a04949ed35..539202626e 100644 --- a/Common/Data/Encoding/Utf16.h +++ b/Common/Data/Encoding/Utf16.h @@ -25,12 +25,17 @@ public: char32_t next() { const char32_t u = UTF16_Swap(c_[index_++]); - - // Surrogate pair. UTF-16 is so simple. We assume it's valid. - if ((u & 0xF800) == 0xD800) { - return 0x10000 + (((u & 0x3FF) << 10) | (UTF16_Swap(c_[index_++]) & 0x3FF)); + if ((u & 0xF800) != 0xD800) { + return u; } - return u; + + // A surrogate is only valid as a high one followed by a low one. + const char32_t low = UTF16_Swap(c_[index_]); + if ((u & 0xFC00) != 0xD800 || (low & 0xFC00) != 0xDC00) { + return INVALID; + } + index_++; + return 0x10000 + (((u & 0x3FF) << 10) | (low & 0x3FF)); } bool end() const { diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 90ec581f63..153a2b9fb1 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -58,6 +58,8 @@ #include "Common/Data/Text/Parsers.h" #include "Common/Data/Text/WrapText.h" #include "Common/Data/Encoding/Utf8.h" +#include "Common/Data/Encoding/Utf16.h" +#include "Common/Data/Encoding/Shiftjis.h" #include "Common/Buffer.h" #include "Common/File/Path.h" #include "Common/Log/LogManager.h" @@ -353,6 +355,55 @@ bool TestUtf8() { return true; } +bool TestTextDecoders() { + { + const char16_t pair[] = { 0xD83C, 0xDFAE, 0 }; + UTF16LE utf(pair); + EXPECT_EQ_HEX((uint32_t)utf.next(), 0x1F3AEu); + EXPECT_TRUE(utf.end()); + } + + // An unpaired surrogate is invalid on its own, and decoding carries on after it. + { + const char16_t highThenA[] = { 0xD800, 'A', 0 }; + UTF16LE utf(highThenA); + EXPECT_TRUE(utf.next() == UTF16LE::INVALID); + EXPECT_EQ_HEX((uint32_t)utf.next(), (uint32_t)'A'); + EXPECT_TRUE(utf.end()); + + const char16_t lowThenA[] = { 0xDC00, 'A', 0 }; + UTF16LE utf2(lowThenA); + EXPECT_TRUE(utf2.next() == UTF16LE::INVALID); + EXPECT_EQ_HEX((uint32_t)utf2.next(), (uint32_t)'A'); + } + + // A high surrogate right before the terminator must not consume it. + { + const char16_t highAtEnd[] = { 0xD800, 0, 'A', 0 }; + UTF16LE utf(highAtEnd); + EXPECT_TRUE(utf.next() == UTF16LE::INVALID); + EXPECT_EQ_INT(utf.shortIndex(), 1); + EXPECT_TRUE(utf.end()); + } + + { + ShiftJIS sjis("A\xA1\x82\xA0"); + EXPECT_EQ_HEX(sjis.next(), 0x41u); + EXPECT_EQ_HEX(sjis.next(), 0xA1u); + EXPECT_EQ_HEX(sjis.next(), 0x2422u); + EXPECT_TRUE(sjis.end()); + } + + // Lead bytes that encode nothing. + for (const char *bad : { "\x80", "\xA0", "\xFD", "\xFE", "\xFF" }) { + ShiftJIS sjis(bad); + EXPECT_TRUE(sjis.next() == ShiftJIS::INVALID); + EXPECT_EQ_INT(sjis.byteIndex(), 1); + } + + return true; +} + // PointerWrap is the savestate serializer. The same DoState() code runs in MEASURE, WRITE and READ // mode, so mistakes here don't show up as compile errors - they show up as savestates that don't // load, or worse. Everything read back came off disk and is therefore attacker-controllable, so the @@ -3550,6 +3601,7 @@ TestItem availableTests[] = { TEST_ITEM(Breakpoints), TEST_ITEM(TempBreakpoints), TEST_ITEM(Utf8), + TEST_ITEM(TextDecoders), TEST_ITEM(IRPassSimplify), TEST_ITEM(Jit), TEST_ITEM(JitDotBench),