diff --git a/Common/Data/Encoding/Shiftjis.h b/Common/Data/Encoding/Shiftjis.h index 35ffb8e752..a1c2b42c01 100644 --- a/Common/Data/Encoding/Shiftjis.h +++ b/Common/Data/Encoding/Shiftjis.h @@ -33,11 +33,17 @@ struct ShiftJIS { } else if (j < 0xFD) { row = ((j & 0x7F) << 1) - 0x1B; } else { - return j; + return INVALID; } break; - // Anything else (i.e. <= 0x7x, 0xAx, 0xBx, 0xCx, and 0xDx) is JIS X 0201, return directly. + case 0xA: + if (j == 0xA0) { + return INVALID; + } + return j; + + // Anything else (i.e. <= 0x7x, 0xBx, 0xCx, and 0xDx) is JIS X 0201, return directly. default: return j; } diff --git a/Common/Data/Encoding/Utf16.h b/Common/Data/Encoding/Utf16.h index a04949ed35..539202626e 100644 --- a/Common/Data/Encoding/Utf16.h +++ b/Common/Data/Encoding/Utf16.h @@ -25,12 +25,17 @@ public: char32_t next() { const char32_t u = UTF16_Swap(c_[index_++]); - - // Surrogate pair. UTF-16 is so simple. We assume it's valid. - if ((u & 0xF800) == 0xD800) { - return 0x10000 + (((u & 0x3FF) << 10) | (UTF16_Swap(c_[index_++]) & 0x3FF)); + if ((u & 0xF800) != 0xD800) { + return u; } - return u; + + // A surrogate is only valid as a high one followed by a low one. + const char32_t low = UTF16_Swap(c_[index_]); + if ((u & 0xFC00) != 0xD800 || (low & 0xFC00) != 0xDC00) { + return INVALID; + } + index_++; + return 0x10000 + (((u & 0x3FF) << 10) | (low & 0x3FF)); } bool end() const { diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 90ec581f63..153a2b9fb1 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -58,6 +58,8 @@ #include "Common/Data/Text/Parsers.h" #include "Common/Data/Text/WrapText.h" #include "Common/Data/Encoding/Utf8.h" +#include "Common/Data/Encoding/Utf16.h" +#include "Common/Data/Encoding/Shiftjis.h" #include "Common/Buffer.h" #include "Common/File/Path.h" #include "Common/Log/LogManager.h" @@ -353,6 +355,55 @@ bool TestUtf8() { return true; } +bool TestTextDecoders() { + { + const char16_t pair[] = { 0xD83C, 0xDFAE, 0 }; + UTF16LE utf(pair); + EXPECT_EQ_HEX((uint32_t)utf.next(), 0x1F3AEu); + EXPECT_TRUE(utf.end()); + } + + // An unpaired surrogate is invalid on its own, and decoding carries on after it. + { + const char16_t highThenA[] = { 0xD800, 'A', 0 }; + UTF16LE utf(highThenA); + EXPECT_TRUE(utf.next() == UTF16LE::INVALID); + EXPECT_EQ_HEX((uint32_t)utf.next(), (uint32_t)'A'); + EXPECT_TRUE(utf.end()); + + const char16_t lowThenA[] = { 0xDC00, 'A', 0 }; + UTF16LE utf2(lowThenA); + EXPECT_TRUE(utf2.next() == UTF16LE::INVALID); + EXPECT_EQ_HEX((uint32_t)utf2.next(), (uint32_t)'A'); + } + + // A high surrogate right before the terminator must not consume it. + { + const char16_t highAtEnd[] = { 0xD800, 0, 'A', 0 }; + UTF16LE utf(highAtEnd); + EXPECT_TRUE(utf.next() == UTF16LE::INVALID); + EXPECT_EQ_INT(utf.shortIndex(), 1); + EXPECT_TRUE(utf.end()); + } + + { + ShiftJIS sjis("A\xA1\x82\xA0"); + EXPECT_EQ_HEX(sjis.next(), 0x41u); + EXPECT_EQ_HEX(sjis.next(), 0xA1u); + EXPECT_EQ_HEX(sjis.next(), 0x2422u); + EXPECT_TRUE(sjis.end()); + } + + // Lead bytes that encode nothing. + for (const char *bad : { "\x80", "\xA0", "\xFD", "\xFE", "\xFF" }) { + ShiftJIS sjis(bad); + EXPECT_TRUE(sjis.next() == ShiftJIS::INVALID); + EXPECT_EQ_INT(sjis.byteIndex(), 1); + } + + return true; +} + // PointerWrap is the savestate serializer. The same DoState() code runs in MEASURE, WRITE and READ // mode, so mistakes here don't show up as compile errors - they show up as savestates that don't // load, or worse. Everything read back came off disk and is therefore attacker-controllable, so the @@ -3550,6 +3601,7 @@ TestItem availableTests[] = { TEST_ITEM(Breakpoints), TEST_ITEM(TempBreakpoints), TEST_ITEM(Utf8), + TEST_ITEM(TextDecoders), TEST_ITEM(IRPassSimplify), TEST_ITEM(Jit), TEST_ITEM(JitDotBench),