From 6f7de71ed078e25743a6df023201b764b3d453ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Fri, 13 Mar 2026 10:52:49 +0100 Subject: [PATCH] Fix bugs in some UTF8 string handling utility functions --- Common/Data/Encoding/Utf8.cpp | 44 ++++++++++++------------------- Common/Data/Encoding/Utf8.h | 5 ---- unittest/UnitTest.cpp | 49 +++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/Common/Data/Encoding/Utf8.cpp b/Common/Data/Encoding/Utf8.cpp index 737b049bcd..f32c829697 100644 --- a/Common/Data/Encoding/Utf8.cpp +++ b/Common/Data/Encoding/Utf8.cpp @@ -113,35 +113,17 @@ uint32_t u8_nextchar(const char *s, int *index, size_t size) { do { ch = (ch << 6) + (unsigned char)s[i++]; sz++; + // Prevent reading past the offsetsFromUTF8 array (max valid UTF-8 is 4 bytes, array has 6 elements) + if (sz >= 6) { + break; + } } while (i < size && s[i] && ((s[i]) & 0xC0) == 0x80); *index = i; + // Clamp sz to valid range + if (sz > 6) sz = 6; return ch - offsetsFromUTF8[sz - 1]; } -uint32_t u8_nextchar_unsafe(const char *s, int *i) { - uint32_t ch = (unsigned char)s[(*i)++]; - int sz = 1; - if (ch >= 0xF0) { - sz++; - ch &= ~0x10; - } - if (ch >= 0xE0) { - sz++; - ch &= ~0x20; - } - if (ch >= 0xC0) { - sz++; - ch &= ~0xC0; - } - - // Just assume the bytes must be there. This is the logic used on the PSP. - for (int j = 1; j < sz; ++j) { - ch <<= 6; - ch += ((unsigned char)s[(*i)++]) & 0x3F; - } - return ch; -} - void u8_inc(const char *s, int *i) { (void)(isutf(s[++(*i)]) || isutf(s[++(*i)]) || isutf(s[++(*i)]) || ++(*i)); @@ -203,11 +185,13 @@ std::string ConvertWStringToUTF8(const std::wstring &wstr) { } void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, std::string_view source) { + if (destSize == 0) return; int len = (int)source.size(); destSize -= 1; // account for the \0. int size = (int)MultiByteToWideChar(CP_UTF8, 0, source.data(), len, NULL, 0); - MultiByteToWideChar(CP_UTF8, 0, source.data(), len, dest, std::min((int)destSize, size)); - dest[size] = 0; + int actualSize = std::min((int)destSize, size); + MultiByteToWideChar(CP_UTF8, 0, source.data(), len, dest, actualSize); + dest[actualSize] = 0; // Write null terminator at the correct position } std::wstring ConvertUTF8ToWString(const std::string_view source) { @@ -240,13 +224,18 @@ std::string ConvertUCS2ToUTF8(const std::u16string &wstr) { std::string SanitizeUTF8(std::string_view utf8string) { UTF8 utf(utf8string); std::string s; + // Check for overflow + if (utf8string.size() > SIZE_MAX / 4) { + ERROR_LOG(Log::Common, "SanitizeUTF8: Input too large"); + return std::string(); + } // Worst case. s.resize(utf8string.size() * 4); // This stops at invalid start bytes. size_t pos = 0; while (!utf.end() && !utf.invalid()) { - int c = utf.next_unsafe(); + int c = utf.next(); pos += UTF8::encode(&s[pos], c); } s.resize(pos); @@ -371,7 +360,6 @@ void ConvertUTF8ToJavaModifiedUTF8(std::string *output, std::string_view input) } } output->resize(out_idx); - _dbg_assert_(output->size() >= input.size()); } std::string NormalizeForSearch(std::string_view input) { diff --git a/Common/Data/Encoding/Utf8.h b/Common/Data/Encoding/Utf8.h index 9a752f8c70..5ad49ed162 100644 --- a/Common/Data/Encoding/Utf8.h +++ b/Common/Data/Encoding/Utf8.h @@ -22,7 +22,6 @@ #include uint32_t u8_nextchar(const char *s, int *i, size_t size); -uint32_t u8_nextchar_unsafe(const char *s, int *i); int u8_wc_toutf8(char *dest, uint32_t ch); void u8_inc(const char *s, int *i); void u8_dec(const char *s, int *i); @@ -52,10 +51,6 @@ public: uint32_t next() { return u8_nextchar(c_, &index_, size_); } - // Allow invalid continuation bytes. - uint32_t next_unsafe() { - return u8_nextchar_unsafe(c_, &index_); - } uint32_t peek() const { int tempIndex = index_; return u8_nextchar(c_, &tempIndex, size_); diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 3edcd22eb5..4b77ccc20b 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -417,6 +417,54 @@ bool TestTruncateCpy() { len = truncate_cat(catBuf, sizeof(catBuf), "", 0, "", 0); EXPECT_EQ_INT((int)len, 0); EXPECT_EQ_INT((int)catBuf[0], 0); + return true; +} + +bool TestUtf8() { + // Valid multi-byte UTF-8 (ASCII + 2-byte 'é' + 3-byte '€') round-trips unchanged. + const std::string valid = "abc \xC3\xA9 \xE2\x82\xAC"; + EXPECT_TRUE(SanitizeUTF8(valid) == valid); + + // u8_nextchar must stop at the end of the buffer instead of reading past a + // truncated multi-byte sequence (a lead byte with no continuation bytes). + { + std::string s = "abc"; + s += (char)0xF4; + int index = 3; + int size = (int)s.size(); + uint32_t c = u8_nextchar(s.data(), &index, size); + EXPECT_EQ_INT(index, size); + EXPECT_EQ_INT((int)c, 0xF4); + } + + // A long run of stray continuation bytes must not walk off the end of the + // internal offsetsFromUTF8 table (used to read arbitrarily far out of bounds). + { + std::string s(32, (char)0x80); + int index = 0; + int size = (int)s.size(); + uint32_t c = u8_nextchar(s.data(), &index, size); + EXPECT_TRUE(index > 0 && index <= size); + } + + // SanitizeUTF8 on a string that ends mid-sequence must not read or write past + // the buffer, and must preserve the well-formed leading portion. + { + std::string truncated = "abc"; + truncated += (char)0xF4; + std::string sanitized = SanitizeUTF8(truncated); + EXPECT_TRUE(sanitized.substr(0, 3) == "abc"); + } + + // ConvertUTF8ToJavaModifiedUTF8 must simply drop an incomplete trailing + // sequence rather than asserting or crashing. + { + std::string input = "abc"; + input += (char)0xF0; + std::string output; + ConvertUTF8ToJavaModifiedUTF8(&output, input); + EXPECT_TRUE(output == "abc"); + } return true; } @@ -1518,6 +1566,7 @@ TestItem availableTests[] = { TEST_ITEM(Parsers), TEST_ITEM(TruncateCpy), TEST_ITEM(MemBlockInfoSaveState), + TEST_ITEM(Utf8), TEST_ITEM(IRPassSimplify), TEST_ITEM(Jit), TEST_ITEM(VFPUMatrixTranspose),