From 75cdc11fdb90b324a2d4cf1115821112c387a1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 22 Mar 2020 16:44:19 +0100 Subject: [PATCH 1/5] Text input: Clearly use 16-bit chars like the PSP, don't rely on wchar_t. --- Common/ChunkFile.cpp | 13 ++++ Common/ChunkFile.h | 3 +- Common/Log.h | 4 +- Core/Dialog/PSPOskDialog.cpp | 130 ++++++++++++++++++---------------- Core/Dialog/PSPOskDialog.h | 8 +-- ext/native/util/text/utf8.cpp | 22 +++--- ext/native/util/text/utf8.h | 12 ++-- 7 files changed, 105 insertions(+), 87 deletions(-) diff --git a/Common/ChunkFile.cpp b/Common/ChunkFile.cpp index 22327a7575..8334e9b5a7 100644 --- a/Common/ChunkFile.cpp +++ b/Common/ChunkFile.cpp @@ -117,6 +117,19 @@ void PointerWrap::Do(std::wstring &x) { (*ptr) += stringLen; } +void PointerWrap::Do(std::u16string &x) { + int stringLen = sizeof(char16_t) * ((int)x.length() + 1); + Do(stringLen); + + switch (mode) { + case MODE_READ: x = (char16_t*)*ptr; break; + case MODE_WRITE: memcpy(*ptr, x.c_str(), stringLen); break; + case MODE_MEASURE: break; + case MODE_VERIFY: _dbg_assert_msg_(COMMON, x == (char16_t*)*ptr, "Savestate verification failure: (at %p).\n", x.c_str()); break; + } + (*ptr) += stringLen; +} + struct standard_tm { int tm_sec; int tm_min; diff --git a/Common/ChunkFile.h b/Common/ChunkFile.h index dba69b0da0..42d5b38675 100644 --- a/Common/ChunkFile.h +++ b/Common/ChunkFile.h @@ -459,7 +459,8 @@ public: // Store strings. void Do(std::string &x); - void Do(std::wstring &x); + void Do(std::wstring &x); // DEPRECATED, do not save wstrings + void Do(std::u16string &x); void Do(tm &t); diff --git a/Common/Log.h b/Common/Log.h index 76f867923e..2f8f09794a 100644 --- a/Common/Log.h +++ b/Common/Log.h @@ -122,7 +122,7 @@ void AndroidAssertLog(const char *func, const char *file, int line, const char * #if MAX_LOGLEVEL >= DEBUG_LEVEL #define _dbg_assert_(_t_, _a_) \ if (!(_a_)) {\ - ERROR_LOG(_t_, "Error...\n\n Line: %d\n File: %s\n\nIgnore and continue?", \ + ERROR_LOG(_t_, #_a_ ## "\n\nError...\n\n Line: %d\n File: %s\n\nIgnore and continue?", \ __LINE__, __FILE__); \ if (!PanicYesNo("*** Assertion ***\n")) { Crash(); } \ } @@ -184,4 +184,4 @@ void AndroidAssertLog(const char *func, const char *file, int line, const char * Crash(); \ } -#endif // __ANDROID__ \ No newline at end of file +#endif // __ANDROID__ diff --git a/Core/Dialog/PSPOskDialog.cpp b/Core/Dialog/PSPOskDialog.cpp index b97eae0eb7..d83af0ea62 100755 --- a/Core/Dialog/PSPOskDialog.cpp +++ b/Core/Dialog/PSPOskDialog.cpp @@ -71,73 +71,73 @@ const int kor_lconsCom[] = {18,0,2,21,3,4,26,3,5,0,7,8,15,7,9,16,7,10,18,7,11,24 // Korean (Hangul) last consonant Separation key const int kor_lconsSpr[] = {2,1,9,4,4,12,5,4,18,8,8,0,9,8,6,10,8,7,11,8,9,12,8,16,13,8,17,14,8,18,17,17,9}; -static const wchar_t oskKeys[OSK_KEYBOARD_COUNT][5][14] = +static const char16_t oskKeys[OSK_KEYBOARD_COUNT][5][14] = { { // Latin Lowercase - {L"1234567890-+"}, - {L"qwertyuiop[]"}, - {L"asdfghjkl;@~"}, - {L"zxcvbnm,./?\\"}, + {u"1234567890-+"}, + {u"qwertyuiop[]"}, + {u"asdfghjkl;@~"}, + {u"zxcvbnm,./?\\"}, }, { // Latin Uppercase - {L"!@#$%^&*()_+"}, - {L"QWERTYUIOP{}"}, - {L"ASDFGHJKL:\"`"}, - {L"ZXCVBNM<>/?|"}, + {u"!@#$%^&*()_+"}, + {u"QWERTYUIOP{}"}, + {u"ASDFGHJKL:\"`"}, + {u"ZXCVBNM<>/?|"}, }, { // Hiragana - {L"あかさたなはまやらわぁゃっ"}, - {L"いきしちにひみ　り　ぃ　　"}, - {L"うくすつぬふむゆるをぅゅ゛"}, - {L"えけせてねへめ　れ　ぇ　゜"}, - {L"おこそとのほもよろんぉょー"}, + {u"あかさたなはまやらわぁゃっ"}, + {u"いきしちにひみ　り　ぃ　　"}, + {u"うくすつぬふむゆるをぅゅ゛"}, + {u"えけせてねへめ　れ　ぇ　゜"}, + {u"おこそとのほもよろんぉょー"}, }, { // Katakana - {L"アカサタナハマヤラワァャッ"}, - {L"イキシチニヒミ　リ　ィ　　"}, - {L"ウクスツヌフムユルヲゥュ゛"}, - {L"エケセテネヘメ　レ　ェ　゜"}, - {L"オコソトノホモヨロンォョー"}, + {u"アカサタナハマヤラワァャッ"}, + {u"イキシチニヒミ　リ　ィ　　"}, + {u"ウクスツヌフムユルヲゥュ゛"}, + {u"エケセテネヘメ　レ　ェ　゜"}, + {u"オコソトノホモヨロンォョー"}, }, { // Korean(Hangul) - {L"1234567890-+"}, - {L"ㅃㅉㄸㄲㅆ!@#$%^&"}, - {L"ㅂㅈㄷㄱㅅㅛㅕㅑㅐㅔ[]"}, - {L"ㅁㄴㅇㄹㅎㅗㅓㅏㅣ;@~"}, - {L"ㅋㅌㅊㅍㅠㅜㅡ<>/?|"}, + {u"1234567890-+"}, + {u"ㅃㅉㄸㄲㅆ!@#$%^&"}, + {u"ㅂㅈㄷㄱㅅㅛㅕㅑㅐㅔ[]"}, + {u"ㅁㄴㅇㄹㅎㅗㅓㅏㅣ;@~"}, + {u"ㅋㅌㅊㅍㅠㅜㅡ<>/?|"}, }, { // Russian Lowercase - {L"1234567890-+"}, - {L"йцукенгшщзхъ"}, - {L"фывапролджэё"}, - {L"ячсмитьбю/?|"}, + {u"1234567890-+"}, + {u"йцукенгшщзхъ"}, + {u"фывапролджэё"}, + {u"ячсмитьбю/?|"}, }, { // Russian Uppercase - {L"!@#$%^&*()_+"}, - {L"ЙЦУКЕНГШЩЗХЪ"}, - {L"ФЫВАПРОЛДЖЭЁ"}, - {L"ЯЧСМИТЬБЮ/?|"}, + {u"!@#$%^&*()_+"}, + {u"ЙЦУКЕНГШЩЗХЪ"}, + {u"ФЫВАПРОЛДЖЭЁ"}, + {u"ЯЧСМИТЬБЮ/?|"}, }, { // Latin Full-width Lowercase - { L"１２３４５６７８９０－＋" }, - { L"ｑｗｅｒｔｙｕｉｏｐ［］" }, - { L"ａｓｄｆｇｈｊｋｌ；＠～" }, - { L"ｚｘｃｖｂｎｍ，．／？￥￥" }, + { u"１２３４５６７８９０－＋" }, + { u"ｑｗｅｒｔｙｕｉｏｐ［］" }, + { u"ａｓｄｆｇｈｊｋｌ；＠～" }, + { u"ｚｘｃｖｂｎｍ，．／？￥￥" }, }, { // Latin Full-width Uppercase - { L"！＠＃＄％＾＆＊（）＿＋" }, - { L"ＱＷＥＲＴＹＵＩＯＰ｛｝" }, - { L"ＡＳＤＦＧＨＪＫＬ：￥”‘" }, - { L"ＺＸＣＶＢＮＭ＜＞／？｜" }, + { u"！＠＃＄％＾＆＊（）＿＋" }, + { u"ＱＷＥＲＴＹＵＩＯＰ｛｝" }, + { u"ＡＳＤＦＧＨＪＫＬ：￥”‘" }, + { u"ＺＸＣＶＢＮＭ＜＞／？｜" }, }, }; @@ -167,7 +167,7 @@ void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const PSPPointer& em_address) +void GetWideStringFromPSPPointer(std::u16string& _string, const PSPPointer& em_address) { if (!em_address.IsValid()) { - _string = L""; + _string.clear(); return; } const size_t maxLength = 2047; - wchar_t stringBuffer[maxLength + 1]; - wchar_t *string = stringBuffer; + char16_t stringBuffer[maxLength + 1]; + char16_t *string = stringBuffer; u16_le *input = &em_address[0]; int c; @@ -214,7 +214,7 @@ void GetWideStringFromPSPPointer(std::wstring& _string, const PSPPointer _string = stringBuffer; } -void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const wchar_t *input) +void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const char16_t *input) { char stringBuffer[2048]; char *string = stringBuffer; @@ -315,7 +315,7 @@ int PSPOskDialog::Init(u32 oskPtr) { i_level = 0; - inputChars = L""; + inputChars.clear(); if (oskParams->fields[0].intext.IsValid()) { auto src = oskParams->fields[0].intext; @@ -336,9 +336,9 @@ int PSPOskDialog::Init(u32 oskPtr) { return 0; } -std::wstring PSPOskDialog::CombinationKorean(bool isInput) +std::u16string PSPOskDialog::CombinationKorean(bool isInput) { - std::wstring string; + std::u16string string; isCombinated = true; @@ -578,9 +578,9 @@ std::wstring PSPOskDialog::CombinationKorean(bool isInput) return string; } -std::wstring PSPOskDialog::CombinationString(bool isInput) +std::u16string PSPOskDialog::CombinationString(bool isInput) { - std::wstring string; + std::u16string string; isCombinated = false; @@ -752,7 +752,7 @@ void PSPOskDialog::RenderKeyboard() int selectedRow = selectedChar / numKeyCols[currentKeyboard]; int selectedCol = selectedChar % numKeyCols[currentKeyboard]; - wchar_t temp[2]; + char16_t temp[2]; temp[1] = '\0'; std::string buffer; @@ -769,7 +769,7 @@ void PSPOskDialog::RenderKeyboard() PPGeDrawText(oskDesc.c_str(), title , 20, PPGE_ALIGN_CENTER, 0.5f, CalcFadedColor(0xFFFFFFFF)); - std::wstring result; + std::u16string result; result = CombinationString(false); @@ -855,16 +855,16 @@ int PSPOskDialog::NativeKeyboard() { } if (beginInputBox) { - std::wstring titleText; + std::u16string titleText; GetWideStringFromPSPPointer(titleText, oskParams->fields[0].desc); - std::wstring defaultText; + std::u16string defaultText; GetWideStringFromPSPPointer(defaultText, oskParams->fields[0].intext); if (defaultText.empty()) - defaultText.assign(L"VALUE"); + defaultText.assign(u"VALUE"); - System_InputBoxGetString(ConvertWStringToUTF8(titleText), ConvertWStringToUTF8(defaultText), [&](bool result, const std::string &value) { + System_InputBoxGetString(ConvertUCS16ToUTF8(titleText), ConvertUCS16ToUTF8(defaultText), [&](bool result, const std::string &value) { std::lock_guard guard(nativeMutex_); if (nativeStatus_ != PSPOskNativeStatus::WAITING) { return; @@ -874,7 +874,7 @@ int PSPOskDialog::NativeKeyboard() { nativeStatus_ = result ? PSPOskNativeStatus::SUCCESS : PSPOskNativeStatus::FAILURE; }); } else if (nativeStatus_ == PSPOskNativeStatus::SUCCESS) { - inputChars = ConvertUTF8ToWString(nativeValue_); + inputChars = ConvertUTF8ToUCS16(nativeValue_); nativeValue_.clear(); u32 maxLength = FieldMaxLength(); @@ -1080,9 +1080,9 @@ int PSPOskDialog::Update(int animSpeed) { } else if (IsButtonPressed(CTRL_SQUARE) && inputChars.size() < FieldMaxLength()) { // Use a regular space if the current keyboard isn't Japanese nor full-width English if (currentKeyboardLanguage != OSK_LANGUAGE_JAPANESE && currentKeyboardLanguage != OSK_LANGUAGE_ENGLISH_FW) - inputChars += L" "; + inputChars += u" "; else - inputChars += L"　"; + inputChars += u"　"; } EndDraw(); @@ -1123,7 +1123,7 @@ void PSPOskDialog::DoState(PointerWrap &p) { PSPDialog::DoState(p); - auto s = p.Section("PSPOskDialog", 1); + auto s = p.Section("PSPOskDialog", 2); if (!s) return; @@ -1132,7 +1132,13 @@ void PSPOskDialog::DoState(PointerWrap &p) p.Do(oskIntext); p.Do(oskOuttext); p.Do(selectedChar); - p.Do(inputChars); + if (s >= 2) { + p.Do(inputChars); + } else { + // Discard the wstring. + std::wstring wstr; + p.Do(wstr); + } // Don't need to save state native status or value. } diff --git a/Core/Dialog/PSPOskDialog.h b/Core/Dialog/PSPOskDialog.h index fc57bcf2e4..a244fd419a 100644 --- a/Core/Dialog/PSPOskDialog.h +++ b/Core/Dialog/PSPOskDialog.h @@ -229,12 +229,12 @@ protected: private: void ConvertUCS2ToUTF8(std::string& _string, const PSPPointer& em_address); - void ConvertUCS2ToUTF8(std::string& _string, const wchar_t *input); + void ConvertUCS2ToUTF8(std::string& _string, const char16_t *input); void RenderKeyboard(); int NativeKeyboard(); - std::wstring CombinationString(bool isInput); // for Japanese, Korean - std::wstring CombinationKorean(bool isInput); // for Korea + std::u16string CombinationString(bool isInput); // for Japanese, Korean + std::u16string CombinationKorean(bool isInput); // for Korea void RemoveKorean(); // for Korean character removal u32 FieldMaxLength(); @@ -246,7 +246,7 @@ private: std::string oskOuttext; int selectedChar; - std::wstring inputChars; + std::u16string inputChars; OskKeyboardDisplay currentKeyboard; OskKeyboardLanguage currentKeyboardLanguage; bool isCombinated; diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index 4183463d46..b3d3aceb1a 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -461,9 +461,9 @@ std::wstring ConvertUTF8ToWString(const std::string &source) { return str; } -#else +#endif -std::string ConvertWStringToUTF8(const std::wstring &wstr) { +std::string ConvertUCS16ToUTF8(const std::u16string &wstr) { std::string s; // Worst case. s.resize(wstr.size() * 4); @@ -477,9 +477,9 @@ std::string ConvertWStringToUTF8(const std::wstring &wstr) { return s; } -static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const std::string &source) { - const wchar_t *const orig = dest; - const wchar_t *const destEnd = dest + destSize; +static size_t ConvertUTF8ToUCS16Internal(char16_t *dest, size_t destSize, const std::string &source) { + const char16_t *const orig = dest; + const char16_t *const destEnd = dest + destSize; UTF8 utf(source.c_str()); @@ -509,17 +509,15 @@ static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const return dest - orig; } -void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source) { - ConvertUTF8ToWStringInternal(dest, destSize, source); +void ConvertUTF8ToUCS16(char16_t *dest, size_t destSize, const std::string &source) { + ConvertUTF8ToUCS16Internal(dest, destSize, source); } -std::wstring ConvertUTF8ToWString(const std::string &source) { - std::wstring dst; +std::u16string ConvertUTF8ToUCS16(const std::string &source) { + std::u16string dst; // utf-8 won't be less bytes than there are characters. But need +1 for terminator. dst.resize(source.size() + 1, 0); - size_t realLen = ConvertUTF8ToWStringInternal(&dst[0], source.size() + 1, source); + size_t realLen = ConvertUTF8ToUCS16Internal(&dst[0], source.size() + 1, source); dst.resize(realLen); return dst; } - -#endif diff --git a/ext/native/util/text/utf8.h b/ext/native/util/text/utf8.h index 6303c25b45..3c3b47cc7d 100644 --- a/ext/native/util/text/utf8.h +++ b/ext/native/util/text/utf8.h @@ -82,13 +82,13 @@ bool UTF8StringHasNonASCII(const char *utf8string); std::string ConvertWStringToUTF8(const std::wstring &wstr); std::string ConvertWStringToUTF8(const wchar_t *wstr); - -#else - -std::string ConvertWStringToUTF8(const std::wstring &wstr); +void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source); +std::wstring ConvertUTF8ToWString(const std::string &source); #endif +std::string ConvertUCS16ToUTF8(const std::u16string &wstr); + // Dest size in units, not bytes. -void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source); -std::wstring ConvertUTF8ToWString(const std::string &source); +void ConvertUTF8ToUCS16(char16_t *dest, size_t destSize, const std::string &source); +std::u16string ConvertUTF8ToUCS16(const std::string &source); From d1196f2fc006378aa1343259f88309805e227362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 22 Mar 2020 17:03:49 +0100 Subject: [PATCH 2/5] Support a few more wstring functions on non-windows platforms, due to the armips assembler. --- Core/Debugger/SymbolMap.cpp | 2 + Core/Dialog/PSPOskDialog.cpp | 5 ++- ext/native/util/text/utf8.cpp | 73 ++++++++++++++++++++++++++++++----- ext/native/util/text/utf8.h | 12 ++++-- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Core/Debugger/SymbolMap.cpp b/Core/Debugger/SymbolMap.cpp index 5786898624..c0f48f14ff 100644 --- a/Core/Debugger/SymbolMap.cpp +++ b/Core/Debugger/SymbolMap.cpp @@ -1003,6 +1003,7 @@ void SymbolMap::GetLabels(std::vector &dest) { if (activeNeedUpdate_) UpdateActiveSymbols(); +#ifdef _WIN32 std::lock_guard guard(lock_); for (auto it = activeLabels.begin(); it != activeLabels.end(); it++) { LabelDefinition entry; @@ -1010,6 +1011,7 @@ void SymbolMap::GetLabels(std::vector &dest) { entry.name = ConvertUTF8ToWString(it->second.name); dest.push_back(entry); } +#endif } #if defined(_WIN32) && !PPSSPP_PLATFORM(UWP) diff --git a/Core/Dialog/PSPOskDialog.cpp b/Core/Dialog/PSPOskDialog.cpp index d83af0ea62..3625526088 100755 --- a/Core/Dialog/PSPOskDialog.cpp +++ b/Core/Dialog/PSPOskDialog.cpp @@ -864,7 +864,8 @@ int PSPOskDialog::NativeKeyboard() { if (defaultText.empty()) defaultText.assign(u"VALUE"); - System_InputBoxGetString(ConvertUCS16ToUTF8(titleText), ConvertUCS16ToUTF8(defaultText), [&](bool result, const std::string &value) { + // There's already ConvertUCS2ToUTF8 in this file. Should we use that instead of the global ones? + System_InputBoxGetString(::ConvertUCS2ToUTF8(titleText), ::ConvertUCS2ToUTF8(defaultText), [&](bool result, const std::string &value) { std::lock_guard guard(nativeMutex_); if (nativeStatus_ != PSPOskNativeStatus::WAITING) { return; @@ -874,7 +875,7 @@ int PSPOskDialog::NativeKeyboard() { nativeStatus_ = result ? PSPOskNativeStatus::SUCCESS : PSPOskNativeStatus::FAILURE; }); } else if (nativeStatus_ == PSPOskNativeStatus::SUCCESS) { - inputChars = ConvertUTF8ToUCS16(nativeValue_); + inputChars = ConvertUTF8ToUCS2(nativeValue_); nativeValue_.clear(); u32 maxLength = FieldMaxLength(); diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index b3d3aceb1a..f1e6efc4cd 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -463,7 +463,7 @@ std::wstring ConvertUTF8ToWString(const std::string &source) { #endif -std::string ConvertUCS16ToUTF8(const std::u16string &wstr) { +std::string ConvertUCS2ToUTF8(const std::u16string &wstr) { std::string s; // Worst case. s.resize(wstr.size() * 4); @@ -477,12 +477,69 @@ std::string ConvertUCS16ToUTF8(const std::u16string &wstr) { return s; } -static size_t ConvertUTF8ToUCS16Internal(char16_t *dest, size_t destSize, const std::string &source) { +static size_t ConvertUTF8ToUCS2Internal(char16_t *dest, size_t destSize, const std::string &source) { const char16_t *const orig = dest; const char16_t *const destEnd = dest + destSize; UTF8 utf(source.c_str()); + char16_t *destw = (char16_t *)dest; + const char16_t *const destwEnd = destw + destSize; + + // TODO: Instead of encoding UTF16 here we should encode UCS2. + while (uint32_t c = utf.next()) { + if (destw + UTF16LE::encodeUnits(c) >= destwEnd) { + break; + } + // TODO: Update UTF16LE to take uint16_t + destw += UTF16LE::encode((uint16_t *)destw, c); + } + + // No ++ to not count the terminal in length. + if (dest < destEnd) { + *dest = 0; + } + + return dest - orig; +} + +void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, const std::string &source) { + ConvertUTF8ToUCS2Internal(dest, destSize, source); +} + +std::u16string ConvertUTF8ToUCS2(const std::string &source) { + std::u16string dst; + // utf-8 won't be less bytes than there are characters. But need +1 for terminator. + dst.resize(source.size() + 1, 0); + size_t realLen = ConvertUTF8ToUCS2Internal(&dst[0], source.size() + 1, source); + dst.resize(realLen); + return dst; +} + +#ifndef _WIN32 + +// Replacements for the Win32 functions. Not to be used from emulation code! + +std::string ConvertWStringToUTF8(const std::wstring &wstr) { + std::string s; + // Worst case. + s.resize(wstr.size() * 4); + + size_t pos = 0; + for (wchar_t c : wstr) { + pos += UTF8::encode(&s[pos], c); + } + + s.resize(pos); + return s; +} + +static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const std::string &source) { + const wchar_t *const orig = dest; + const wchar_t *const destEnd = dest + destSize; + + UTF8 utf(source.c_str()); + if (sizeof(wchar_t) == 2) { uint16_t *destw = (uint16_t *)dest; const uint16_t *const destwEnd = destw + destSize; @@ -509,15 +566,13 @@ static size_t ConvertUTF8ToUCS16Internal(char16_t *dest, size_t destSize, const return dest - orig; } -void ConvertUTF8ToUCS16(char16_t *dest, size_t destSize, const std::string &source) { - ConvertUTF8ToUCS16Internal(dest, destSize, source); -} - -std::u16string ConvertUTF8ToUCS16(const std::string &source) { - std::u16string dst; +std::wstring ConvertUTF8ToWString(const std::string &source) { + std::wstring dst; // utf-8 won't be less bytes than there are characters. But need +1 for terminator. dst.resize(source.size() + 1, 0); - size_t realLen = ConvertUTF8ToUCS16Internal(&dst[0], source.size() + 1, source); + size_t realLen = ConvertUTF8ToWStringInternal(&dst[0], source.size() + 1, source); dst.resize(realLen); return dst; } + +#endif diff --git a/ext/native/util/text/utf8.h b/ext/native/util/text/utf8.h index 3c3b47cc7d..fd6ddf64e7 100644 --- a/ext/native/util/text/utf8.h +++ b/ext/native/util/text/utf8.h @@ -85,10 +85,16 @@ std::string ConvertWStringToUTF8(const wchar_t *wstr); void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source); std::wstring ConvertUTF8ToWString(const std::string &source); +#else + +// Used by SymbolMap/assembler +std::wstring ConvertUTF8ToWString(const std::string &source); +std::string ConvertWStringToUTF8(const std::wstring &wstr); + #endif -std::string ConvertUCS16ToUTF8(const std::u16string &wstr); +std::string ConvertUCS2ToUTF8(const std::u16string &wstr); // Dest size in units, not bytes. -void ConvertUTF8ToUCS16(char16_t *dest, size_t destSize, const std::string &source); -std::u16string ConvertUTF8ToUCS16(const std::string &source); +void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, const std::string &source); +std::u16string ConvertUTF8ToUCS2(const std::string &source); From bb30169c882ef6ba71b9e14ba9bbe3d796e67e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 22 Mar 2020 19:20:23 +0100 Subject: [PATCH 3/5] Address feedback --- Core/Debugger/SymbolMap.cpp | 2 -- ext/native/util/text/utf8.cpp | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/Debugger/SymbolMap.cpp b/Core/Debugger/SymbolMap.cpp index c0f48f14ff..5786898624 100644 --- a/Core/Debugger/SymbolMap.cpp +++ b/Core/Debugger/SymbolMap.cpp @@ -1003,7 +1003,6 @@ void SymbolMap::GetLabels(std::vector &dest) { if (activeNeedUpdate_) UpdateActiveSymbols(); -#ifdef _WIN32 std::lock_guard guard(lock_); for (auto it = activeLabels.begin(); it != activeLabels.end(); it++) { LabelDefinition entry; @@ -1011,7 +1010,6 @@ void SymbolMap::GetLabels(std::vector &dest) { entry.name = ConvertUTF8ToWString(it->second.name); dest.push_back(entry); } -#endif } #if defined(_WIN32) && !PPSSPP_PLATFORM(UWP) diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index f1e6efc4cd..21c6a1fb7b 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -486,12 +486,13 @@ static size_t ConvertUTF8ToUCS2Internal(char16_t *dest, size_t destSize, const s char16_t *destw = (char16_t *)dest; const char16_t *const destwEnd = destw + destSize; - // TODO: Instead of encoding UTF16 here we should encode UCS2. + // TODO: Ignore characters that are invalid in UCS2 by encoding to UTF-16, as we do, and + // then failing/ignoring any 16-bit unit produced by UTF-16 in the range U+D800 U+DFFF. while (uint32_t c = utf.next()) { if (destw + UTF16LE::encodeUnits(c) >= destwEnd) { break; } - // TODO: Update UTF16LE to take uint16_t + // TODO: Update UTF16LE to take char16_t destw += UTF16LE::encode((uint16_t *)destw, c); } @@ -518,7 +519,7 @@ std::u16string ConvertUTF8ToUCS2(const std::string &source) { #ifndef _WIN32 -// Replacements for the Win32 functions. Not to be used from emulation code! +// Replacements for the Win32 wstring functions. Not to be used from emulation code! std::string ConvertWStringToUTF8(const std::wstring &wstr) { std::string s; From d6ba1b7f99831b8f285b0238d6c68a9e97364cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 22 Mar 2020 19:29:25 +0100 Subject: [PATCH 4/5] More correctly handle UCS2 encoding. --- Core/HLE/sceCcc.cpp | 4 ++-- ext/native/util/text/utf16.h | 33 +++++++++++++++++++++++++-------- ext/native/util/text/utf8.cpp | 8 +++----- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Core/HLE/sceCcc.cpp b/Core/HLE/sceCcc.cpp index 65ae665f7d..71cbfa6e40 100644 --- a/Core/HLE/sceCcc.cpp +++ b/Core/HLE/sceCcc.cpp @@ -28,9 +28,9 @@ #include "Core/Reporting.h" typedef PSPPointer PSPCharPointer; -typedef PSPPointer PSPWCharPointer; +typedef PSPPointer PSPWCharPointer; typedef PSPPointer PSPConstCharPointer; -typedef PSPPointer PSPConstWCharPointer; +typedef PSPPointer PSPConstWCharPointer; static u16 errorUTF8; static u16 errorUTF16; diff --git a/ext/native/util/text/utf16.h b/ext/native/util/text/utf16.h index 2475b7292a..830eac2c51 100644 --- a/ext/native/util/text/utf16.h +++ b/ext/native/util/text/utf16.h @@ -17,12 +17,12 @@ uint16_t UTF16_Swap(uint16_t u) { template struct UTF16_Type { public: - static const uint32_t INVALID = (uint32_t)-1; + static const char32_t INVALID = (char32_t)-1; - UTF16_Type(const uint16_t *c) : c_(c), index_(0) {} + UTF16_Type(const char16_t *c) : c_(c), index_(0) {} - uint32_t next() { - const uint32_t u = UTF16_Swap(c_[index_++]); + char32_t next() { + const char32_t u = UTF16_Swap(c_[index_++]); // Surrogate pair. UTF-16 is so simple. We assume it's valid. if ((u & 0xF800) == 0xD800) { @@ -46,19 +46,29 @@ public: return index_; } - static int encode(uint16_t *dest, uint32_t u) { + static int encode(char16_t *dest, char32_t u) { if (u >= 0x10000) { u -= 0x10000; *dest++ = UTF16_Swap(0xD800 + ((u >> 10) & 0x3FF)); *dest = UTF16_Swap(0xDC00 + ((u >> 0) & 0x3FF)); return 2; } else { - *dest = UTF16_Swap((uint16_t)u); + *dest = UTF16_Swap((char16_t)u); return 1; } } - static int encodeUnits(uint32_t u) { + // Rejects non-UCS2 codepoints. + static int encodeUCS2(char16_t *dest, char32_t u) { + if (u >= 0x10000 || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } else { + *dest = UTF16_Swap((char16_t)u); + return 1; + } + } + + static int encodeUnits(char32_t u) { if (u >= 0x10000) { return 2; } else { @@ -66,8 +76,15 @@ public: } } + static int encodeUnitsUCS2(char32_t u) { + if (u >= 0x10000 || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } else { + return 1; + } + } private: - const uint16_t *c_; + const char16_t *c_; int index_; }; diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index 21c6a1fb7b..7817ac7545 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -486,14 +486,12 @@ static size_t ConvertUTF8ToUCS2Internal(char16_t *dest, size_t destSize, const s char16_t *destw = (char16_t *)dest; const char16_t *const destwEnd = destw + destSize; - // TODO: Ignore characters that are invalid in UCS2 by encoding to UTF-16, as we do, and - // then failing/ignoring any 16-bit unit produced by UTF-16 in the range U+D800 U+DFFF. + // Ignores characters outside the BMP. while (uint32_t c = utf.next()) { - if (destw + UTF16LE::encodeUnits(c) >= destwEnd) { + if (destw + UTF16LE::encodeUnitsUCS2(c) >= destwEnd) { break; } - // TODO: Update UTF16LE to take char16_t - destw += UTF16LE::encode((uint16_t *)destw, c); + destw += UTF16LE::encodeUCS2(destw, c); } // No ++ to not count the terminal in length. From 1ad65c7c7ac95da9d1e71cf3c806eec0f9fab835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sun, 22 Mar 2020 21:28:06 +0100 Subject: [PATCH 5/5] Remove some more uint16_t --- ext/native/util/text/utf8.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index 7817ac7545..ab734231db 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -540,16 +540,16 @@ static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const UTF8 utf(source.c_str()); if (sizeof(wchar_t) == 2) { - uint16_t *destw = (uint16_t *)dest; - const uint16_t *const destwEnd = destw + destSize; - while (uint32_t c = utf.next()) { + char16_t *destw = (char16_t *)dest; + const char16_t *const destwEnd = destw + destSize; + while (char32_t c = utf.next()) { if (destw + UTF16LE::encodeUnits(c) >= destwEnd) { break; } destw += UTF16LE::encode(destw, c); } } else { - while (uint32_t c = utf.next()) { + while (char32_t c = utf.next()) { if (dest + 1 >= destEnd) { break; }