diff --git a/Common/ChunkFile.cpp b/Common/ChunkFile.cpp index 22327a7575..8334e9b5a7 100644 --- a/Common/ChunkFile.cpp +++ b/Common/ChunkFile.cpp @@ -117,6 +117,19 @@ void PointerWrap::Do(std::wstring &x) { (*ptr) += stringLen; } +void PointerWrap::Do(std::u16string &x) { + int stringLen = sizeof(char16_t) * ((int)x.length() + 1); + Do(stringLen); + + switch (mode) { + case MODE_READ: x = (char16_t*)*ptr; break; + case MODE_WRITE: memcpy(*ptr, x.c_str(), stringLen); break; + case MODE_MEASURE: break; + case MODE_VERIFY: _dbg_assert_msg_(COMMON, x == (char16_t*)*ptr, "Savestate verification failure: (at %p).\n", x.c_str()); break; + } + (*ptr) += stringLen; +} + struct standard_tm { int tm_sec; int tm_min; diff --git a/Common/ChunkFile.h b/Common/ChunkFile.h index dba69b0da0..42d5b38675 100644 --- a/Common/ChunkFile.h +++ b/Common/ChunkFile.h @@ -459,7 +459,8 @@ public: // Store strings. void Do(std::string &x); - void Do(std::wstring &x); + void Do(std::wstring &x); // DEPRECATED, do not save wstrings + void Do(std::u16string &x); void Do(tm &t); diff --git a/Common/Log.h b/Common/Log.h index 76f867923e..2f8f09794a 100644 --- a/Common/Log.h +++ b/Common/Log.h @@ -122,7 +122,7 @@ void AndroidAssertLog(const char *func, const char *file, int line, const char * #if MAX_LOGLEVEL >= DEBUG_LEVEL #define _dbg_assert_(_t_, _a_) \ if (!(_a_)) {\ - ERROR_LOG(_t_, "Error...\n\n Line: %d\n File: %s\n\nIgnore and continue?", \ + ERROR_LOG(_t_, #_a_ ## "\n\nError...\n\n Line: %d\n File: %s\n\nIgnore and continue?", \ __LINE__, __FILE__); \ if (!PanicYesNo("*** Assertion ***\n")) { Crash(); } \ } @@ -184,4 +184,4 @@ void AndroidAssertLog(const char *func, const char *file, int line, const char * Crash(); \ } -#endif // __ANDROID__ \ No newline at end of file +#endif // __ANDROID__ diff --git a/Core/Dialog/PSPOskDialog.cpp b/Core/Dialog/PSPOskDialog.cpp index b97eae0eb7..3625526088 100755 --- a/Core/Dialog/PSPOskDialog.cpp +++ b/Core/Dialog/PSPOskDialog.cpp @@ -71,73 +71,73 @@ const int kor_lconsCom[] = {18,0,2,21,3,4,26,3,5,0,7,8,15,7,9,16,7,10,18,7,11,24 // Korean (Hangul) last consonant Separation key const int kor_lconsSpr[] = {2,1,9,4,4,12,5,4,18,8,8,0,9,8,6,10,8,7,11,8,9,12,8,16,13,8,17,14,8,18,17,17,9}; -static const wchar_t oskKeys[OSK_KEYBOARD_COUNT][5][14] = +static const char16_t oskKeys[OSK_KEYBOARD_COUNT][5][14] = { { // Latin Lowercase - {L"1234567890-+"}, - {L"qwertyuiop[]"}, - {L"asdfghjkl;@~"}, - {L"zxcvbnm,./?\\"}, + {u"1234567890-+"}, + {u"qwertyuiop[]"}, + {u"asdfghjkl;@~"}, + {u"zxcvbnm,./?\\"}, }, { // Latin Uppercase - {L"!@#$%^&*()_+"}, - {L"QWERTYUIOP{}"}, - {L"ASDFGHJKL:\"`"}, - {L"ZXCVBNM<>/?|"}, + {u"!@#$%^&*()_+"}, + {u"QWERTYUIOP{}"}, + {u"ASDFGHJKL:\"`"}, + {u"ZXCVBNM<>/?|"}, }, { // Hiragana - {L"あかさたなはまやらわぁゃっ"}, - {L"いきしちにひみ　り　ぃ　　"}, - {L"うくすつぬふむゆるをぅゅ゛"}, - {L"えけせてねへめ　れ　ぇ　゜"}, - {L"おこそとのほもよろんぉょー"}, + {u"あかさたなはまやらわぁゃっ"}, + {u"いきしちにひみ　り　ぃ　　"}, + {u"うくすつぬふむゆるをぅゅ゛"}, + {u"えけせてねへめ　れ　ぇ　゜"}, + {u"おこそとのほもよろんぉょー"}, }, { // Katakana - {L"アカサタナハマヤラワァャッ"}, - {L"イキシチニヒミ　リ　ィ　　"}, - {L"ウクスツヌフムユルヲゥュ゛"}, - {L"エケセテネヘメ　レ　ェ　゜"}, - {L"オコソトノホモヨロンォョー"}, + {u"アカサタナハマヤラワァャッ"}, + {u"イキシチニヒミ　リ　ィ　　"}, + {u"ウクスツヌフムユルヲゥュ゛"}, + {u"エケセテネヘメ　レ　ェ　゜"}, + {u"オコソトノホモヨロンォョー"}, }, { // Korean(Hangul) - {L"1234567890-+"}, - {L"ㅃㅉㄸㄲㅆ!@#$%^&"}, - {L"ㅂㅈㄷㄱㅅㅛㅕㅑㅐㅔ[]"}, - {L"ㅁㄴㅇㄹㅎㅗㅓㅏㅣ;@~"}, - {L"ㅋㅌㅊㅍㅠㅜㅡ<>/?|"}, + {u"1234567890-+"}, + {u"ㅃㅉㄸㄲㅆ!@#$%^&"}, + {u"ㅂㅈㄷㄱㅅㅛㅕㅑㅐㅔ[]"}, + {u"ㅁㄴㅇㄹㅎㅗㅓㅏㅣ;@~"}, + {u"ㅋㅌㅊㅍㅠㅜㅡ<>/?|"}, }, { // Russian Lowercase - {L"1234567890-+"}, - {L"йцукенгшщзхъ"}, - {L"фывапролджэё"}, - {L"ячсмитьбю/?|"}, + {u"1234567890-+"}, + {u"йцукенгшщзхъ"}, + {u"фывапролджэё"}, + {u"ячсмитьбю/?|"}, }, { // Russian Uppercase - {L"!@#$%^&*()_+"}, - {L"ЙЦУКЕНГШЩЗХЪ"}, - {L"ФЫВАПРОЛДЖЭЁ"}, - {L"ЯЧСМИТЬБЮ/?|"}, + {u"!@#$%^&*()_+"}, + {u"ЙЦУКЕНГШЩЗХЪ"}, + {u"ФЫВАПРОЛДЖЭЁ"}, + {u"ЯЧСМИТЬБЮ/?|"}, }, { // Latin Full-width Lowercase - { L"１２３４５６７８９０－＋" }, - { L"ｑｗｅｒｔｙｕｉｏｐ［］" }, - { L"ａｓｄｆｇｈｊｋｌ；＠～" }, - { L"ｚｘｃｖｂｎｍ，．／？￥￥" }, + { u"１２３４５６７８９０－＋" }, + { u"ｑｗｅｒｔｙｕｉｏｐ［］" }, + { u"ａｓｄｆｇｈｊｋｌ；＠～" }, + { u"ｚｘｃｖｂｎｍ，．／？￥￥" }, }, { // Latin Full-width Uppercase - { L"！＠＃＄％＾＆＊（）＿＋" }, - { L"ＱＷＥＲＴＹＵＩＯＰ｛｝" }, - { L"ＡＳＤＦＧＨＪＫＬ：￥”‘" }, - { L"ＺＸＣＶＢＮＭ＜＞／？｜" }, + { u"！＠＃＄％＾＆＊（）＿＋" }, + { u"ＱＷＥＲＴＹＵＩＯＰ｛｝" }, + { u"ＡＳＤＦＧＨＪＫＬ：￥”‘" }, + { u"ＺＸＣＶＢＮＭ＜＞／？｜" }, }, }; @@ -167,7 +167,7 @@ void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const PSPPointer& em_address) +void GetWideStringFromPSPPointer(std::u16string& _string, const PSPPointer& em_address) { if (!em_address.IsValid()) { - _string = L""; + _string.clear(); return; } const size_t maxLength = 2047; - wchar_t stringBuffer[maxLength + 1]; - wchar_t *string = stringBuffer; + char16_t stringBuffer[maxLength + 1]; + char16_t *string = stringBuffer; u16_le *input = &em_address[0]; int c; @@ -214,7 +214,7 @@ void GetWideStringFromPSPPointer(std::wstring& _string, const PSPPointer _string = stringBuffer; } -void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const wchar_t *input) +void PSPOskDialog::ConvertUCS2ToUTF8(std::string& _string, const char16_t *input) { char stringBuffer[2048]; char *string = stringBuffer; @@ -315,7 +315,7 @@ int PSPOskDialog::Init(u32 oskPtr) { i_level = 0; - inputChars = L""; + inputChars.clear(); if (oskParams->fields[0].intext.IsValid()) { auto src = oskParams->fields[0].intext; @@ -336,9 +336,9 @@ int PSPOskDialog::Init(u32 oskPtr) { return 0; } -std::wstring PSPOskDialog::CombinationKorean(bool isInput) +std::u16string PSPOskDialog::CombinationKorean(bool isInput) { - std::wstring string; + std::u16string string; isCombinated = true; @@ -578,9 +578,9 @@ std::wstring PSPOskDialog::CombinationKorean(bool isInput) return string; } -std::wstring PSPOskDialog::CombinationString(bool isInput) +std::u16string PSPOskDialog::CombinationString(bool isInput) { - std::wstring string; + std::u16string string; isCombinated = false; @@ -752,7 +752,7 @@ void PSPOskDialog::RenderKeyboard() int selectedRow = selectedChar / numKeyCols[currentKeyboard]; int selectedCol = selectedChar % numKeyCols[currentKeyboard]; - wchar_t temp[2]; + char16_t temp[2]; temp[1] = '\0'; std::string buffer; @@ -769,7 +769,7 @@ void PSPOskDialog::RenderKeyboard() PPGeDrawText(oskDesc.c_str(), title , 20, PPGE_ALIGN_CENTER, 0.5f, CalcFadedColor(0xFFFFFFFF)); - std::wstring result; + std::u16string result; result = CombinationString(false); @@ -855,16 +855,17 @@ int PSPOskDialog::NativeKeyboard() { } if (beginInputBox) { - std::wstring titleText; + std::u16string titleText; GetWideStringFromPSPPointer(titleText, oskParams->fields[0].desc); - std::wstring defaultText; + std::u16string defaultText; GetWideStringFromPSPPointer(defaultText, oskParams->fields[0].intext); if (defaultText.empty()) - defaultText.assign(L"VALUE"); + defaultText.assign(u"VALUE"); - System_InputBoxGetString(ConvertWStringToUTF8(titleText), ConvertWStringToUTF8(defaultText), [&](bool result, const std::string &value) { + // There's already ConvertUCS2ToUTF8 in this file. Should we use that instead of the global ones? + System_InputBoxGetString(::ConvertUCS2ToUTF8(titleText), ::ConvertUCS2ToUTF8(defaultText), [&](bool result, const std::string &value) { std::lock_guard guard(nativeMutex_); if (nativeStatus_ != PSPOskNativeStatus::WAITING) { return; @@ -874,7 +875,7 @@ int PSPOskDialog::NativeKeyboard() { nativeStatus_ = result ? PSPOskNativeStatus::SUCCESS : PSPOskNativeStatus::FAILURE; }); } else if (nativeStatus_ == PSPOskNativeStatus::SUCCESS) { - inputChars = ConvertUTF8ToWString(nativeValue_); + inputChars = ConvertUTF8ToUCS2(nativeValue_); nativeValue_.clear(); u32 maxLength = FieldMaxLength(); @@ -1080,9 +1081,9 @@ int PSPOskDialog::Update(int animSpeed) { } else if (IsButtonPressed(CTRL_SQUARE) && inputChars.size() < FieldMaxLength()) { // Use a regular space if the current keyboard isn't Japanese nor full-width English if (currentKeyboardLanguage != OSK_LANGUAGE_JAPANESE && currentKeyboardLanguage != OSK_LANGUAGE_ENGLISH_FW) - inputChars += L" "; + inputChars += u" "; else - inputChars += L"　"; + inputChars += u"　"; } EndDraw(); @@ -1123,7 +1124,7 @@ void PSPOskDialog::DoState(PointerWrap &p) { PSPDialog::DoState(p); - auto s = p.Section("PSPOskDialog", 1); + auto s = p.Section("PSPOskDialog", 2); if (!s) return; @@ -1132,7 +1133,13 @@ void PSPOskDialog::DoState(PointerWrap &p) p.Do(oskIntext); p.Do(oskOuttext); p.Do(selectedChar); - p.Do(inputChars); + if (s >= 2) { + p.Do(inputChars); + } else { + // Discard the wstring. + std::wstring wstr; + p.Do(wstr); + } // Don't need to save state native status or value. } diff --git a/Core/Dialog/PSPOskDialog.h b/Core/Dialog/PSPOskDialog.h index fc57bcf2e4..a244fd419a 100644 --- a/Core/Dialog/PSPOskDialog.h +++ b/Core/Dialog/PSPOskDialog.h @@ -229,12 +229,12 @@ protected: private: void ConvertUCS2ToUTF8(std::string& _string, const PSPPointer& em_address); - void ConvertUCS2ToUTF8(std::string& _string, const wchar_t *input); + void ConvertUCS2ToUTF8(std::string& _string, const char16_t *input); void RenderKeyboard(); int NativeKeyboard(); - std::wstring CombinationString(bool isInput); // for Japanese, Korean - std::wstring CombinationKorean(bool isInput); // for Korea + std::u16string CombinationString(bool isInput); // for Japanese, Korean + std::u16string CombinationKorean(bool isInput); // for Korea void RemoveKorean(); // for Korean character removal u32 FieldMaxLength(); @@ -246,7 +246,7 @@ private: std::string oskOuttext; int selectedChar; - std::wstring inputChars; + std::u16string inputChars; OskKeyboardDisplay currentKeyboard; OskKeyboardLanguage currentKeyboardLanguage; bool isCombinated; diff --git a/Core/HLE/sceCcc.cpp b/Core/HLE/sceCcc.cpp index 65ae665f7d..71cbfa6e40 100644 --- a/Core/HLE/sceCcc.cpp +++ b/Core/HLE/sceCcc.cpp @@ -28,9 +28,9 @@ #include "Core/Reporting.h" typedef PSPPointer PSPCharPointer; -typedef PSPPointer PSPWCharPointer; +typedef PSPPointer PSPWCharPointer; typedef PSPPointer PSPConstCharPointer; -typedef PSPPointer PSPConstWCharPointer; +typedef PSPPointer PSPConstWCharPointer; static u16 errorUTF8; static u16 errorUTF16; diff --git a/ext/native/util/text/utf16.h b/ext/native/util/text/utf16.h index 2475b7292a..830eac2c51 100644 --- a/ext/native/util/text/utf16.h +++ b/ext/native/util/text/utf16.h @@ -17,12 +17,12 @@ uint16_t UTF16_Swap(uint16_t u) { template struct UTF16_Type { public: - static const uint32_t INVALID = (uint32_t)-1; + static const char32_t INVALID = (char32_t)-1; - UTF16_Type(const uint16_t *c) : c_(c), index_(0) {} + UTF16_Type(const char16_t *c) : c_(c), index_(0) {} - uint32_t next() { - const uint32_t u = UTF16_Swap(c_[index_++]); + char32_t next() { + const char32_t u = UTF16_Swap(c_[index_++]); // Surrogate pair. UTF-16 is so simple. We assume it's valid. if ((u & 0xF800) == 0xD800) { @@ -46,19 +46,29 @@ public: return index_; } - static int encode(uint16_t *dest, uint32_t u) { + static int encode(char16_t *dest, char32_t u) { if (u >= 0x10000) { u -= 0x10000; *dest++ = UTF16_Swap(0xD800 + ((u >> 10) & 0x3FF)); *dest = UTF16_Swap(0xDC00 + ((u >> 0) & 0x3FF)); return 2; } else { - *dest = UTF16_Swap((uint16_t)u); + *dest = UTF16_Swap((char16_t)u); return 1; } } - static int encodeUnits(uint32_t u) { + // Rejects non-UCS2 codepoints. + static int encodeUCS2(char16_t *dest, char32_t u) { + if (u >= 0x10000 || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } else { + *dest = UTF16_Swap((char16_t)u); + return 1; + } + } + + static int encodeUnits(char32_t u) { if (u >= 0x10000) { return 2; } else { @@ -66,8 +76,15 @@ public: } } + static int encodeUnitsUCS2(char32_t u) { + if (u >= 0x10000 || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } else { + return 1; + } + } private: - const uint16_t *c_; + const char16_t *c_; int index_; }; diff --git a/ext/native/util/text/utf8.cpp b/ext/native/util/text/utf8.cpp index 4183463d46..ab734231db 100644 --- a/ext/native/util/text/utf8.cpp +++ b/ext/native/util/text/utf8.cpp @@ -461,7 +461,63 @@ std::wstring ConvertUTF8ToWString(const std::string &source) { return str; } -#else +#endif + +std::string ConvertUCS2ToUTF8(const std::u16string &wstr) { + std::string s; + // Worst case. + s.resize(wstr.size() * 4); + + size_t pos = 0; + for (wchar_t c : wstr) { + pos += UTF8::encode(&s[pos], c); + } + + s.resize(pos); + return s; +} + +static size_t ConvertUTF8ToUCS2Internal(char16_t *dest, size_t destSize, const std::string &source) { + const char16_t *const orig = dest; + const char16_t *const destEnd = dest + destSize; + + UTF8 utf(source.c_str()); + + char16_t *destw = (char16_t *)dest; + const char16_t *const destwEnd = destw + destSize; + + // Ignores characters outside the BMP. + while (uint32_t c = utf.next()) { + if (destw + UTF16LE::encodeUnitsUCS2(c) >= destwEnd) { + break; + } + destw += UTF16LE::encodeUCS2(destw, c); + } + + // No ++ to not count the terminal in length. + if (dest < destEnd) { + *dest = 0; + } + + return dest - orig; +} + +void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, const std::string &source) { + ConvertUTF8ToUCS2Internal(dest, destSize, source); +} + +std::u16string ConvertUTF8ToUCS2(const std::string &source) { + std::u16string dst; + // utf-8 won't be less bytes than there are characters. But need +1 for terminator. + dst.resize(source.size() + 1, 0); + size_t realLen = ConvertUTF8ToUCS2Internal(&dst[0], source.size() + 1, source); + dst.resize(realLen); + return dst; +} + +#ifndef _WIN32 + +// Replacements for the Win32 wstring functions. Not to be used from emulation code! std::string ConvertWStringToUTF8(const std::wstring &wstr) { std::string s; @@ -484,16 +540,16 @@ static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const UTF8 utf(source.c_str()); if (sizeof(wchar_t) == 2) { - uint16_t *destw = (uint16_t *)dest; - const uint16_t *const destwEnd = destw + destSize; - while (uint32_t c = utf.next()) { + char16_t *destw = (char16_t *)dest; + const char16_t *const destwEnd = destw + destSize; + while (char32_t c = utf.next()) { if (destw + UTF16LE::encodeUnits(c) >= destwEnd) { break; } destw += UTF16LE::encode(destw, c); } } else { - while (uint32_t c = utf.next()) { + while (char32_t c = utf.next()) { if (dest + 1 >= destEnd) { break; } @@ -509,10 +565,6 @@ static size_t ConvertUTF8ToWStringInternal(wchar_t *dest, size_t destSize, const return dest - orig; } -void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source) { - ConvertUTF8ToWStringInternal(dest, destSize, source); -} - std::wstring ConvertUTF8ToWString(const std::string &source) { std::wstring dst; // utf-8 won't be less bytes than there are characters. But need +1 for terminator. diff --git a/ext/native/util/text/utf8.h b/ext/native/util/text/utf8.h index 6303c25b45..fd6ddf64e7 100644 --- a/ext/native/util/text/utf8.h +++ b/ext/native/util/text/utf8.h @@ -82,13 +82,19 @@ bool UTF8StringHasNonASCII(const char *utf8string); std::string ConvertWStringToUTF8(const std::wstring &wstr); std::string ConvertWStringToUTF8(const wchar_t *wstr); +void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source); +std::wstring ConvertUTF8ToWString(const std::string &source); #else +// Used by SymbolMap/assembler +std::wstring ConvertUTF8ToWString(const std::string &source); std::string ConvertWStringToUTF8(const std::wstring &wstr); #endif +std::string ConvertUCS2ToUTF8(const std::u16string &wstr); + // Dest size in units, not bytes. -void ConvertUTF8ToWString(wchar_t *dest, size_t destSize, const std::string &source); -std::wstring ConvertUTF8ToWString(const std::string &source); +void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, const std::string &source); +std::u16string ConvertUTF8ToUCS2(const std::string &source);