Merge pull request #19805 from hrydgard/osk-fixes

Don't cut off the last char from strings from NativeKeyboard

Common/Data/Encoding/Utf8.cpp

@@ -50,102 +50,6 @@ static const uint8_t trailingBytesForUTF8[256] = {
 	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5,
 };
 
-/* conversions without error checking
-   only works for valid UTF-8, i.e. no 5- or 6-byte sequences
-   srcsz = source size in bytes, or -1 if 0-terminated
-   sz = dest size in # of wide characters
-
-   returns # characters converted
-   dest will always be L'\0'-terminated, even if there isn't enough room
-   for all the characters.
-   if sz = srcsz+1 (i.e. 4*srcsz+4 bytes), there will always be enough space.
-*/
-int u8_toucs(uint32_t *dest, int sz, const char *src, int srcsz) 
-{
-  uint32_t ch;
-  const char *src_end = src + srcsz;
-  int nb;
-  int i=0;
-
-  while (i < sz-1) {
-    nb = trailingBytesForUTF8[(unsigned char)*src];
-    if (srcsz == -1) {
-      if (*src == 0)
-        goto done_toucs;
-    }
-    else {
-      if (src + nb >= src_end)
-        goto done_toucs;
-    }
-    ch = 0;
-    switch (nb) {
-      /* these fall through deliberately */
-    case 3: ch += (unsigned char)*src++; ch <<= 6;
-    case 2: ch += (unsigned char)*src++; ch <<= 6;
-    case 1: ch += (unsigned char)*src++; ch <<= 6;
-    case 0: ch += (unsigned char)*src++;
-    }
-    ch -= offsetsFromUTF8[nb];
-    dest[i++] = ch;
-  }
- done_toucs:
-  dest[i] = 0;
-  return i;
-}
-
-/* srcsz = number of source characters, or -1 if 0-terminated
-   sz = size of dest buffer in bytes
-
-   returns # characters converted
-   dest will only be '\0'-terminated if there is enough space. this is
-   for consistency; imagine there are 2 bytes of space left, but the next
-   character requires 3 bytes. in this case we could NUL-terminate, but in
-   general we can't when there's insufficient space. therefore this function
-   only NUL-terminates if all the characters fit, and there's space for
-   the NUL as well.
-   the destination string will never be bigger than the source string.
-*/
-int u8_toutf8(char *dest, int sz, const uint32_t *src, int srcsz)
-{
-  uint32_t ch;
-  int i = 0;
-  char *dest_end = dest + sz;
-
-  while (srcsz<0 ? src[i]!=0 : i < srcsz) {
-    ch = src[i];
-    if (ch < 0x80) {
-      if (dest >= dest_end)
-        return i;
-      *dest++ = (char)ch;
-    }
-    else if (ch < 0x800) {
-      if (dest >= dest_end-1)
-        return i;
-      *dest++ = (ch>>6) | 0xC0;
-      *dest++ = (ch & 0x3F) | 0x80;
-    }
-    else if (ch < 0x10000) {
-      if (dest >= dest_end-2)
-        return i;
-      *dest++ = (ch>>12) | 0xE0;
-      *dest++ = ((ch>>6) & 0x3F) | 0x80;
-      *dest++ = (ch & 0x3F) | 0x80;
-    }
-    else if (ch < 0x110000) {
-      if (dest >= dest_end-3)
-        return i;
-      *dest++ = (ch>>18) | 0xF0;
-      *dest++ = ((ch>>12) & 0x3F) | 0x80;
-      *dest++ = ((ch>>6) & 0x3F) | 0x80;
-      *dest++ = (ch & 0x3F) | 0x80;
-    }
-    i++;
-  }
-  if (dest < dest_end)
-    *dest = '\0';
-  return i;
-}
-
 int u8_wc_toutf8(char *dest, uint32_t ch)
 {
   if (ch < 0x80) {
@@ -237,133 +141,14 @@ uint32_t u8_nextchar_unsafe(const char *s, int *i) {
 	return ch;
 }
 
-void u8_inc(const char *s, int *i)
-{
-  (void)(isutf(s[++(*i)]) || isutf(s[++(*i)]) ||
-       isutf(s[++(*i)]) || ++(*i));
+void u8_inc(const char *s, int *i) {
+	(void)(isutf(s[++(*i)]) || isutf(s[++(*i)]) ||
+		isutf(s[++(*i)]) || ++(*i));
 }
 
-void u8_dec(const char *s, int *i)
-{
-  (void)(isutf(s[--(*i)]) || isutf(s[--(*i)]) ||
-       isutf(s[--(*i)]) || --(*i));
-}
-
-int octal_digit(char c)
-{
-  return (c >= '0' && c <= '7');
-}
-
-int hex_digit(char c)
-{
-  return ((c >= '0' && c <= '9') ||
-      (c >= 'A' && c <= 'F') ||
-      (c >= 'a' && c <= 'f'));
-}
-
-/* assumes that src points to the character after a backslash
-   returns number of input characters processed */
-int u8_read_escape_sequence(const char *str, uint32_t *dest)
-{
-  long ch;
-  char digs[9]="\0\0\0\0\0\0\0\0";
-  int dno=0, i=1;
-
-  ch = (uint32_t)str[0];  /* take literal character */
-  if (str[0] == 'n')
-    ch = L'\n';
-  else if (str[0] == 't')
-    ch = L'\t';
-  else if (str[0] == 'r')
-    ch = L'\r';
-  else if (str[0] == 'b')
-    ch = L'\b';
-  else if (str[0] == 'f')
-    ch = L'\f';
-  else if (str[0] == 'v')
-    ch = L'\v';
-  else if (str[0] == 'a')
-    ch = L'\a';
-  else if (octal_digit(str[0])) {
-    i = 0;
-    do {
-      digs[dno++] = str[i++];
-    } while (octal_digit(str[i]) && dno < 3);
-    ch = strtol(digs, NULL, 8);
-  }
-  else if (str[0] == 'x') {
-    while (hex_digit(str[i]) && dno < 2) {
-      digs[dno++] = str[i++];
-    }
-    if (dno > 0)
-      ch = strtol(digs, NULL, 16);
-  }
-  else if (str[0] == 'u') {
-    while (hex_digit(str[i]) && dno < 4) {
-      digs[dno++] = str[i++];
-    }
-    if (dno > 0)
-      ch = strtol(digs, NULL, 16);
-  }
-  else if (str[0] == 'U') {
-    while (hex_digit(str[i]) && dno < 8) {
-      digs[dno++] = str[i++];
-    }
-    if (dno > 0)
-      ch = strtol(digs, NULL, 16);
-  }
-  *dest = (uint32_t)ch;
-
-  return i;
-}
-
-/* convert a string with literal \uxxxx or \Uxxxxxxxx characters to UTF-8
-   example: u8_unescape(mybuf, 256, "hello\\u220e")
-   note the double backslash is needed if called on a C string literal */
-int u8_unescape(char *buf, int sz, char *src)
-{
-  int c=0, amt;
-  uint32_t ch;
-  char temp[4];
-
-  while (*src && c < sz) {
-    if (*src == '\\') {
-      src++;
-      amt = u8_read_escape_sequence(src, &ch);
-    }
-    else {
-      ch = (uint32_t)*src;
-      amt = 1;
-    }
-    src += amt;
-    amt = u8_wc_toutf8(temp, ch);
-    if (amt > sz-c)
-      break;
-    memcpy(&buf[c], temp, amt);
-    c += amt;
-  }
-  if (c < sz)
-    buf[c] = '\0';
-  return c;
-}
-
-int u8_is_locale_utf8(const char *locale)
-{
-  /* this code based on libutf8 */
-  const char* cp = locale;
-
-  for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++) {
-    if (*cp == '.') {
-      const char* encoding = ++cp;
-      for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++)
-        ;
-      if ((cp-encoding == 5 && !strncmp(encoding, "UTF-8", 5))
-        || (cp-encoding == 4 && !strncmp(encoding, "utf8", 4)))
-        return 1; /* it's UTF-8 */
-      break;
-    }
-  }
-  return 0;
+void u8_dec(const char *s, int *i) {
+	(void)(isutf(s[--(*i)]) || isutf(s[--(*i)]) ||
+		isutf(s[--(*i)]) || --(*i));
 }
 
 bool AnyEmojiInString(std::string_view str, size_t byteCount) {
@@ -492,15 +277,10 @@ static size_t ConvertUTF8ToUCS2Internal(char16_t *dest, size_t destSize, std::st
 	return destw - orig;
 }
 
-void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, const std::string &source) {
-	ConvertUTF8ToUCS2Internal(dest, destSize, source);
-}
-
 std::u16string ConvertUTF8ToUCS2(std::string_view source) {
 	std::u16string dst;
-	// utf-8 won't be less bytes than there are characters.
-	dst.resize(source.size(), 0);
-	size_t realLen = ConvertUTF8ToUCS2Internal(&dst[0], source.size(), source);
+	dst.resize(source.size() + 1, 0);  // multiple UTF-8 chars will be one UCS2 char. But we need to leave space for a terminating null.
+	size_t realLen = ConvertUTF8ToUCS2Internal(&dst[0], dst.size(), source);
 	dst.resize(realLen);
 	return dst;
 }

Common/Data/Encoding/Utf8.h

@@ -124,7 +124,6 @@ std::string ConvertWStringToUTF8(const std::wstring &wstr);
 std::string ConvertUCS2ToUTF8(const std::u16string &wstr);
 
 // Dest size in units, not bytes.
-void ConvertUTF8ToUCS2(char16_t *dest, size_t destSize, std::string_view source);
 std::u16string ConvertUTF8ToUCS2(std::string_view source);
 
 // Java needs 4-byte UTF-8 to be converted to surrogate pairs, each component of which get

Core/Dialog/PSPOskDialog.cpp

@@ -54,6 +54,15 @@ static const uint8_t kor_lconsCom[33] = { 18,0,2,21,3,4,26,3,5,0,7,8,15,7,9,16,7
 // Korean (Hangul) last consonant Separation key
 static const uint8_t kor_lconsSpr[33] = { 2,1,9,4,4,12,5,4,18,8,8,0,9,8,6,10,8,7,11,8,9,12,8,16,13,8,17,14,8,18,17,17,9 };
 
+static const char * const OskKeyboardNames[] =
+{
+	"en_US",
+	"ja_JP",
+	"ko_KR",
+	"ru_RU",
+	"English Full-width",
+};
+
 // This isn't a complete representation of these flags, it just helps ensure we show the right keyboards.
 static const int allowedInputFlagsMap[OSK_KEYBOARD_COUNT] = {
 	PSP_UTILITY_OSK_INPUTTYPE_LATIN_LOWERCASE | PSP_UTILITY_OSK_INPUTTYPE_LATIN_UPPERCASE | PSP_UTILITY_OSK_INPUTTYPE_LATIN_SYMBOL | PSP_UTILITY_OSK_INPUTTYPE_LATIN_DIGIT,
@@ -703,7 +712,7 @@ void PSPOskDialog::RenderKeyboard()
 	float title = (480.0f - (0.5f * drawLimit)) / 2.0f;
 
 	PPGeStyle descStyle = FadedStyle(PPGeAlign::BOX_CENTER, 0.5f);
-	PPGeDrawText(oskDesc.c_str(), title, 20, descStyle);
+	PPGeDrawText(oskDesc, title, 20, descStyle);
 
 	PPGeStyle textStyle = FadedStyle(PPGeAlign::BOX_HCENTER, 0.5f);
 
@@ -723,7 +732,7 @@ void PSPOskDialog::RenderKeyboard()
 		{
 			temp[0] = result[drawIndex];
 			ConvertUCS2ToUTF8(buffer, temp);
-			PPGeDrawText(buffer.c_str(), previewLeftSide + (i * characterWidth), 40.0f, textStyle);
+			PPGeDrawText(buffer, previewLeftSide + (i * characterWidth), 40.0f, textStyle);
 		}
 		else
 		{
@@ -741,7 +750,7 @@ void PSPOskDialog::RenderKeyboard()
 
 					ConvertUCS2ToUTF8(buffer, temp);
 
-					PPGeDrawText(buffer.c_str(), previewLeftSide + (i * characterWidth), 40.0f, animStyle);
+					PPGeDrawText(buffer, previewLeftSide + (i * characterWidth), 40.0f, animStyle);
 
 					// Also draw the underline for the same reason.
 					PPGeDrawText("_", previewLeftSide + (i * characterWidth), 40.0f, textStyle);
@@ -749,7 +758,7 @@ void PSPOskDialog::RenderKeyboard()
 				else
 				{
 					ConvertUCS2ToUTF8(buffer, temp);
-					PPGeDrawText(buffer.c_str(), previewLeftSide + (i * characterWidth), 40.0f, textStyle);
+					PPGeDrawText(buffer, previewLeftSide + (i * characterWidth), 40.0f, textStyle);
 				}
 			}
 			else
@@ -768,10 +777,10 @@ void PSPOskDialog::RenderKeyboard()
 			ConvertUCS2ToUTF8(buffer, temp);
 
 			if (selectedRow == row && col == selectedCol) {
-				PPGeDrawText(buffer.c_str(), keyboardLeftSide + (25.0f * col) + characterWidth / 2.0, 70.0f + (25.0f * row), selectedKeyStyle);
+				PPGeDrawText(buffer, keyboardLeftSide + (25.0f * col) + characterWidth / 2.0, 70.0f + (25.0f * row), selectedKeyStyle);
 				PPGeDrawText("_", keyboardLeftSide + (25.0f * col) + characterWidth / 2.0, 70.0f + (25.0f * row), keyStyle);
 			} else {
-				PPGeDrawText(buffer.c_str(), keyboardLeftSide + (25.0f * col) + characterWidth / 2.0, 70.0f + (25.0f * row), keyStyle);
+				PPGeDrawText(buffer, keyboardLeftSide + (25.0f * col) + characterWidth / 2.0, 70.0f + (25.0f * row), keyStyle);
 			}
 		}
 	}
@@ -931,7 +940,7 @@ int PSPOskDialog::Update(int animSpeed) {
 		}
 
 		// Now, let's grab the name.
-		const char *countryCode = OskKeyboardNames[lang].c_str();
+		const char *countryCode = OskKeyboardNames[lang];
 		const char *language = languageMapping[countryCode].first.c_str();
 
 		// It seems like this is a "fake" country code for extra keyboard purposes.

Core/Dialog/PSPOskDialog.h

@@ -177,15 +177,6 @@ const OskKeyboardDisplay OskKeyboardCases[OSK_LANGUAGE_COUNT][2] =
 	{ OSK_KEYBOARD_LATIN_FW_LOWERCASE, OSK_KEYBOARD_LATIN_FW_UPPERCASE }
 };
 
-static const std::string OskKeyboardNames[] =
-{
-	"en_US",
-	"ja_JP",
-	"ko_KR",
-	"ru_RU",
-	"English Full-width",
-};
-
 enum class PSPOskNativeStatus {
 	IDLE,
 	DONE,