Switch to a working UTF-8 mb/wc implementation.

Although glibc gets by with an 8-byte mbstate_t, OpenBSD uses 12 bytes (of
the 128 bytes it reserves!).

We can actually implement UTF-8 encoding/decoding with a 0-byte mbstate_t
which means we can make things work on LP32 too, as long as we accept the
limitation that the caller needs to present us with a complete sequence
before we'll process it.

Our behavior is fine when going from characters to bytes; we just
update the source wchar_t** to say how far through the input we got.

I'll come back and use the 4 bytes we do have to cope with byte sequences
split across multiple input buffers. The fact that we don't support
UTF-8 sequences longer than 4 bytes plus the fact that the first byte of
a UTF-8 sequence encodes the length means we shouldn't need the other
fields OpenBSD used (at the cost of some recomputation in cases where a
sequence is split across buffers).

This patch also makes the minimal changes necessary to setlocale(3) to
make us behave like glibc when an app requests UTF-8. (The difference
being that our "C" locale is the same as our "C.UTF-8" locale.)

Change-Id: Ied327a8c4643744b3611bf6bb005a9b389ba4c2f

1 files changed, 56 insertions, 3 deletions

diff --git a/tests/wchar_test.cpp b/tests/wchar_test.cpp
index f6c26837c..e98c14408 100644
--- a/tests/wchar_test.cpp
+++ b/tests/wchar_test.cpp
@@ -18,6 +18,7 @@
 
 #include <errno.h>
 #include <limits.h>
+#include <locale.h>
 #include <stdint.h>
 #include <wchar.h>
 
@@ -49,18 +50,46 @@ TEST(wchar, wctomb_wcrtomb) {
   EXPECT_EQ(1U, wcrtomb(bytes, L'\0', NULL));
 
   // ...and for regular characters.
-  bytes[0] = 'x';
+  memset(bytes, 0, sizeof(bytes));
   EXPECT_EQ(1, wctomb(bytes, L'h'));
   EXPECT_EQ('h', bytes[0]);
+  memset(bytes, 0, sizeof(bytes));
+  EXPECT_EQ(1U, wcrtomb(bytes, L'h', NULL));
+  EXPECT_EQ('h', bytes[0]);
+
+  ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
+  uselocale(LC_GLOBAL_LOCALE);
 
-  bytes[0] = 'x';
+  // 1-byte UTF-8.
+  memset(bytes, 0, sizeof(bytes));
   EXPECT_EQ(1U, wcrtomb(bytes, L'h', NULL));
   EXPECT_EQ('h', bytes[0]);
+  // 2-byte UTF-8.
+  memset(bytes, 0, sizeof(bytes));
+  EXPECT_EQ(2U, wcrtomb(bytes, 0x00a2, NULL));
+  EXPECT_EQ('\xc2', bytes[0]);
+  EXPECT_EQ('\xa2', bytes[1]);
+  // 3-byte UTF-8.
+  memset(bytes, 0, sizeof(bytes));
+  EXPECT_EQ(3U, wcrtomb(bytes, 0x20ac, NULL));
+  EXPECT_EQ('\xe2', bytes[0]);
+  EXPECT_EQ('\x82', bytes[1]);
+  EXPECT_EQ('\xac', bytes[2]);
+  // 4-byte UTF-8.
+  memset(bytes, 0, sizeof(bytes));
+  EXPECT_EQ(4U, wcrtomb(bytes, 0x24b62, NULL));
+  EXPECT_EQ('\xf0', bytes[0]);
+  EXPECT_EQ('\xa4', bytes[1]);
+  EXPECT_EQ('\xad', bytes[2]);
+  EXPECT_EQ('\xa2', bytes[3]);
+  // Invalid code point.
+  EXPECT_EQ(static_cast<size_t>(-1), wcrtomb(bytes, 0xffffffff, NULL));
+  EXPECT_EQ(EILSEQ, errno);
 }
 
 TEST(wchar, wcstombs_wcrtombs) {
   const wchar_t chars[] = { L'h', L'e', L'l', L'l', L'o', 0 };
-  const wchar_t bad_chars[] = { L'h', L'i', 666, 0 };
+  const wchar_t bad_chars[] = { L'h', L'i', 0xffffffff, 0 };
   const wchar_t* src;
   char bytes[BUFSIZ];
 
@@ -212,6 +241,30 @@ TEST(wchar, mbrtowc) {
   ASSERT_EQ(1U, mbrtowc(NULL, "hello", 1, NULL));
 
   ASSERT_EQ(0U, mbrtowc(NULL, NULL, 0, NULL));
+
+  ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
+  uselocale(LC_GLOBAL_LOCALE);
+
+  // 1-byte UTF-8.
+  ASSERT_EQ(1U, mbrtowc(out, "abcdef", 6, NULL));
+  ASSERT_EQ(L'a', out[0]);
+  // 2-byte UTF-8.
+  ASSERT_EQ(2U, mbrtowc(out, "\xc2\xa2" "cdef", 6, NULL));
+  ASSERT_EQ(0x00a2, out[0]);
+  // 3-byte UTF-8.
+  ASSERT_EQ(3U, mbrtowc(out, "\xe2\x82\xac" "def", 6, NULL));
+  ASSERT_EQ(0x20ac, out[0]);
+  // 4-byte UTF-8.
+  ASSERT_EQ(4U, mbrtowc(out, "\xf0\xa4\xad\xa2" "ef", 6, NULL));
+  ASSERT_EQ(0x24b62, out[0]);
+#if __BIONIC__ // glibc allows this.
+  // Illegal 5-byte UTF-8.
+  ASSERT_EQ(static_cast<size_t>(-1), mbrtowc(out, "\xf8\xa1\xa2\xa3\xa4" "f", 6, NULL));
+  ASSERT_EQ(EILSEQ, errno);
+#endif
+  // Illegal over-long sequence.
+  ASSERT_EQ(static_cast<size_t>(-1), mbrtowc(out, "\xf0\x82\x82\xac" "ef", 6, NULL));
+  ASSERT_EQ(EILSEQ, errno);
 }
 
 TEST(wchar, wcstod) {