Skip to content

Commit 97370e8

Browse files
sbc100sbc100
authored
Use codePointAt in lengthBytesUTF8 to match stringToUTF8Array (#28009)
In #24356, `stringToUTF8Array` was updated to use `str.codePointAt(i)` instead of `str.charCodeAt(i)`, causing lone surrogates to be encoded as 3 bytes without skipping the following code unit. `lengthBytesUTF8` was not updated, so it still counted lone surrogates as 4 bytes and skipped the next code unit. Update `lengthBytesUTF8` to also iterate using `str.codePointAt(i)` so that `lengthBytesUTF8` and `stringToUTF8Array` always agree, while also saving a few bytes of JS code size. Fixes: #28008
1 parent 119421b commit 97370e8

23 files changed

Lines changed: 108 additions & 101 deletions

‎src/lib/libstrings.js‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -237,19 +237,18 @@ addToLibrary({
237237
$lengthBytesUTF8: (str) => {
238238
var len = 0;
239239
for (var i = 0; i < str.length; ++i) {
240-
// Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code
241-
// unit, not a Unicode code point of the character! So decode
242-
// UTF16->UTF32->UTF8.
243-
// See http://unicode.org/faq/utf_bom.html#utf16-3
244-
var c = str.charCodeAt(i); // possibly a lead surrogate
245-
if (c <= 0x7F) {
240+
var u = str.codePointAt(i);
241+
if (u <= 0x7F) {
246242
len++;
247-
} else if (c <= 0x7FF) {
243+
} else if (u <= 0x7FF) {
248244
len += 2;
249-
} else if (c >= 0xD800 && c <= 0xDFFF) {
250-
len += 4; ++i;
251-
} else {
245+
} else if (u <= 0xFFFF) {
252246
len += 3;
247+
} else {
248+
len += 4;
249+
// Gotcha: if codePoint is over 0xFFFF, it is represented as a surrogate pair in UTF-16.
250+
// We need to manually skip over the second code unit for correct iteration.
251+
i++;
253252
}
254253
}
255254
return len;

‎test/codesize/test_codesize_cxx_ctors1.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19126,
3-
"a.out.js.gz": 8079,
2+
"a.out.js": 19117,
3+
"a.out.js.gz": 8073,
44
"a.out.nodebug.wasm": 133999,
55
"a.out.nodebug.wasm.gz": 51252,
6-
"total": 153125,
7-
"total_gz": 59331,
6+
"total": 153116,
7+
"total_gz": 59325,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_cxx_ctors2.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19103,
3-
"a.out.js.gz": 8063,
2+
"a.out.js": 19094,
3+
"a.out.js.gz": 8056,
44
"a.out.nodebug.wasm": 133427,
55
"a.out.nodebug.wasm.gz": 50944,
6-
"total": 152530,
7-
"total_gz": 59007,
6+
"total": 152521,
7+
"total_gz": 59000,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_cxx_except.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 22820,
3-
"a.out.js.gz": 9041,
2+
"a.out.js": 22811,
3+
"a.out.js.gz": 9031,
44
"a.out.nodebug.wasm": 176407,
55
"a.out.nodebug.wasm.gz": 58760,
6-
"total": 199227,
7-
"total_gz": 67801,
6+
"total": 199218,
7+
"total_gz": 67791,
88
"sent": [
99
"__cxa_begin_catch",
1010
"__cxa_end_catch",

‎test/codesize/test_codesize_cxx_except_wasm.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 18925,
3-
"a.out.js.gz": 7999,
2+
"a.out.js": 18916,
3+
"a.out.js.gz": 7988,
44
"a.out.nodebug.wasm": 149664,
55
"a.out.nodebug.wasm.gz": 56300,
6-
"total": 168589,
7-
"total_gz": 64299,
6+
"total": 168580,
7+
"total_gz": 64288,
88
"sent": [
99
"_abort_js",
1010
"_tzset_js",

‎test/codesize/test_codesize_cxx_except_wasm_legacy.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19003,
3-
"a.out.js.gz": 8022,
2+
"a.out.js": 18994,
3+
"a.out.js.gz": 8012,
44
"a.out.nodebug.wasm": 147446,
55
"a.out.nodebug.wasm.gz": 55963,
6-
"total": 166449,
7-
"total_gz": 63985,
6+
"total": 166440,
7+
"total_gz": 63975,
88
"sent": [
99
"_abort_js",
1010
"_tzset_js",

‎test/codesize/test_codesize_cxx_lto.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 18466,
3-
"a.out.js.gz": 7775,
2+
"a.out.js": 18457,
3+
"a.out.js.gz": 7760,
44
"a.out.nodebug.wasm": 101050,
55
"a.out.nodebug.wasm.gz": 38287,
6-
"total": 119516,
7-
"total_gz": 46062,
6+
"total": 119507,
7+
"total_gz": 46047,
88
"sent": [
99
"a (emscripten_resize_heap)",
1010
"b (_setitimer_js)",

‎test/codesize/test_codesize_cxx_mangle.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 22870,
3-
"a.out.js.gz": 9061,
2+
"a.out.js": 22861,
3+
"a.out.js.gz": 9052,
44
"a.out.nodebug.wasm": 242687,
55
"a.out.nodebug.wasm.gz": 80983,
6-
"total": 265557,
7-
"total_gz": 90044,
6+
"total": 265548,
7+
"total_gz": 90035,
88
"sent": [
99
"__cxa_begin_catch",
1010
"__cxa_end_catch",

‎test/codesize/test_codesize_cxx_noexcept.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 19126,
3-
"a.out.js.gz": 8079,
2+
"a.out.js": 19117,
3+
"a.out.js.gz": 8073,
44
"a.out.nodebug.wasm": 135906,
55
"a.out.nodebug.wasm.gz": 51862,
6-
"total": 155032,
7-
"total_gz": 59941,
6+
"total": 155023,
7+
"total_gz": 59935,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

‎test/codesize/test_codesize_cxx_wasmfs.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"a.out.js": 6599,
3-
"a.out.js.gz": 3148,
2+
"a.out.js": 6590,
3+
"a.out.js.gz": 3132,
44
"a.out.nodebug.wasm": 173631,
55
"a.out.nodebug.wasm.gz": 64613,
6-
"total": 180230,
7-
"total_gz": 67761,
6+
"total": 180221,
7+
"total_gz": 67745,
88
"sent": [
99
"__cxa_throw",
1010
"_abort_js",

0 commit comments

Comments
 (0)