Skip to content

Commit 71d5479

Browse files
committed
[0969] herk_to_utf8 补 4 字节 UTF-8 分支修复 emoji 乱码
append_utf8_code 缺少 U+10000 以上码点的 4 字节编码分支,herk_to_utf8 还原 <#1F60A> 等逃逸串时按 3 字节截断,输出非法 UTF-8(FF 98 8A), 经聊天显示与 .tmu 保存逐字节 Latin-1 再编码后即用户所见的乱码。 补齐分支并与 encode_as_utf8 行为对齐,附 herk_4byte_codepoints 回归测试。
1 parent d68896c commit 71d5479

2 files changed

Lines changed: 26 additions & 1 deletion

File tree

lolly/lolly/data/herk.cpp

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,19 @@ append_utf8_code (string& r, int code) {
118118
r << (char) (0xC0 | (code >> 6));
119119
r << (char) (0x80 | (code & 0x3F));
120120
}
121-
else {
121+
else if (code < 0x10000) {
122122
r << (char) (0xE0 | (code >> 12));
123123
r << (char) (0x80 | ((code >> 6) & 0x3F));
124124
r << (char) (0x80 | (code & 0x3F));
125125
}
126+
else if (code <= 0x1FFFFF) {
127+
// 4 字节形式(U+10000 以上,含 emoji);缺失此分支会把 17+ 位码点
128+
// 截成 3 字节非法 UTF-8(如 U+1F60A 曾输出 FF 98 8A 即 LLM emoji 乱码)
129+
r << (char) (0xF0 | (code >> 18));
130+
r << (char) (0x80 | ((code >> 12) & 0x3F));
131+
r << (char) (0x80 | ((code >> 6) & 0x3F));
132+
r << (char) (0x80 | (code & 0x3F));
133+
}
126134
}
127135

128136
string

lolly/tests/lolly/data/herk_test.cpp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,6 +746,23 @@ TEST_CASE ("herk_escapes") {
746746
string_eq (utf8_to_herk ("<#00FF>"), "<#00FF>");
747747
}
748748

749+
TEST_CASE ("herk_4byte_codepoints") {
750+
// 辅助平面码点(emoji 等)往返:herk 侧逃逸为 <#XXXX>,还原须是 4 字节
751+
// UTF-8 而非截断的 3 字节(U+1F60A 曾被截成 FF 98 8A)
752+
string_eq (herk_to_utf8 ("<#1F60A>"), "\xF0\x9F\x98\x8A"); // U+1F60A 😊
753+
string_eq (utf8_to_herk ("\xF0\x9F\x98\x8A"), "<#1F60A>"); // U+1F60A
754+
string_eq (herk_to_utf8 ("<#1F642>"), "\xF0\x9F\x99\x82"); // U+1F642 🙂
755+
string_eq (utf8_to_herk ("\xF0\x9F\x99\x82"), "<#1F642>"); // U+1F642
756+
string_eq (herk_to_utf8 ("<#20000>"),
757+
"\xF0\xA0\x80\x80"); // U+20000 𠀀
758+
string_eq (utf8_to_herk ("\xF0\xA0\x80\x80"), "<#20000>"); // U+20000
759+
string_eq (herk_to_utf8 ("<#10FFFF>"),
760+
"\xF4\x8F\xBF\xBF"); // U+10FFFF
761+
string_eq (utf8_to_herk ("\xF4\x8F\xBF\xBF"), "<#10FFFF>"); // U+10FFFF
762+
string_eq (herk_to_utf8 ("A<#1F60A>B"), "A\xF0\x9F\x98\x8A"
763+
"B");
764+
}
765+
749766
TEST_CASE ("herk_named_escapes") {
750767
string_eq (utf8_to_herk ("<less>"), "<less>");
751768
string_eq (herk_to_utf8 ("<less>"), "<less>");

0 commit comments

Comments
 (0)