Skip to content

Commit 30c9af0

Browse files
hejsztynxCopilot
hejsztynx
and
Copilot
authored
fix: checkbox list normalization and stripped empty list elements (#668)
# Summary There were two issues: 1. The normalizer didn't understand the Tiptap's checkbox list structure and the Google Docs' or MS Word's one, so when pasting their HTML, the lists were broken. This also included a case where we just copy-pasted a checkbox list internally in the web's `EnrichedTextInput` - the list became a normal unordered list. 2. The empty `<li></li>` were stripped by the normalizer, which was incorrect, as we allow this structure. ## Test Plan - Write some checkbox lists in the web `EnrichedTextInput` and copy-paste them. The pasted list should be parsed and displayed correctly now. - Go to Google Docs and MS Word, create some checkbox lists and paste them into the `EnrichedTextInput` on a chosen platform. They should be correctly parsed now. - Try creating a list with empty elements on the web and copy-paste it to the same input or to an input on a mobile platform. The empty elements should not be stripped now. ## Screenshots / Videos The internal Tiptap copy-paste issue Before: https://github.com/user-attachments/assets/db47c1bd-c0ae-481d-9389-1f2fd3a62240 After: https://github.com/user-attachments/assets/c7033f9d-9c5e-465b-b9c5-5816b7346de1 ## Compatibility | OS | Implemented | | ------- | :---------: | | iOS | ✅ | | Android | ✅ | | Web | ✅ | ## Checklist - [x] E2E tests are passing - [x] Required E2E tests have been added (if applicable) --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent b557de9 commit 30c9af0

4 files changed

Lines changed: 218 additions & 9 deletions

File tree

‎cpp/parser/GumboNormalizer.c‎

Lines changed: 61 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,34 @@ static void emit_one_attr(buffer_t *out, GumboElement *el,
411411
}
412412
}
413413

414+
static bool is_checkbox_list(GumboElement *el) {
415+
const char *val = get_attr(el, "data-type");
416+
if (val && (strcmp(val, "checkbox") == 0 || strcmp(val, "checkboxList") == 0)) {
417+
return true;
418+
}
419+
420+
// In Google Docs and MS Word the <li> elements define if it is a checkbox
421+
// list. We only need to check the first <li>.
422+
GumboVector *children = &el->children;
423+
for (unsigned int i = 0; i < children->length; i++) {
424+
GumboNode *child = children->data[i];
425+
if (is_element(child)) {
426+
char child_tag[64];
427+
if (get_tag_name(child, child_tag, sizeof(child_tag)) && strcmp(child_tag, "li") == 0) {
428+
GumboElement *child_el = &child->v.element;
429+
const char *role = get_attr(child_el, "role");
430+
const char *cls = get_attr(child_el, "class");
431+
432+
// Matches Google Docs (role="checkbox") OR MS Word (class includes "checklist")
433+
return (role && strcmp(role, "checkbox") == 0) ||
434+
(cls && strstr(cls, "checklist") != NULL);
435+
}
436+
}
437+
}
438+
439+
return false;
440+
}
441+
414442
static void emit_attributes(GumboElement *el, const char *tag_name,
415443
buffer_t *out) {
416444
if (strcmp(tag_name, "a") == 0) {
@@ -421,12 +449,21 @@ static void emit_attributes(GumboElement *el, const char *tag_name,
421449
emit_one_attr(out, el, "width");
422450
emit_one_attr(out, el, "height");
423451
} else if (strcmp(tag_name, "ul") == 0) {
424-
const char *val = get_attr(el, "data-type");
425-
if (val && strcmp(val, "checkbox") == 0)
452+
if (is_checkbox_list(el)) {
426453
buffer_append_str(out, " data-type=\"checkbox\"");
454+
}
427455
} else if (strcmp(tag_name, "li") == 0) {
428-
if (gumbo_get_attribute(&el->attributes, "checked") != NULL)
456+
const char *data_checked = get_attr(el, "data-checked");
457+
const char *aria_checked = get_attr(el, "aria-checked");
458+
const char *level_text = get_attr(el, "data-leveltext");
459+
460+
// "\xEF\x83\xBE" is the UTF-8 hex encoding for U+F0FE (MS Word Checked Box)
461+
if (gumbo_get_attribute(&el->attributes, "checked") != NULL ||
462+
(data_checked && strcmp(data_checked, "true") == 0) ||
463+
(aria_checked && strcmp(aria_checked, "true") == 0) ||
464+
(level_text && strcmp(level_text, "\xEF\x83\xBE") == 0)) {
429465
buffer_append_str(out, " checked");
466+
}
430467
} else if (strcmp(tag_name, "mention") == 0) {
431468
emit_one_attr(out, el, "id");
432469
emit_one_attr(out, el, "text");
@@ -511,6 +548,7 @@ typedef struct {
511548
GumboNode **nested_lists;
512549
int *nested_count;
513550
int max_nested;
551+
bool has_emitted;
514552
} li_ctx_t;
515553

516554
static void flatten_li_node(GumboNode *node, buffer_t *ib, buffer_t *out,
@@ -527,6 +565,7 @@ static void flush_li_buffer(buffer_t *ib, buffer_t *out, li_ctx_t *ctx) {
527565
emit_styles_close(out, ctx->styles);
528566
buffer_append_str(out, "</li>");
529567
buffer_clear(ib);
568+
ctx->has_emitted = true;
530569
}
531570

532571
static void flatten_li_children(GumboNode *node, buffer_t *ib, buffer_t *out,
@@ -551,6 +590,17 @@ static void flatten_li_node(GumboNode *node, buffer_t *ib, buffer_t *out,
551590
flatten_li_children(node, ib, out, ctx);
552591
return;
553592
}
593+
594+
char buf[64];
595+
const char *tag = get_tag_name(node, buf, sizeof(buf));
596+
if (tag && strcmp(tag, "img") == 0) {
597+
const char *role = get_attr(ctx->el, "role");
598+
// strip the <img> that Google Docs uses for the display of a checkbox icon
599+
if (role && strcmp(role, "checkbox") == 0) {
600+
return;
601+
}
602+
}
603+
554604
if (is_list_node(node)) {
555605
if (*ctx->nested_count < ctx->max_nested) {
556606
ctx->nested_lists[*ctx->nested_count] = node;
@@ -837,6 +887,14 @@ static void walk_node(GumboNode *node, buffer_t *out) {
837887
li_ctx_t ctx = {el, es, nested_lists, &nested_count, 16};
838888
flatten_li_children(node, &li_ib, out, &ctx);
839889
flush_li_buffer(&li_ib, out, &ctx);
890+
891+
/* if nothing emitted - the <li> is empty, we add it manually */
892+
if (!ctx.has_emitted) {
893+
buffer_append_str(out, "<li");
894+
emit_attributes(el, "li", out);
895+
buffer_append_str(out, "></li>");
896+
}
897+
840898
free(li_ib.data);
841899
for (int k = 0; k < nested_count; k++)
842900
walk_children(nested_lists[k], out);

‎cpp/tests/GumboParserTest.cpp‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,60 @@ TEST(GumboParserTest, ListFlattening) {
448448
"<ul><li><b>another one </b>hi kacper,</li><li>hi</li></ul>");
449449
}
450450

451+
TEST(GumboParserTest, TiptapCheckboxList) {
452+
EXPECT_EQ(
453+
GumboParser::normalizeHtml(
454+
"<ul data-type=\"checkboxList\"><li data-checked=\"true\" "
455+
"data-type=\"checkboxItem\"><label><input type=\"checkbox\" "
456+
"checked=\"checked\"><span></span></label><div><p>first</p></div></"
457+
"li><li data-checked=\"false\" data-type=\"checkboxItem\"><label>"
458+
"<input type=\"checkbox\"><span></span></label><div><p>second</p></"
459+
"div></li></ul>"),
460+
"<ul data-type=\"checkbox\"><li checked>first</li><li>second</li></ul>");
461+
}
462+
463+
TEST(GumboParserTest, GoogleDocsCheckboxList) {
464+
EXPECT_EQ(GumboParser::normalizeHtml(
465+
"<ul><li role=\"checkbox\" aria-checked=\"true\"><img "
466+
"src=\"data:...\" /><p>Checked</p></li><li role=\"checkbox\" "
467+
"aria-checked=\"false\"><img src=\"data:...\" "
468+
"/><p>Unchecked</p></li></ul>"),
469+
"<ul data-type=\"checkbox\"><li "
470+
"checked>Checked</li><li>Unchecked</li></ul>");
471+
}
472+
473+
TEST(GumboParserTest, MSWordCheckboxList) {
474+
// \xEF\x83\xBE is the UTF-8 hex for U+F0FE (Checked MS Word box)
475+
// \xEF\x82\xA8 is the UTF-8 hex for U+F0A8 (Unchecked MS Word box)
476+
EXPECT_EQ(
477+
GumboParser::normalizeHtml(
478+
"<ul><li class=\"OutlineElement checklist\" "
479+
"data-leveltext=\"\xEF\x83\xBE\">Checked</li><li "
480+
"class=\"OutlineElement "
481+
"checklist\" data-leveltext=\"\xEF\x82\xA8\">Unchecked</li></ul>"),
482+
"<ul data-type=\"checkbox\"><li "
483+
"checked>Checked</li><li>Unchecked</li></ul>");
484+
}
485+
486+
TEST(GumboParserTest, EmptyListItems) {
487+
EXPECT_EQ(GumboParser::normalizeHtml("<ul><li></li><li>first</li><li></"
488+
"li><li>second</li><li></li><li></li>"
489+
"</ul>"),
490+
"<ul><li></li><li>first</li><li></li><li>second</li><li></li><li></"
491+
"li></ul>");
492+
EXPECT_EQ(GumboParser::normalizeHtml("<ol><li></li><li>first</li><li></"
493+
"li><li>second</li><li></li><li></li>"
494+
"</ol>"),
495+
"<ol><li></li><li>first</li><li></li><li>second</li><li></li><li></"
496+
"li></ol>");
497+
EXPECT_EQ(
498+
GumboParser::normalizeHtml(
499+
"<ul data-type=\"checkbox\"><li checked></li><li>first</li><li>"
500+
"</li><li checked>second</li><li></li><li></li></ul>"),
501+
"<ul data-type=\"checkbox\"><li checked></li><li>first</li><li></li><li "
502+
"checked>second</li><li></li><li></li></ul>");
503+
}
504+
451505
TEST(GumboParserTest, BrRemappings) {
452506
EXPECT_EQ(GumboParser::normalizeHtml(
453507
"<p><b>Asdasdasd</b></p><br><br><p>Sent with<span> </span><a "

‎src/web/__tests__/htmlNormalizer.test.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,56 @@ describe('htmlNormalizer', () => {
417417
});
418418
});
419419

420+
describe('EmptyListItems', () => {
421+
test.each([
422+
[
423+
'<ul><li></li><li>first</li><li></li><li>second</li><li></li><li></li></ul>',
424+
'<ul><li></li><li>first</li><li></li><li>second</li><li></li><li></li></ul>',
425+
],
426+
[
427+
'<ol><li></li><li>first</li><li></li><li>second</li><li></li><li></li></ol>',
428+
'<ol><li></li><li>first</li><li></li><li>second</li><li></li><li></li></ol>',
429+
],
430+
[
431+
'<ul data-type="checkbox"><li checked></li><li>first</li><li></li><li checked>second</li><li></li><li></li></ul>',
432+
'<ul data-type="checkbox"><li checked></li><li>first</li><li></li><li checked>second</li><li></li><li></li></ul>',
433+
],
434+
])('%s → %s', (input, expected) => {
435+
expect(normalizeHtml(input)).toBe(expected);
436+
});
437+
});
438+
439+
describe('TiptapCheckboxList', () => {
440+
test("tiptap's internal checkbox list structure gets correctly parsed", () => {
441+
expect(
442+
normalizeHtml(
443+
`<ul data-type="checkboxList"><li data-checked="true" data-type="checkboxItem"><label>` +
444+
`<input type="checkbox" checked="checked"><span></span></label><div><p>first</p></div></li>` +
445+
`<li data-checked="false" data-type="checkboxItem"><label><input type="checkbox"><span></span></label><div><p>second</p></div></li></ul>`
446+
)
447+
).toBe(
448+
'<ul data-type="checkbox"><li checked>first</li><li>second</li></ul>'
449+
);
450+
});
451+
});
452+
453+
describe('Checkbox Lists (Google Docs & MS Word)', () => {
454+
test.each([
455+
// Google Docs format
456+
[
457+
'<ul><li role="checkbox" aria-checked="true"><img src="data:image/png;base64,..." /><p>Checked</p></li><li role="checkbox" aria-checked="false"><img src="data:image/png;base64,..." /><p>Unchecked</p></li></ul>',
458+
'<ul data-type="checkbox"><li checked>Checked</li><li>Unchecked</li></ul>',
459+
],
460+
// MS Word format
461+
[
462+
'<ul><li class="OutlineElement checklist" data-leveltext="\uF0FE">Checked</li><li class="OutlineElement checklist" data-leveltext="\uF0A8">Unchecked</li></ul>',
463+
'<ul data-type="checkbox"><li checked>Checked</li><li>Unchecked</li></ul>',
464+
],
465+
])('%s → %s', (input, expected) => {
466+
expect(normalizeHtml(input)).toBe(expected);
467+
});
468+
});
469+
420470
describe('BrRemappings', () => {
421471
test('inline collapses around <br> stay flat', () => {
422472
expect(

‎src/web/normalization/htmlNormalizer.ts‎

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -249,12 +249,16 @@ function emitAttributes(el: Element, name: string): string {
249249
emitOneAttr(el, 'width') +
250250
emitOneAttr(el, 'height')
251251
);
252-
case 'ul': {
253-
const val = el.getAttribute('data-type');
254-
return val === 'checkbox' ? ' data-type="checkbox"' : '';
255-
}
252+
case 'ul':
253+
return isCheckboxList(el) ? ' data-type="checkbox"' : '';
256254
case 'li':
257-
return el.hasAttribute('checked') ? ' checked' : '';
255+
// "" is U+F0FE (MS Word checked box); often encoded as "\xEF\x83\xBE" in UTF-8.
256+
const isChecked =
257+
el.hasAttribute('checked') ||
258+
el.getAttribute('data-checked') === 'true' ||
259+
el.getAttribute('aria-checked') === 'true' ||
260+
el.getAttribute('data-leveltext') === ''; // MS Word checked box
261+
return isChecked ? ' checked' : '';
258262
case 'mention':
259263
return (
260264
emitOneAttr(el, 'id') +
@@ -266,6 +270,32 @@ function emitAttributes(el: Element, name: string): string {
266270
}
267271
}
268272

273+
function isCheckboxList(el: Element): boolean {
274+
if (
275+
el.getAttribute('data-type') === 'checkbox' ||
276+
el.getAttribute('data-type') === 'checkboxList'
277+
) {
278+
return true;
279+
}
280+
281+
// In Google Docs and MS Word the <li> elements define if it is a checkbox
282+
// list. We only need to check the first <li>.
283+
const firstLi = Array.from(el.children).find(
284+
(c) => c.tagName.toLowerCase() === 'li'
285+
);
286+
if (firstLi) {
287+
const role = firstLi.getAttribute('role');
288+
const className = firstLi.getAttribute('class') || '';
289+
290+
// Matches Google Docs (role="checkbox") OR MS Word (class includes "checklist")
291+
if (role === 'checkbox' || className.includes('checklist')) {
292+
return true;
293+
}
294+
}
295+
296+
return false;
297+
}
298+
269299
function isGoogleDocsWrapper(el: Element, tag: string): boolean {
270300
if (tag !== 'b') return false;
271301
const id = el.getAttribute('id');
@@ -340,6 +370,7 @@ type LiCtx = {
340370
el: Element;
341371
styles: CssStyles;
342372
nestedLists: Element[];
373+
hasEmitted: boolean;
343374
};
344375

345376
function flushLiBuffer(
@@ -354,6 +385,7 @@ function flushLiBuffer(
354385
out.buf += emitStylesClose(ctx.styles);
355386
out.buf += '</li>';
356387
ib.buf = '';
388+
ctx.hasEmitted = true;
357389
}
358390

359391
function flattenLiChildren(
@@ -378,6 +410,15 @@ function flattenLiNode(
378410
return;
379411
}
380412
if (!isElement(node)) return;
413+
414+
if (tagName(node) === 'img') {
415+
const role = ctx.el.getAttribute('role');
416+
// strip the <img> that Google Docs uses for the display of a checkbox icon
417+
if (role === 'checkbox') {
418+
return;
419+
}
420+
}
421+
381422
if (isListNode(node)) {
382423
ctx.nestedLists.push(node);
383424
return;
@@ -573,9 +614,15 @@ function walkNode(node: Node, out: { buf: string }): void {
573614
if (outName === 'li') {
574615
const nestedLists: Element[] = [];
575616
const liIb = { buf: '' };
576-
const ctx: LiCtx = { el: node, styles: es, nestedLists };
617+
const ctx: LiCtx = { el: node, styles: es, nestedLists, hasEmitted: false };
577618
flattenLiChildren(node, liIb, out, ctx);
578619
flushLiBuffer(liIb, out, ctx);
620+
621+
// if nothing emitted - the <li> is empty, we add it manually
622+
if (!ctx.hasEmitted) {
623+
out.buf += `<li${emitAttributes(ctx.el, 'li')}></li>`;
624+
}
625+
579626
for (const nl of nestedLists) walkChildren(nl, out);
580627
return;
581628
}

0 commit comments

Comments
 (0)