Skip to content

fix: whitespaces parsing inside normalizer #704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kacperzolkiewski merged 6 commits into from
Jul 17, 2026
Merged

fix: whitespaces parsing inside normalizer #704

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
566900a
fix: whitespaces parsing inside normalizer
kacperzolkiewski Jul 16, 2026
9e965c9
fix(web): parsing whitespaces in normalizer
kacperzolkiewski Jul 16, 2026
d5b5f76
fix: add buffer cleaning
kacperzolkiewski Jul 16, 2026
59a9cc0
Update src/web/normalization/htmlNormalizer.ts
kacperzolkiewski Jul 16, 2026
e15ac36
Merge branch 'main' into @kacperzolkiewski/fix-whitespaces-parsing
kacperzolkiewski Jul 16, 2026
9b32d08
fix: code review fixes
kacperzolkiewski Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions cpp/parser/GumboNormalizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,8 +497,23 @@ static void walk_node(GumboNode *node, buffer_t *out);

static void flatten_bq_node(GumboNode *node, buffer_t *ib, buffer_t *out);

/** True if buf is empty or contains only ASCII whitespace. */
static bool is_whitespace_only(const char *data, size_t len) {
for (size_t i = 0; i < len; i++) {
unsigned char c = (unsigned char)data[i];
if (c != ' ' && c != '\t' && c != '\n' && c != '\r' && c != '\f')
return false;
}
return true;
}

/**
* Flush buffered inline content as a <p>. Inter-block whitespace (newlines /
* spaces between block tags in pretty-printed HTML) is discarded so it does
* not become empty paragraphs that later serialize as extra <br>s.
*/
static void flush_inline_p(buffer_t *ib, buffer_t *out) {
if (ib->len > 0) {
if (ib->len > 0 && !is_whitespace_only(ib->data, ib->len)) {
buffer_append_str(out, "<p>");
buffer_append(out, ib->data, ib->len);
buffer_append_str(out, "</p>");
Comment thread
kacperzolkiewski marked this conversation as resolved.
Outdated
Expand Down Expand Up @@ -674,7 +689,8 @@ static void walk_children(GumboNode *node, buffer_t *out) {
!is_blockquote_node(children->data[i])) {
child = children->data[i];
if (is_br_node(child)) {
if (ib.len > 0)
/* Whitespace-only buffer is layout noise; treat like empty → <br> */
if (ib.len > 0 && !is_whitespace_only(ib.data, ib.len))
flush_inline_p(&ib, out);
else
buffer_append_str(out, "<br>");
Comment thread
kacperzolkiewski marked this conversation as resolved.
Expand Down
27 changes: 27 additions & 0 deletions cpp/tests/GumboParserTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,3 +509,30 @@ TEST(GumboParserTest, BrRemappings) {
"<p><b>Asdasdasd</b></p><br><br><p>Sent with <a "
"href=\"https://google.com\">Net</a></p>");
}

TEST(GumboParserTest, InterBlockWhitespace) {
// Pretty-printed consecutive paragraphs must not gain empty <p>s from the
// newlines between them (those would later serialize as extra <br>s).
EXPECT_EQ(GumboParser::normalizeHtml(
"<p>Asdasd</p>\n<p>Asdasd</p>\n<p>Asdasda</p>"),
"<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>");
EXPECT_EQ(GumboParser::normalizeHtml(
"<p>Asdasd</p>\n\n<p>Asdasd</p>\n\n<p>Asdasda</p>"),
"<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>");
EXPECT_EQ(GumboParser::normalizeHtml(
"<html>\n<p>Asdasd</p>\n<p>Asdasd</p>\n<p>Asdasda</p>\n</html>"),
"<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>");
EXPECT_EQ(GumboParser::normalizeHtml("<p>Asdasd</p> <p>Asdasd</p>"),
"<p>Asdasd</p><p>Asdasd</p>");

// Significant inline content between blocks is still wrapped in <p>.
EXPECT_EQ(GumboParser::normalizeHtml("<p>a</p> hello <p>b</p>"),
"<p>a</p><p> hello </p><p>b</p>");

// Spaces inside text / between inlines must be preserved.
EXPECT_EQ(GumboParser::normalizeHtml("hello world"), "hello world");
EXPECT_EQ(GumboParser::normalizeHtml("<p>hello world</p>"),
"<p>hello world</p>");
EXPECT_EQ(GumboParser::normalizeHtml("<b>hello</b> <i>world</i>"),
"<b>hello</b> <i>world</i>");
}
28 changes: 28 additions & 0 deletions src/web/__tests__/htmlNormalizer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,4 +497,32 @@ describe('htmlNormalizer', () => {
expect(normalizeHtml(input)).toBe(expected);
});
});

describe('InterBlockWhitespace', () => {
// Pretty-printed consecutive paragraphs must not gain empty <p>s from the
// newlines between them (those would later serialize as extra <br>s).
test.each([
[
'<p>Asdasd</p>\n<p>Asdasd</p>\n<p>Asdasda</p>',
'<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>',
],
[
'<p>Asdasd</p>\n\n<p>Asdasd</p>\n\n<p>Asdasda</p>',
'<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>',
],
[
'<html>\n<p>Asdasd</p>\n<p>Asdasd</p>\n<p>Asdasda</p>\n</html>',
'<p>Asdasd</p><p>Asdasd</p><p>Asdasda</p>',
],
['<p>Asdasd</p> <p>Asdasd</p>', '<p>Asdasd</p><p>Asdasd</p>'],
// Significant inline content between blocks is still wrapped in <p>.
['<p>a</p> hello <p>b</p>', '<p>a</p><p> hello </p><p>b</p>'],
// Spaces inside text / between inlines must be preserved.
['hello world', 'hello world'],
['<p>hello world</p>', '<p>hello world</p>'],
['<b>hello</b> <i>world</i>', '<b>hello</b> <i>world</i>'],
])('%s → %s', (input, expected) => {
expect(normalizeHtml(input)).toBe(expected);
});
});
});
24 changes: 21 additions & 3 deletions src/web/normalization/htmlNormalizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,11 +323,27 @@ function escapeText(s: string): string {

// --- Blockquote content flattening ---

function isWhitespaceOnly(value: string): boolean {
for (let i = 0; i < value.length; i++) {
const c = value.charCodeAt(i);
// space, tab, LF, CR, FF — mirrors GumboNormalizer.c
Comment thread
kacperzolkiewski marked this conversation as resolved.
Outdated
if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d && c !== 0x0c) {
return false;
}
}
return true;
}

/**
* Flush buffered inline content as a <p>. Inter-block whitespace (newlines /
* spaces between block tags in pretty-printed HTML) is discarded so it does
* not become empty paragraphs that later serialize as extra <br>s.
*/
function flushInlineP(ib: { buf: string }, out: { buf: string }): void {
if (ib.buf.length > 0) {
if (ib.buf.length > 0 && !isWhitespaceOnly(ib.buf)) {
out.buf += `<p>${ib.buf}</p>`;
ib.buf = '';
}
ib.buf = '';
}

function flattenBqChildren(
Expand Down Expand Up @@ -493,9 +509,11 @@ function walkChildren(node: Element, out: { buf: string }): void {
break;
}
if (isElement(cur) && isBrNode(cur)) {
if (ib.buf.length > 0) {
// Whitespace-only buffer is layout noise; treat like empty → <br>
if (ib.buf.length > 0 && !isWhitespaceOnly(ib.buf)) {
flushInlineP(ib, out);
} else {
ib.buf = '';
out.buf += '<br>';
}
i++;
Expand Down
Loading