From 92b95fca204ebd18b75e8a7d1ac9a7ce5882cde0 Mon Sep 17 00:00:00 2001 From: Felix Boehm <188768+fb55@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:45:22 +0000 Subject: [PATCH 1/2] Add tokenizer regressions for surrogates and buffered lookahead --- tokenizer/parse5-regressions.test | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tokenizer/parse5-regressions.test diff --git a/tokenizer/parse5-regressions.test b/tokenizer/parse5-regressions.test new file mode 100644 index 0000000..0436bb4 --- /dev/null +++ b/tokenizer/parse5-regressions.test @@ -0,0 +1,64 @@ +{ + "tests": [ + { + "description": "Consecutive low surrogates followed by a valid surrogate pair", + "doubleEscaped": true, + "input": "\\uDC00\\uDC00\\uD83D\\uDE00", + "output": [ + [ + "Character", + "\\uDC00\\uDC00\\uD83D\\uDE00" + ] + ], + "errors": [ + { + "code": "surrogate-in-input-stream", + "line": 1, + "col": 1 + }, + { + "code": "surrogate-in-input-stream", + "line": 1, + "col": 2 + } + ] + }, + { + "description": "Two-code-point named character reference followed by text and a start tag", + "input": " <⃒text", + "output": [ + [ + "Character", + " <\u20d2text" + ], + [ + "StartTag", + "a", + {} + ] + ] + }, + { + "description": "Tag open lookahead after a long whitespace run", + "input": " <", + "output": [ + [ + "Character", + " <" + ], + [ + "StartTag", + "x", + {} + ] + ], + "errors": [ + { + "code": "invalid-first-character-of-tag-name", + "line": 1, + "col": 65539 + } + ] + } + ] +} From 40535fd96c388170212527fba59ca8da0676e718 Mon Sep 17 00:00:00 2001 From: Felix Boehm <188768+fb55@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:11:52 +0000 Subject: [PATCH 2/2] Keep UTF-16 surrogate regressions in the surrogate-specific suite --- tokenizer/parse5-regressions.test | 23 ----------------------- tokenizer/unicodeCharsProblematic.test | 26 +++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tokenizer/parse5-regressions.test b/tokenizer/parse5-regressions.test index 0436bb4..f11bd37 100644 --- a/tokenizer/parse5-regressions.test +++ b/tokenizer/parse5-regressions.test @@ -1,28 +1,5 @@ { "tests": [ - { - "description": "Consecutive low surrogates followed by a valid surrogate pair", - "doubleEscaped": true, - "input": "\\uDC00\\uDC00\\uD83D\\uDE00", - "output": [ - [ - "Character", - "\\uDC00\\uDC00\\uD83D\\uDE00" - ] - ], - "errors": [ - { - "code": "surrogate-in-input-stream", - "line": 1, - "col": 1 - }, - { - "code": "surrogate-in-input-stream", - "line": 1, - "col": 2 - } - ] - }, { "description": "Two-code-point named character reference followed by text and a start tag", "input": " <⃒text", diff --git a/tokenizer/unicodeCharsProblematic.test b/tokenizer/unicodeCharsProblematic.test index 3ddb96c..d7a6e76 100644 --- a/tokenizer/unicodeCharsProblematic.test +++ b/tokenizer/unicodeCharsProblematic.test @@ -36,6 +36,30 @@ "output":[["Character", "\n\u0000"]], "errors":[ { "code": "unexpected-null-character", "line": 2, "col": 1 } -]} +]}, + +{ + "description": "Consecutive low surrogates followed by a valid surrogate pair", + "doubleEscaped": true, + "input": "\\uDC00\\uDC00\\uD83D\\uDE00", + "output": [ + [ + "Character", + "\\uDC00\\uDC00\\uD83D\\uDE00" + ] + ], + "errors": [ + { + "code": "surrogate-in-input-stream", + "line": 1, + "col": 1 + }, + { + "code": "surrogate-in-input-stream", + "line": 1, + "col": 2 + } + ] +} ] } \ No newline at end of file