Skip to content

Commit 44481fb

Browse files
JLHwungmathiasbynens
JLHwung
authored and
mathiasbynens
committed
Ignore whitespace-only lines and indented parser comments
Normalize line guards before splitting fields and recognize padded blank lines when skipping legacy property sections. Check every parser against padded input for all supported Unicode versions. Assisted-By: GPT-6 Issue: #110
1 parent e723c07 commit 44481fb

12 files changed

Lines changed: 90 additions & 58 deletions

‎scripts/parse-bidi-class.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,11 @@ const parseBidiClass = async (version) => {
8888
let flag = false;
8989
let first = 0;
9090
for (const line of lines) {
91-
if (line === '' || line.startsWith('#')) {
91+
const trimmed = line.trim();
92+
if (!trimmed || trimmed.startsWith('#')) {
9293
continue;
9394
}
94-
const data = line.trim().split(';');
95+
const data = trimmed.split(';');
9596
const codePoint = parseInt(data[0], 16);
9697
const name = data[1];
9798
const bidiCategory = data[4] === '' ? undefined : bidiAliases.get(data[4]);

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,11 @@ const parseDerivedBinaryProperties = async (version) => {
136136
}
137137
const result = [];
138138
for (const line of source.split('\n')) {
139-
if (line === '' || line.startsWith('#')) {
139+
const trimmed = line.trim();
140+
if (!trimmed || trimmed.startsWith('#')) {
140141
continue;
141142
}
142-
const data = line.split(';');
143+
const data = trimmed.split(';');
143144
if (data[9] === 'Y') {
144145
const codePoint = parseInt(data[0], 16);
145146
result.push(codePoint);
@@ -182,10 +183,11 @@ const parseDerivedGeneralCategory = async (version) => {
182183
// http://unicode.org/reports/tr44/#GC_Values_Table
183184
// http://unicode.org/reports/tr18/#Categories
184185
for (const line of lines) {
185-
if (line === '' || line.startsWith('#')) {
186+
const trimmed = line.trim();
187+
if (!trimmed || trimmed.startsWith('#')) {
186188
continue;
187189
}
188-
const data = line.trim().split(';');
190+
const data = trimmed.split(';');
189191
const codePoint = parseInt(data[0], 16);
190192
const name = data[1];
191193
const generalCategory = data[2];
@@ -339,7 +341,7 @@ const parseProperties = async (version) => {
339341
const lines = source.trimEnd().split('\n');
340342
let currentProperty, maybeProperty, maybeRange;
341343
for (let i = 0; i < lines.length; i++) {
342-
const line = lines[i];
344+
const line = lines[i].trim();
343345
if ((maybeProperty = /0x[A-F\d]+ \((.+?)\)$/.exec(line)) != null) {
344346
currentProperty = maybeProperty[1];
345347
if (
@@ -350,7 +352,7 @@ const parseProperties = async (version) => {
350352
) {
351353
// Skip contents.
352354
i++;
353-
while (++i < lines.length && lines[i] !== '');
355+
while (++i < lines.length && lines[i].trim() !== '');
354356
continue;
355357
}
356358
if (currentProperty === 'Not a Character') {

‎scripts/parse-case-folding.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ const parseCaseFolding = async (version) => {
88
}
99
const lines = source.split('\n');
1010
for (const line of lines) {
11-
if (!line || /^#/.test(line)) {
11+
const trimmed = line.trim();
12+
if (!trimmed || trimmed.startsWith('#')) {
1213
continue;
1314
}
14-
const data = line.trim().split(';');
15+
const data = trimmed.split(';');
1516
const codePoint = parseInt(data[0], 16);
1617
const status = data[1].trim();
1718
const mappings = data[2]

‎scripts/parse-composition-exclusions.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,11 @@ const parseCompositionExclusions = async (version) => {
99
const result = regenerate();
1010
const lines = source.split('\n');
1111
for (const line of lines) {
12-
if (!line || line.startsWith('#')) {
12+
const trimmed = line.trim();
13+
if (!trimmed || trimmed.startsWith('#')) {
1314
continue;
1415
}
15-
const data = line.trim().split('#');
16+
const data = trimmed.split('#');
1617
const charRange = data[0].replace('..', '-').trim();
1718
const rangeParts = charRange.split('-');
1819
if (rangeParts.length === 2) {

‎scripts/parse-grapheme-word-sentence-break.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@ const parseGraphemeWordSentenceBreak = async (version, kind) => {
1212
};
1313
const lines = source.split('\n');
1414
for (const line of lines) {
15-
if (!line || /^#/.test(line)) {
15+
const trimmed = line.trim();
16+
if (!trimmed || trimmed.startsWith('#')) {
1617
continue;
1718
}
18-
const data = line.trim().split(';');
19+
const data = trimmed.split(';');
1920
const charRange = data[0].replace('..', '-').trim();
2021
const rangeParts = charRange.split('-');
2122
const value = data[1].split('#')[0].trim();

‎scripts/parse-indic-positional-category.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@ const parseIndicPositionalCategory = async (version) => {
1313
};
1414
const lines = source.split('\n');
1515
for (const line of lines) {
16-
if (!line || /^#/.test(line)) {
16+
const trimmed = line.trim();
17+
if (!trimmed || trimmed.startsWith('#')) {
1718
continue;
1819
}
19-
const data = line.trim().split(';');
20+
const data = trimmed.split(';');
2021
const charRange = data[0].replace('..', '-').trim();
2122
const rangeParts = charRange.split('-');
2223
const value = data[1].split('#')[0].trim();

‎scripts/parse-indic-syllabic-category.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@ const parseIndicSyllabicCategory = async (version) => {
1212
};
1313
const lines = source.split('\n');
1414
for (const line of lines) {
15-
if (!line || /^#/.test(line)) {
15+
const trimmed = line.trim();
16+
if (!trimmed || trimmed.startsWith('#')) {
1617
continue;
1718
}
18-
const data = line.trim().split(';');
19+
const data = trimmed.split(';');
1920
const charRange = data[0].replace('..', '-').trim();
2021
const rangeParts = charRange.split('-');
2122
const value = data[1].split('#')[0].trim();

‎scripts/parse-line-break.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@ const parseLineBreak = async (version) => {
2626
};
2727
const lines = source.split('\n');
2828
for (const line of lines) {
29-
if (!line || /^#/.test(line)) {
29+
const trimmed = line.trim();
30+
if (!trimmed || trimmed.startsWith('#')) {
3031
continue;
3132
}
32-
const data = line.trim().split(';');
33+
const data = trimmed.split(';');
3334
const charRange = data[0].replace('..', '-').trim();
3435
const rangeParts = charRange.split('-');
3536
const value = data[1].split('#')[0].trim();

‎scripts/parse-names.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,11 @@ const parseNames = async (version) => {
110110
let flag = false;
111111
let first = 0;
112112
for (const line of lines) {
113-
if (line === '' || line.startsWith('#')) {
113+
const trimmed = line.trim();
114+
if (!trimmed || trimmed.startsWith('#')) {
114115
continue;
115116
}
116-
const data = line.trim().split(';');
117+
const data = trimmed.split(';');
117118
const codePoint = parseInt(data[0], 16);
118119
const name = data[1];
119120
if (flag) {

‎scripts/parse-script-extensions.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ const parseScriptExtensions = async (version, scriptsMap) => {
1919

2020
const lines = source.split('\n');
2121
for (const line of lines) {
22-
if (!line || /^#/.test(line)) {
22+
const trimmed = line.trim();
23+
if (!trimmed || trimmed.startsWith('#')) {
2324
continue;
2425
}
25-
const data = line.trim().split(';');
26+
const data = trimmed.split(';');
2627
const charRange = data[0].replace('..', '-').trim();
2728
const rangeParts = charRange.split('-');
2829
const scripts = data[1].split('#')[0].trim().split(' ');

0 commit comments

Comments
 (0)