Skip to content

Commit 83c3c79

Browse files
committed
Strip inline comments before interpreting Unicode data fields
Assisted-By: GPT-6
1 parent 7dbd8d9 commit 83c3c79

3 files changed

Lines changed: 63 additions & 8 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ const parseBlocksScriptsProperties = async (type, version) => {
4747
}
4848
const lines = source.split('\n');
4949
for (const line of lines) {
50-
const trimmed = line.trim();
50+
const trimmed = line.split('#')[0].trim();
5151
if (!trimmed || trimmed.startsWith('#')) {
5252
continue;
5353
}
@@ -57,10 +57,7 @@ const parseBlocksScriptsProperties = async (type, version) => {
5757
continue;
5858
}
5959
const charRange = data[0].replace('..', '-').trim();
60-
let item = data[1]
61-
.split(type === 'blocks' ? ';' : '#')[0]
62-
.trim()
63-
.replace(/\x20/g, '_');
60+
let item = data[1].trim().replace(/\x20/g, '_');
6461
if (type === 'derived-normalization-properties') {
6562
if (item === 'FNC') {
6663
// Old Unicode versions up to v4.0.0 use the `FNC` alias instead of
@@ -297,10 +294,13 @@ const parseBlocks = async (version) => {
297294
}
298295
const map = {};
299296
for (const line of source.trimEnd().split('\n')) {
300-
if (line.startsWith('#')) {
297+
const record = line.split('#')[0].trim();
298+
if (!record) {
301299
continue;
302300
}
303-
const [start, end, blockName] = line.split('; ');
301+
const [start, end, blockName] = record
302+
.split(';')
303+
.map((field) => field.trim());
304304
const canonicalBlockName = looseMatch('Block', blockName).value;
305305
map[canonicalBlockName] = regenerate().addRange(
306306
parseInt(start, 16),

‎scripts/parse-name-aliases.mjs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ const parseNameAliases = async (version) => {
99
const lines = source.split('\n');
1010

1111
for (const line of lines) {
12-
const data = line.trim().split(';');
12+
const record = line.split('#')[0].trim();
13+
if (!record) {
14+
continue;
15+
}
16+
const data = record.split(';').map((field) => field.trim());
1317
const codePoint = parseInt(data[0], 16);
1418
const name = data[1];
1519
const type = data[2];

‎tests/parser-formats.mjs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ import parseSpecialCasing from '../scripts/parse-special-casing.mjs';
33
import parseEmoji from '../scripts/parse-emoji.mjs';
44
import parseEmojiSequences from '../scripts/parse-emoji-sequences.mjs';
55
import parseBidiBrackets from '../scripts/parse-bidi-brackets.mjs';
6+
import parseNameAliases from '../scripts/parse-name-aliases.mjs';
7+
import {
8+
parseBlocks,
9+
parseDerivedCoreProperties,
10+
} from '../scripts/parse-blocks-scripts-properties.mjs';
611
import utils from '../scripts/utils.mjs';
712

813
test('SpecialCasing accepts optional comments and flexible whitespace', async (t) => {
@@ -28,6 +33,52 @@ FB00 ; FB00 ; 0046 0066 ; 0046 0046 ; # comment with # and ; characters
2833
]);
2934
});
3035

36+
test('Semicolons inside comments do not turn binary records into enumerated records', async (t) => {
37+
t.mock.method(
38+
utils,
39+
'readDataFile',
40+
() => `
41+
0041 ; Alphabetic # note; still a comment
42+
0300 ; InCB; Extend # an actual enumerated property
43+
`,
44+
);
45+
const map = await parseDerivedCoreProperties('18.0.0');
46+
t.assert.deepEqual(Object.keys(map), ['Alphabetic']);
47+
t.assert.deepEqual(map.Alphabetic.toArray(), [0x41]);
48+
});
49+
50+
test('Block names exclude inline comments in modern and legacy formats', async (t) => {
51+
t.mock.method(utils, 'readDataFile', (version) =>
52+
version === '3.0.1'
53+
? '\n # comment\n0000;007F;Basic Latin # ASCII; block\n'
54+
: '\n # comment\n0000..007F;Basic Latin # ASCII; block\n',
55+
);
56+
for (const version of ['3.0.1', '18.0.0']) {
57+
const map = await parseBlocks(version);
58+
t.assert.deepEqual(Object.keys(map), ['Basic_Latin']);
59+
t.assert.ok(map.Basic_Latin.contains(0));
60+
t.assert.ok(map.Basic_Latin.contains(0x7f));
61+
t.assert.ok(!map.Basic_Latin.contains(0x80));
62+
}
63+
});
64+
65+
test('Name aliases ignore inline comments without splitting alias groups', async (t) => {
66+
t.mock.method(
67+
utils,
68+
'readDataFile',
69+
() => `
70+
# comment
71+
000A;LINE FEED;control # note; still a comment
72+
000A ; NEW LINE ; control
73+
000A;LF;abbreviation # another name
74+
`,
75+
);
76+
t.assert.deepEqual(await parseNameAliases('18.0.0'), {
77+
control: { 10: ['LINE FEED', 'NEW LINE'] },
78+
abbreviation: { 10: ['LF'] },
79+
});
80+
});
81+
3182
test('Emoji properties accept compact and padded field separators', async (t) => {
3283
let source;
3384
t.mock.method(utils, 'readDataFile', () => source);

0 commit comments

Comments
 (0)