Skip to content

Commit a05644a

Browse files
committed
Parse SpecialCasing comments and whitespace safely
Assisted-By: GPT-6
1 parent c3d57d5 commit a05644a

3 files changed

Lines changed: 55 additions & 22 deletions

File tree

‎scripts/parse-special-casing.mjs‎

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,26 @@ const parseSpecialCasing = async (version) => {
77
return;
88
}
99
for (const line of source.split('\n')) {
10-
if (line === '' || line.startsWith('#')) {
10+
const record = line.split('#')[0].trim();
11+
if (!record) {
1112
continue;
1213
}
13-
const data = line.slice(0, line.lastIndexOf('#')).split('; ');
14+
const data = record.split(';').map((field) => field.trim());
1415
const from = parseInt(data[0], 16);
15-
16-
const lowercase =
17-
data[1] === ''
18-
? []
19-
: data[1].split(' ').map((codePoint) => {
20-
return parseInt(codePoint, 16);
21-
}); // Note: this could be zero or two characters!
22-
const titlecase =
23-
data[2] === ''
24-
? []
25-
: data[2].split(' ').map((codePoint) => {
26-
return parseInt(codePoint, 16);
27-
}); // Note: this could be zero or two characters!
28-
const uppercase =
29-
data[3] === ''
30-
? []
31-
: data[3].split(' ').map((codePoint) => {
32-
return parseInt(codePoint, 16);
33-
}); // Note: this could be zero or two characters!
16+
const [lowercase, titlecase, uppercase] = data
17+
.slice(1, 4)
18+
.map((mapping) =>
19+
mapping === ''
20+
? []
21+
: mapping.split(/\s+/).map((codePoint) => parseInt(codePoint, 16)),
22+
);
3423

3524
// Conditions can contain a language ID defined by BCP 47 and casing contexts
3625
// defined in Table 3-17 Context Specification for Casing:
3726
// https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G54277
3827
// Because the language ID may contain `-`, we let `--` be the separator
3928
// between the language ID and the casing contexts.
40-
const conditions = data[4].trim().replaceAll(' ', '--');
29+
const conditions = (data[4] ?? '').replace(/\s+/g, '--');
4130
const storageKeyPostfix = conditions ? '--' + conditions : '';
4231
specialCasingMap['Lowercase' + storageKeyPostfix] ??= new Map();
4332
specialCasingMap['Lowercase' + storageKeyPostfix].set(from, lowercase);

‎tests/parser-formats.mjs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { test } from 'node:test';
2+
import parseSpecialCasing from '../scripts/parse-special-casing.mjs';
3+
import utils from '../scripts/utils.mjs';
4+
5+
test('SpecialCasing accepts optional comments and flexible whitespace', async (t) => {
6+
t.mock.method(
7+
utils,
8+
'readDataFile',
9+
() => `
10+
# Comment-only lines and whitespace-only lines are ignored.
11+
\t
12+
00DF;00DF;0053 0073;0053\t0053;
13+
FB00 ; FB00 ; 0046 0066 ; 0046 0046 ; # comment with # and ; characters
14+
`,
15+
);
16+
const map = await parseSpecialCasing('18.0.0');
17+
t.assert.deepEqual(map.Uppercase.get(0xdf), [0x53, 0x53]);
18+
t.assert.deepEqual(map.Titlecase.get(0xdf), [0x53, 0x73]);
19+
t.assert.deepEqual(map.Lowercase.get(0xdf), [0xdf]);
20+
t.assert.deepEqual(map.Uppercase.get(0xfb00), [0x46, 0x46]);
21+
t.assert.deepEqual(Object.keys(map).sort(), [
22+
'Lowercase',
23+
'Titlecase',
24+
'Uppercase',
25+
]);
26+
});
27+
28+
test('SpecialCasing preserves empty mappings and normalizes condition separators', async (t) => {
29+
t.mock.method(
30+
utils,
31+
'readDataFile',
32+
() => '0307;0307;;;lt After_Soft_Dotted;\n',
33+
);
34+
const map = await parseSpecialCasing('18.0.0');
35+
t.assert.deepEqual(
36+
map['Lowercase--lt--After_Soft_Dotted'].get(0x307),
37+
[0x307],
38+
);
39+
t.assert.deepEqual(map['Titlecase--lt--After_Soft_Dotted'].get(0x307), []);
40+
t.assert.deepEqual(map['Uppercase--lt--After_Soft_Dotted'].get(0x307), []);
41+
});

‎tests/tests.mjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ if (!process.env.CI) {
1313
await generateData(newest);
1414
}
1515

16+
// Format fixtures mock the data reader, so register them after generation.
17+
await import('./parser-formats.mjs');
18+
1619
suite(`The generated Unicode ${oldest} js`, () => {
1720
test('General_Category/Modifier_Letter should match the snapshot', async (t) => {
1821
const { default: data } = await import(

0 commit comments

Comments
 (0)