Skip to content

Commit 45d0131

Browse files
JLHwungmathiasbynens
JLHwung
authored and
mathiasbynens
committed
Confine historical data workarounds to the versions that need them
Add utils.isVersionBefore and guard each workaround for historical data files with the range of releases it applies to, so parsing current versions skips them: - case-insensitive property names such as White_space: before 3.2.0 - FNC and NFC_NO-style Quick_Check records: before 4.0.1 - uppercase Scripts.txt names such as LATIN: before 4.0.1 - trailing spaces in UnicodeData names: before 4.0.1 - two-field NameAliases records: before 6.1.0 - the LineBreak AL default for assigned code points: before 4.0.0 - First/Last range records in LineBreak: before 3.1.0 Every parser produces identical output for every version in data/. Assisted-By: Claude Opus 5.5 Issue: #110
1 parent 37bd6b5 commit 45d0131

6 files changed

Lines changed: 123 additions & 26 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,14 @@ const canonicalPropertyNames = new Map(
1111
[...propertyAliases.values()].map((name) => [name.toLowerCase(), name]),
1212
);
1313

14-
const findCanonicalName = (shortName) => {
14+
const findCanonicalName = (shortName, version) => {
1515
const canonicalName =
1616
propertyAliases.get(shortName) ??
17-
canonicalPropertyNames.get(shortName.toLowerCase());
17+
// Unicode 3.1.1 and earlier spell some names differently, such as
18+
// `White_space` for `White_Space`.
19+
(utils.isVersionBefore(version, '3.2.0')
20+
? canonicalPropertyNames.get(shortName.toLowerCase())
21+
: undefined);
1822
if (!canonicalName) {
1923
return shortName;
2024
}
@@ -59,14 +63,17 @@ const parseBlocksScriptsProperties = async (type, version) => {
5963
const charRange = data[0].replace('..', '-').trim();
6064
let item = data[1].trim().replace(/\x20/g, '_');
6165
if (type === 'derived-normalization-properties') {
62-
if (item === 'FNC' || /^NFK?[CD]_(?:NO|MAYBE)$/.test(item)) {
66+
if (
67+
utils.isVersionBefore(version, '4.0.1') &&
68+
(item === 'FNC' || /^NFK?[CD]_(?:NO|MAYBE)$/.test(item))
69+
) {
6370
// Old Unicode versions up to v4.0.0 use the `FNC` alias instead of
6471
// `FC_NFKC` (for `FC_NFKC_Closure`), and record Quick_Check values
6572
// as e.g. `NFC_NO` or `NFC_MAYBE` instead of `NFC_QC; N`. These are
6673
// not binary properties.
6774
continue;
6875
} else {
69-
const canonical = findCanonicalName(item);
76+
const canonical = findCanonicalName(item, version);
7077
if (canonical) {
7178
if (
7279
/FC_NFKC_Closure|^NFKC_(?:Simple_)?Casefold$|(?:NFC|NFD|NFKC|NFKD)_Quick_Check/.test(
@@ -94,13 +101,16 @@ const parseBlocksScriptsProperties = async (type, version) => {
94101
} else if (type === 'derived-general-category') {
95102
item = categoryAliases.get(item) || item;
96103
} else if (type === 'scripts') {
97-
// Older Scripts.txt files use names such as LATIN and OLD_ITALIC.
98-
item = looseMatch('Script', item).value;
99-
if (!item) {
100-
throw new Error(`Canonical script name not found in ${line}.`);
104+
if (utils.isVersionBefore(version, '4.0.1')) {
105+
// Scripts.txt up to Unicode 4.0.0 uses names such as LATIN and
106+
// OLD_ITALIC. Later files use the canonical names.
107+
item = looseMatch('Script', item).value;
108+
if (!item) {
109+
throw new Error(`Canonical script name not found in ${line}.`);
110+
}
101111
}
102112
} else {
103-
item = findCanonicalName(item);
113+
item = findCanonicalName(item, version);
104114
}
105115
map[item] ??= regenerate();
106116
const rangeParts = charRange.split('-');
@@ -416,7 +426,8 @@ const parseProperties = async (version) => {
416426
currentProperty = 'Noncharacter_Code_Point';
417427
}
418428
currentProperty = currentProperty.replaceAll(' ', '_');
419-
currentProperty = findCanonicalName(currentProperty) ?? currentProperty;
429+
currentProperty =
430+
findCanonicalName(currentProperty, version) ?? currentProperty;
420431
map[currentProperty] = regenerate();
421432
} else if (line.startsWith('Discrepancy ')) {
422433
// Unicode 2.1.8 includes comparisons with UnicodeData after each

‎scripts/parse-line-break.mjs‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ const parseLineBreak = async (version) => {
2828
// Before Unicode 4.0, only unassigned code points default to `XX`:
2929
// “Assigned characters that are not listed explicitly are given the
3030
// value "AL".”
31-
const assignedDefault =
32-
/Assigned characters that are not listed explicitly are given the value\s*#?\s*"(\w+)"/.exec(
33-
source,
34-
)?.[1];
31+
const assignedDefault = utils.isVersionBefore(version, '4.0.0')
32+
? /Assigned characters that are not listed explicitly are given the value\s*#?\s*"(\w+)"/.exec(
33+
source,
34+
)?.[1]
35+
: undefined;
3536
// Unicode 3.0 lists ranges as UnicodeData does, e.g.
3637
// `3400;ID;<CJK Ideograph Extension A, First>` and
3738
// `4DB5;ID;<CJK Ideograph Extension A, Last>`.
39+
const hasFirstLastRanges = utils.isVersionBefore(version, '3.1.0');
3840
let rangeStart;
3941
const lines = source.split('\n');
4042
for (const line of lines) {
@@ -45,21 +47,23 @@ const parseLineBreak = async (version) => {
4547
const data = trimmed.split(';');
4648
const value = data[1].split('#')[0].trim();
4749
const canonicalName = findCanonicalName(value);
48-
const name = data[2]?.split('#')[0].trim() ?? '';
4950
let [from, to = from] = data[0]
5051
.trim()
5152
.split('..')
5253
.map((codePoint) => parseInt(codePoint, 16));
53-
if (/^<.+, First>$/.test(name)) {
54-
rangeStart = from;
55-
continue;
56-
}
57-
if (/^<.+, Last>$/.test(name)) {
58-
if (rangeStart === undefined) {
59-
throw new Error(`Range end without a start: ${line}`);
54+
if (hasFirstLastRanges) {
55+
const name = data[2]?.split('#')[0].trim() ?? '';
56+
if (/^<.+, First>$/.test(name)) {
57+
rangeStart = from;
58+
continue;
59+
}
60+
if (/^<.+, Last>$/.test(name)) {
61+
if (rangeStart === undefined) {
62+
throw new Error(`Range end without a start: ${line}`);
63+
}
64+
from = rangeStart;
65+
rangeStart = undefined;
6066
}
61-
from = rangeStart;
62-
rangeStart = undefined;
6367
}
6468
map[canonicalName] ??= regenerate();
6569
map['Unknown'].removeRange(from, to);

‎scripts/parse-name-aliases.mjs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ const parseNameAliases = async (version) => {
1717
const codePoint = parseInt(data[0], 16);
1818
const name = data[1];
1919
// Before Unicode 6.1, the two-field format lists only name corrections.
20-
const type = data[2] ?? 'correction';
20+
const type = utils.isVersionBefore(version, '6.1.0')
21+
? 'correction'
22+
: data[2];
2123

2224
if (!isNaN(codePoint)) {
2325
if (map[type] === undefined) {

‎scripts/parse-names.mjs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,9 @@ const parseNames = async (version) => {
117117
const data = trimmed.split(';');
118118
const codePoint = parseInt(data[0], 16);
119119
// Unicode 4.0.0 has trailing spaces after the names of U+02AE and U+0615.
120-
const name = data[1].trim();
120+
const name = utils.isVersionBefore(version, '4.0.1')
121+
? data[1].trim()
122+
: data[1];
121123
if (flag) {
122124
if (/<.+, Last>/.test(name)) {
123125
flag = false;

‎scripts/utils.mjs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,21 @@ const readDataFile = (version, type) => {
437437
}
438438
};
439439

440+
// Whether a dotted Unicode version precedes another one, e.g.
441+
// `isVersionBefore('3.1.1', '3.2.0')`. Use it to confine workarounds for
442+
// historical data files to the releases that need them.
443+
const isVersionBefore = (version, target) => {
444+
const a = version.split('.').map(Number);
445+
const b = target.split('.').map(Number);
446+
for (let i = 0; i < Math.max(a.length, b.length); i++) {
447+
const difference = (a[i] ?? 0) - (b[i] ?? 0);
448+
if (difference !== 0) {
449+
return difference < 0;
450+
}
451+
}
452+
return false;
453+
};
454+
440455
// The `typesVersions` map for package.json. TypeScript picks the most specific
441456
// matching pattern, so mapping directories keep their own declarations while
442457
// everything else resolves to the shared ones in `types/`.
@@ -460,6 +475,7 @@ export default {
460475
range,
461476
append,
462477
extend,
478+
isVersionBefore,
463479
readDataFile,
464480
writeFiles,
465481
};

‎tests/parser-formats.mjs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@ import parseSimpleCaseMapping from '../scripts/parse-simple-case-mapping.mjs';
88
import parseJoiningType from '../scripts/parse-joining-type.mjs';
99
import parseCaseFolding from '../scripts/parse-case-folding.mjs';
1010
import parseScriptExtensions from '../scripts/parse-script-extensions.mjs';
11+
import parseLineBreak from '../scripts/parse-line-break.mjs';
12+
import parseNames from '../scripts/parse-names.mjs';
1113
import {
1214
parseBlocks,
1315
parseDerivedCoreProperties,
16+
parseDerivedNormalizationProperties,
1417
parseProperties,
1518
parseScripts,
1619
} from '../scripts/parse-blocks-scripts-properties.mjs';
@@ -324,3 +327,62 @@ test('SpecialCasing preserves empty mappings and normalizes condition separators
324327
t.assert.deepEqual(map['Titlecase--lt--After_Soft_Dotted'].get(0x307), []);
325328
t.assert.deepEqual(map['Uppercase--lt--After_Soft_Dotted'].get(0x307), []);
326329
});
330+
331+
test('Unicode versions compare numerically', (t) => {
332+
t.assert.ok(utils.isVersionBefore('3.1.1', '3.2.0'));
333+
t.assert.ok(utils.isVersionBefore('4.0.0', '4.0.1'));
334+
t.assert.ok(utils.isVersionBefore('9.0.0', '10.0.0'));
335+
t.assert.ok(utils.isVersionBefore('2.0.14', '2.1.2'));
336+
t.assert.ok(!utils.isVersionBefore('4.0.1', '4.0.1'));
337+
t.assert.ok(!utils.isVersionBefore('12.1.0', '12.0.0'));
338+
t.assert.ok(!utils.isVersionBefore('18.0.0', '4.0.1'));
339+
});
340+
341+
test('Workarounds for historical data are skipped for current versions', async (t) => {
342+
const sources = {
343+
scripts: '0041..005A ; LATIN\n',
344+
properties: '0020 ; White_space\n',
345+
'derived-normalization-properties': '0340..0341 ; NFC_NO\n',
346+
'name-aliases': '01A2;LATIN CAPITAL LETTER GHA\n',
347+
database:
348+
'02AE;LATIN SMALL LETTER TURNED H WITH FISHHOOK ;Ll;0;L;;;;;N;;;;;\n',
349+
'line-break':
350+
'# - Assigned characters that are not listed explicitly are given the value\n# "AL".\n3400;ID;<CJK Ideograph Extension A, First>\n4DB5;ID;<CJK Ideograph Extension A, Last>\n',
351+
};
352+
t.mock.method(utils, 'readDataFile', (version, type) => sources[type]);
353+
// Each workaround applies to the historical release that needs it…
354+
t.assert.ok((await parseScripts('3.1.0')).Latin.contains(0x41));
355+
t.assert.ok((await parseProperties('3.1.0')).White_Space.contains(0x20));
356+
t.assert.equal(
357+
(await parseDerivedNormalizationProperties('3.1.0')).NFC_NO,
358+
undefined,
359+
);
360+
t.assert.ok(
361+
(await parseNameAliases('5.0.0')).correction[0x1a2].includes(
362+
'LATIN CAPITAL LETTER GHA',
363+
),
364+
);
365+
t.assert.ok(
366+
(await parseNames('4.0.0'))[
367+
'LATIN SMALL LETTER TURNED H WITH FISHHOOK'
368+
].contains(0x2ae),
369+
);
370+
t.assert.ok((await parseLineBreak('3.0.0')).Ideographic.contains(0x3401));
371+
// …and current releases parse their data as written.
372+
t.assert.ok((await parseScripts('18.0.0')).LATIN.contains(0x41));
373+
t.assert.ok((await parseProperties('18.0.0')).White_space.contains(0x20));
374+
t.assert.ok(
375+
(await parseDerivedNormalizationProperties('18.0.0')).NFC_NO.contains(
376+
0x340,
377+
),
378+
);
379+
t.assert.equal((await parseNameAliases('18.0.0')).correction, undefined);
380+
t.assert.ok(
381+
(await parseNames('18.0.0'))[
382+
'LATIN SMALL LETTER TURNED H WITH FISHHOOK '
383+
].contains(0x2ae),
384+
);
385+
const lineBreak = await parseLineBreak('18.0.0');
386+
t.assert.ok(!lineBreak.Ideographic.contains(0x3401));
387+
t.assert.ok(lineBreak.Unknown.contains(0x3401));
388+
});

0 commit comments

Comments
 (0)