Skip to content

Commit 2e0a911

Browse files
JLHwungmathiasbynens
authored andcommitted
Let the narrower block entry win in legacy block data
The Unicode 2.0 blocks file lists FE70..FEFF as Arabic Presentation Forms-B and FEFF..FEFF as Specials, so U+FEFF belonged to both blocks in Unicode 2.0.14 through 2.1.8. Apply narrower ranges last so the more specific Specials entry wins, matching the corrected Unicode 2.1.9 file. Assisted-By: Claude Opus 5.5 Issue: #110
1 parent 666e247 commit 2e0a911

2 files changed

Lines changed: 21 additions & 9 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ const parseBlocks = async (version) => {
306306
return;
307307
}
308308
map = {};
309+
const records = [];
309310
for (const line of source.trimEnd().split('\n')) {
310311
const record = line.split('#')[0].trim();
311312
if (!record) {
@@ -316,7 +317,21 @@ const parseBlocks = async (version) => {
316317
.map((field) => field.trim());
317318
const canonicalBlockName = looseMatch('Block', blockName).value;
318319
map[canonicalBlockName] ??= regenerate();
319-
map[canonicalBlockName].addRange(parseInt(start, 16), parseInt(end, 16));
320+
records.push({
321+
block: canonicalBlockName,
322+
start: parseInt(start, 16),
323+
end: parseInt(end, 16),
324+
});
325+
}
326+
// The Unicode 2.0 blocks file lists FE70..FEFF as Arabic Presentation
327+
// Forms-B and FEFF..FEFF as Specials. Apply narrower ranges last so the
328+
// more specific entry wins, as the corrected Unicode 2.1.9 file does.
329+
records.sort((a, b) => b.end - b.start - (a.end - a.start));
330+
for (const { block, start, end } of records) {
331+
for (const codePoints of Object.values(map)) {
332+
codePoints.removeRange(start, end);
333+
}
334+
map[block].addRange(start, end);
320335
}
321336
} else {
322337
map = await parseBlocksScriptsProperties('blocks', version);

‎tests/parsers.mjs‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ test('Historical titlecase digraphs retain their identity mapping', async (t) =>
5959
});
6060

6161
test('Block values cover Unicode, including No_Block gaps', async (t) => {
62-
// Earlier files overlap Arabic Presentation Forms-B and Specials at U+FEFF.
63-
for (const version of ['2.1.9', '3.1.0', '18.0.0']) {
62+
for (const version of ['2.0.14', '2.1.8', '2.1.9', '3.1.0', '18.0.0']) {
6463
const blocks = await parseBlocks(version);
6564
t.assert.ok(blocks.No_Block.contains(0x40000), version);
6665
t.assert.ok(!blocks.No_Block.contains(0x41), version);
@@ -99,12 +98,10 @@ test('Legacy blocks retain every discontiguous Specials range', async (t) => {
9998
t.assert.ok(blocks.Specials.contains(0xfffd), version);
10099
t.assert.ok(!blocks.Specials.contains(0xff00), version);
101100
t.assert.ok(!blocks.No_Block.contains(0xfeff), version);
102-
// Preserve the overlap explicitly recorded by the earliest source files.
103-
t.assert.equal(
104-
blocks.Arabic_Presentation_Forms_B.contains(0xfeff),
105-
['2.0.14', '2.1.2', '2.1.5', '2.1.8'].includes(version),
106-
version,
107-
);
101+
// The earliest source files also list U+FEFF in Arabic Presentation
102+
// Forms-B. The more specific Specials entry wins.
103+
t.assert.ok(!blocks.Arabic_Presentation_Forms_B.contains(0xfeff), version);
104+
t.assert.ok(blocks.Arabic_Presentation_Forms_B.contains(0xfefe), version);
108105
}
109106
});
110107

0 commit comments

Comments
 (0)