Skip to content

Commit c3d57d5

Browse files
committed
Preserve noncharacter records in legacy Unicode properties
Assisted-By: GPT-6
1 parent df8b6c0 commit c3d57d5

2 files changed

Lines changed: 18 additions & 2 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,15 +333,17 @@ const parseProperties = async (version) => {
333333
if (
334334
// Ignore Bidi_Class as they have been generated from UnicodeData.
335335
currentProperty.startsWith('Bidi:') ||
336-
// Ignore Not a Character and Unassigned Code Value as they have been generated from UnicodeData.
337-
currentProperty === 'Not a Character' ||
336+
// Unassigned code points are already available as General_Category=Cn.
338337
currentProperty === 'Unassigned Code Value'
339338
) {
340339
// Skip contents.
341340
i++;
342341
while (++i < lines.length && lines[i] !== '');
343342
continue;
344343
}
344+
if (currentProperty === 'Not a Character') {
345+
currentProperty = 'Noncharacter_Code_Point';
346+
}
345347
currentProperty = currentProperty.replaceAll(' ', '_');
346348
currentProperty = findCanonicalName(currentProperty) ?? currentProperty;
347349
map[currentProperty] = regenerate();

‎tests/parsers.mjs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,20 @@ test('Legacy property ranges preserve six-digit code points', async (t) => {
231231
t.assert.ok(!properties.Private_Use.contains(0xffff));
232232
});
233233

234+
test('Legacy noncharacter records are preserved as a binary property', async (t) => {
235+
const properties = await parseProperties('3.0.1');
236+
const noncharacters = properties.Noncharacter_Code_Point;
237+
t.assert.equal(noncharacters.toArray().length, 34);
238+
for (let plane = 0; plane <= 16; plane++) {
239+
t.assert.ok(noncharacters.contains((plane << 16) + 0xfffe));
240+
t.assert.ok(noncharacters.contains((plane << 16) + 0xffff));
241+
}
242+
t.assert.ok(!noncharacters.contains(0x378));
243+
// FDD0..FDEF became noncharacters in Unicode 3.1.
244+
t.assert.ok(!noncharacters.contains(0xfdd0));
245+
t.assert.equal(properties.Unassigned_Code_Value, undefined);
246+
});
247+
234248
test('Property records do not require spaces around semicolons', async (t) => {
235249
for (const version of ['4.0.1', '4.1.0']) {
236250
const scripts = await parseScripts(version);

0 commit comments

Comments
 (0)