Skip to content

Commit 666e247

Browse files
JLHwungmathiasbynens
authored andcommitted
Assign the Unicode 1.1.5 Unified Han range
UnicodeData-1.1.5 lists only U+4E00 as `<CJK IDEOGRAPH REPRESENTATIVE>`, so the gap logic reported 4E01..9FA5 as unassigned. The Unicode 1.1 Han cross-reference table (CJKXREF.TXT) assigns the whole 4E00..9FA5 range, so give the representative record that range. Assisted-By: Claude Opus 5.5 Issue: #110
1 parent 5457e06 commit 666e247

2 files changed

Lines changed: 22 additions & 2 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,13 @@ const parseDerivedGeneralCategory = async (version) => {
182182
let flag = false;
183183
let first = 0;
184184
let lastCodePoint = -1;
185+
// Unicode 1.1.5 lists only U+4E00 as a representative of the Unified
186+
// Han ideographs. The Unicode 1.1 Han cross-reference table assigns the
187+
// whole 4E00..9FA5 range:
188+
// https://www.unicode.org/Public/1.1-Update/CJKXREF.TXT
189+
const representativeRangeEnds = new Map([
190+
['<CJK IDEOGRAPH REPRESENTATIVE>', 0x9fa5],
191+
]);
185192
// http://unicode.org/reports/tr44/#GC_Values_Table
186193
// http://unicode.org/reports/tr18/#Categories
187194
for (const line of lines) {
@@ -192,6 +199,7 @@ const parseDerivedGeneralCategory = async (version) => {
192199
const data = trimmed.split(';');
193200
const codePoint = parseInt(data[0], 16);
194201
const name = data[1];
202+
const recordEnd = representativeRangeEnds.get(name) ?? codePoint;
195203
const generalCategory = data[2];
196204
const categories = [
197205
categoryAliases.get(generalCategory),
@@ -230,11 +238,11 @@ const parseDerivedGeneralCategory = async (version) => {
230238
first = codePoint;
231239
} else {
232240
for (const category of categories) {
233-
categoryMap[category].add(codePoint);
241+
categoryMap[category].addRange(codePoint, recordEnd);
234242
}
235243
}
236244
}
237-
lastCodePoint = codePoint;
245+
lastCodePoint = recordEnd;
238246
}
239247

240248
if (lastCodePoint < CODEPOINT_MAX) {

‎tests/parsers.mjs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import parseNameAliases from '../scripts/parse-name-aliases.mjs';
1212
import {
1313
parseBlocks,
1414
parseDerivedCoreProperties,
15+
parseDerivedGeneralCategory,
1516
parseDerivedNormalizationProperties,
1617
parseProperties,
1718
parseScripts,
@@ -151,6 +152,17 @@ test('Algorithmic ranges have individual formal names', async (t) => {
151152
}
152153
});
153154

155+
test('Unicode 1.1.5 assigns the Unified Han range represented by U+4E00', async (t) => {
156+
const categories = await parseDerivedGeneralCategory('1.1.5');
157+
for (const cp of [0x4e00, 0x4e01, 0x9fa5]) {
158+
t.assert.ok(categories.Other_Letter.contains(cp), cp.toString(16));
159+
t.assert.ok(categories.Assigned.contains(cp), cp.toString(16));
160+
t.assert.ok(!categories.Unassigned.contains(cp), cp.toString(16));
161+
}
162+
t.assert.ok(categories.Unassigned.contains(0x9fa6));
163+
t.assert.ok(!categories.Assigned.contains(0x9fa6));
164+
});
165+
154166
test('Bidi class applies historical and current unassigned defaults', async (t) => {
155167
for (const version of [
156168
'3.0.0',

0 commit comments

Comments
 (0)