Skip to content

Commit e9c1785

Browse files
committed
Preserve Indic_Conjunct_Break as an enumerated property
Assisted-By: GPT-6
1 parent 96ae3f2 commit e9c1785

4 files changed

Lines changed: 83 additions & 2 deletions

File tree

‎index.mjs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import parseBidiClass from './scripts/parse-bidi-class.mjs';
1111
import parseCompositionExclusions from './scripts/parse-composition-exclusions.mjs';
1212
import parseIndicPositionalCategory from './scripts/parse-indic-positional-category.mjs';
1313
import parseIndicSyllabicCategory from './scripts/parse-indic-syllabic-category.mjs';
14+
import parseIndicConjunctBreak from './scripts/parse-indic-conjunct-break.mjs';
1415
import parseLineBreak from './scripts/parse-line-break.mjs';
1516
import parseScriptExtensions from './scripts/parse-script-extensions.mjs';
1617
import parseSpecialCasing from './scripts/parse-special-casing.mjs';
@@ -208,6 +209,15 @@ const generateData = async (version) => {
208209
type: 'Grapheme_Cluster_Break',
209210
}),
210211
);
212+
console.log('Parsing Unicode v%s `Indic_Conjunct_Break`…', version);
213+
utils.extend(
214+
dirMap,
215+
utils.writeFiles({
216+
version: version,
217+
map: await parseIndicConjunctBreak(version),
218+
type: 'Indic_Conjunct_Break',
219+
}),
220+
);
211221
console.log('Parsing Unicode v%s `Word_Break`…', version);
212222
utils.extend(
213223
dirMap,

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ const parseBlocksScriptsProperties = async (type, version) => {
4646
continue;
4747
}
4848
const data = trimmed.split(';');
49+
if (type === 'derived-core-properties' && data.length > 2) {
50+
// Enumerated properties such as InCB have an explicit third field.
51+
continue;
52+
}
4953
const charRange = data[0].replace('..', '-').trim();
5054
let item = data[1]
5155
.split(type === 'blocks' ? ';' : '#')[0]
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import regenerate from 'regenerate';
2+
import utils from './utils.mjs';
3+
4+
const parseIndicConjunctBreak = async (version) => {
5+
const source = utils.readDataFile(version, 'derived-core-properties');
6+
if (!source) {
7+
return;
8+
}
9+
const map = { None: regenerate().addRange(0, 0x10ffff) };
10+
let found = false;
11+
for (const line of source.split('\n')) {
12+
const fields = line
13+
.split('#')[0]
14+
.trim()
15+
.split(';')
16+
.map((s) => s.trim());
17+
if (fields[1] !== 'InCB') {
18+
continue;
19+
}
20+
found = true;
21+
const [start, end = start] = fields[0]
22+
.split('..')
23+
.map((s) => parseInt(s, 16));
24+
const value = fields[2];
25+
map[value] ??= regenerate();
26+
map.None.removeRange(start, end);
27+
map[value].addRange(start, end);
28+
}
29+
return found ? map : undefined;
30+
};
31+
32+
export default parseIndicConjunctBreak;

‎tests/parsers.mjs‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { test } from 'node:test';
22
import parseDecompositionMapping from '../scripts/parse-decomposition-mapping.mjs';
3+
import parseIndicConjunctBreak from '../scripts/parse-indic-conjunct-break.mjs';
34
import {
45
parseDerivedCoreProperties,
56
parseProperties,
@@ -19,8 +20,35 @@ test('Hangul has pairwise canonical decomposition mappings', async (t) => {
1920
t.assert.equal(await parseDecompositionMapping('1.1.5'), undefined);
2021
});
2122

23+
test('Indic conjunct break retains enumerated values', async (t) => {
24+
for (const version of ['15.1.0', '16.0.0', '17.0.0', '18.0.0']) {
25+
const map = await parseIndicConjunctBreak(version);
26+
t.assert.ok(map.Linker.contains(0x94d));
27+
t.assert.ok(map.Consonant.contains(0x915));
28+
t.assert.ok(map.Extend.contains(0x300));
29+
t.assert.ok(map.None.contains(0x41));
30+
for (const cp of [0x94d, 0x915, 0x300, 0x41]) {
31+
t.assert.equal(
32+
Object.values(map).filter((set) => set.contains(cp)).length,
33+
1,
34+
);
35+
}
36+
const binary = await parseDerivedCoreProperties(version);
37+
t.assert.equal(binary.InCB, undefined);
38+
t.assert.equal(binary.Indic_Conjunct_Break, undefined);
39+
}
40+
t.assert.equal(await parseIndicConjunctBreak('15.0.0'), undefined);
41+
});
42+
2243
test('Binary property aliases use canonical names', async (t) => {
23-
for (const version of ['4.0.1', '4.1.0', '5.0.0', '6.3.0', '8.0.0', '18.0.0']) {
44+
for (const version of [
45+
'4.0.1',
46+
'4.1.0',
47+
'5.0.0',
48+
'6.3.0',
49+
'8.0.0',
50+
'18.0.0',
51+
]) {
2452
const properties = await parseProperties(version);
2553
t.assert.ok(properties.Sentence_Terminal.contains(0x21));
2654
t.assert.equal(properties.STerm, undefined);
@@ -63,7 +91,14 @@ test('Property records do not require spaces around semicolons', async (t) => {
6391
});
6492

6593
test('Script defaults follow the Unicode version', async (t) => {
66-
for (const version of ['3.1.0', '3.1.1', '3.2.0', '4.0.0', '4.0.1', '4.1.0']) {
94+
for (const version of [
95+
'3.1.0',
96+
'3.1.1',
97+
'3.2.0',
98+
'4.0.0',
99+
'4.0.1',
100+
'4.1.0',
101+
]) {
67102
const scripts = await parseScripts(version);
68103
t.assert.ok(scripts.Common.contains(0x20));
69104
t.assert.ok(scripts.Common.contains(0x378));

0 commit comments

Comments
 (0)