Skip to content

Commit 516c94e

Browse files
committed
Derive implicit joining types for historical ArabicShaping data
Assisted-By: GPT-6
1 parent f4afd80 commit 516c94e

2 files changed

Lines changed: 45 additions & 3 deletions

File tree

‎scripts/parse-joining-type.mjs‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import utils from './utils.mjs';
22
import valueAliases from 'unicode-property-value-aliases';
33
import regenerate from 'regenerate';
4+
import { parseDerivedGeneralCategory } from './parse-blocks-scripts-properties.mjs';
45

56
const aliases = valueAliases.get('Joining_Type');
67

@@ -41,8 +42,18 @@ const parseDerivedJoiningType = (source) => {
4142
return map;
4243
};
4344

44-
const parseArabicShaping = (source) => {
45-
const map = {};
45+
const parseArabicShaping = (source, categories) => {
46+
// The older releases that need this fallback define implicit T as Mn or Cf,
47+
// excluding joining controls. Explicit ArabicShaping records override defaults.
48+
const transparent = regenerate()
49+
.add(categories.Nonspacing_Mark)
50+
.add(categories.Format)
51+
.remove(0x200c)
52+
.remove(0x200d);
53+
const map = {
54+
Transparent: transparent,
55+
Non_Joining: regenerate().addRange(0, 0x10ffff).remove(transparent),
56+
};
4657
const lines = source.split('\n');
4758
for (const line of lines) {
4859
const trimmed = line.trim();
@@ -55,6 +66,8 @@ const parseArabicShaping = (source) => {
5566
const data = trimmed.split(';');
5667
const codePoint = parseInt(data[0].trim(), 16);
5768
const joiningType = findCanonicalName(data[2].trim());
69+
map.Transparent.remove(codePoint);
70+
map.Non_Joining.remove(codePoint);
5871
map[joiningType] ??= regenerate();
5972
map[joiningType].add(codePoint);
6073
}
@@ -72,7 +85,7 @@ const parseJoiningType = async (version) => {
7285
if (!source) {
7386
return;
7487
}
75-
return parseArabicShaping(source);
88+
return parseArabicShaping(source, await parseDerivedGeneralCategory(version));
7689
};
7790

7891
export default parseJoiningType;

‎tests/parsers.mjs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { test } from 'node:test';
22
import parseDecompositionMapping from '../scripts/parse-decomposition-mapping.mjs';
33
import parseIndicConjunctBreak from '../scripts/parse-indic-conjunct-break.mjs';
4+
import parseJoiningType from '../scripts/parse-joining-type.mjs';
45
import {
56
parseDerivedCoreProperties,
67
parseDerivedNormalizationProperties,
@@ -21,6 +22,34 @@ test('Hangul has pairwise canonical decomposition mappings', async (t) => {
2122
t.assert.equal(await parseDecompositionMapping('1.1.5'), undefined);
2223
});
2324

25+
test('ArabicShaping fallback derives implicit joining types', async (t) => {
26+
for (const version of ['3.2.0', '4.0.0', '4.0.1']) {
27+
const map = await parseJoiningType(version);
28+
for (const [cp, type] of [
29+
[0x41, 'Non_Joining'],
30+
[0x378, 'Non_Joining'],
31+
[0x300, 'Transparent'],
32+
[0x200e, 'Transparent'],
33+
[0x200c, 'Non_Joining'],
34+
[0x200d, 'Join_Causing'],
35+
[0x628, 'Dual_Joining'],
36+
]) {
37+
t.assert.ok(
38+
map[type].contains(cp),
39+
`${version}: ${cp.toString(16)}=${type}`,
40+
);
41+
t.assert.equal(
42+
Object.values(map).filter((set) => set.contains(cp)).length,
43+
1,
44+
);
45+
}
46+
if (version !== '3.2.0') {
47+
t.assert.ok(map.Non_Joining.contains(0x600));
48+
t.assert.ok(!map.Transparent.contains(0x600));
49+
}
50+
}
51+
});
52+
2453
test('Normalization mappings are not emitted as binary properties', async (t) => {
2554
for (const version of ['15.1.0', '16.0.0', '17.0.0', '18.0.0']) {
2655
const binary = await parseDerivedNormalizationProperties(version);

0 commit comments

Comments
 (0)