Skip to content

Commit 259a466

Browse files
committed
Parse Unicode property records independently of separator spacing
Assisted-By: GPT-6
1 parent b4a470a commit 259a466

2 files changed

Lines changed: 22 additions & 9 deletions

File tree

‎scripts/parse-blocks-scripts-properties.mjs‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,11 @@ const parseBlocksScriptsProperties = async (type, version) => {
3939
}
4040
const lines = source.split('\n');
4141
for (const line of lines) {
42-
if (
43-
/^#/.test(line) ||
44-
!(/^(?:blocks|bidi-mirroring|derived-general-category|properties)$/.test(
45-
type,
46-
)
47-
? /;\x20/.test(line)
48-
: /\x20;\x20/.test(line))
49-
) {
42+
const trimmed = line.trim();
43+
if (!trimmed || trimmed.startsWith('#')) {
5044
continue;
5145
}
52-
const data = line.trim().split(';');
46+
const data = trimmed.split(';');
5347
const charRange = data[0].replace('..', '-').trim();
5448
let item = data[1]
5549
.split(type === 'blocks' ? ';' : '#')[0]

‎tests/parsers.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { test } from 'node:test';
22
import parseDecompositionMapping from '../scripts/parse-decomposition-mapping.mjs';
3+
import {
4+
parseDerivedCoreProperties,
5+
parseProperties,
6+
parseScripts,
7+
} from '../scripts/parse-blocks-scripts-properties.mjs';
38

49
test('Hangul has pairwise canonical decomposition mappings', async (t) => {
510
for (const version of ['2.0.14', '3.2.0', '18.0.0']) {
@@ -13,3 +18,17 @@ test('Hangul has pairwise canonical decomposition mappings', async (t) => {
1318
}
1419
t.assert.equal(await parseDecompositionMapping('1.1.5'), undefined);
1520
});
21+
22+
test('Property records do not require spaces around semicolons', async (t) => {
23+
for (const version of ['4.0.1', '4.1.0']) {
24+
const scripts = await parseScripts(version);
25+
t.assert.ok(scripts.Common.contains(0x100000));
26+
t.assert.ok(scripts.Common.contains(0x10fffd));
27+
t.assert.ok(!scripts.Unknown.contains(0x100000));
28+
}
29+
for (const version of ['4.0.1', '4.1.0', '5.0.0']) {
30+
const core = await parseDerivedCoreProperties(version);
31+
t.assert.ok(core.Default_Ignorable_Code_Point.contains(0x10fffe));
32+
t.assert.ok(core.Default_Ignorable_Code_Point.contains(0x10ffff));
33+
}
34+
});

0 commit comments

Comments
 (0)