Skip to content

Commit 7dbd8d9

Browse files
committed
Accept flexible separators in emoji and bidi bracket data
Assisted-By: GPT-6
1 parent a05644a commit 7dbd8d9

4 files changed

Lines changed: 62 additions & 13 deletions

File tree

‎scripts/parse-bidi-brackets.mjs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@ const parseBidiBrackets = async (version) => {
1919
}
2020
const lines = source.split('\n');
2121
for (const line of lines) {
22-
if (/^#/.test(line) || !/;\x20/.test(line)) {
22+
const record = line.split('#')[0].trim();
23+
if (!record) {
2324
continue;
2425
}
25-
const data = line.trim().split(';');
26+
const data = record.split(';').map((field) => field.trim());
2627
const charRange = data[0].replace('..', '-').trim();
27-
let item = data[2].split('#')[0].trim();
28+
let item = data[2];
2829
item = bidiBracketMap.get(item);
2930
const rangeParts = charRange.split('-');
3031
if (rangeParts.length === 2) {

‎scripts/parse-emoji-sequences.mjs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,14 @@ const parseEmojiSequencesWithId = async ({ version, id }) => {
88
const propertyMap = new Map();
99
const lines = source.split('\n');
1010
for (const line of lines) {
11-
if (!line || /^#/.test(line)) {
11+
const record = line.split('#')[0].trim();
12+
if (!record) {
1213
continue;
1314
}
14-
const data = line.trim().split('; ');
15+
const data = record.split(';').map((field) => field.trim());
1516
const charRange = data[0].replace('..', '-').trim();
1617
const rangeParts = charRange.split('-');
17-
const property = data[1].split('#')[0].trim();
18+
const property = data[1];
1819
if (rangeParts.length === 2) {
1920
utils
2021
.range(parseInt(rangeParts[0], 16), parseInt(rangeParts[1], 16))
@@ -29,7 +30,7 @@ const parseEmojiSequencesWithId = async ({ version, id }) => {
2930
} else {
3031
const codePoints = data[0]
3132
.trim()
32-
.split(' ')
33+
.split(/\s+/)
3334
.map((hex) => {
3435
return parseInt(hex, 16);
3536
});
@@ -56,13 +57,14 @@ const parseEmojiTestData = async ({ version }) => {
5657
const sequences = [];
5758
const lines = source.split('\n');
5859
for (const line of lines) {
59-
if (!line || /^#/.test(line)) {
60+
const record = line.split('#')[0].trim();
61+
if (!record) {
6062
continue;
6163
}
62-
const data = line.trim().split('; ');
64+
const data = record.split(';').map((field) => field.trim());
6365
const codePoints = data[0]
6466
.trim()
65-
.split(' ')
67+
.split(/\s+/)
6668
.map((hex) => {
6769
return parseInt(hex, 16);
6870
});

‎scripts/parse-emoji.mjs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,14 @@ const parseEmoji = async (version) => {
88
const propertyMap = new Map();
99
const lines = source.split('\n');
1010
for (const line of lines) {
11-
if (!line || /^#/.test(line)) {
11+
const record = line.split('#')[0].trim();
12+
if (!record) {
1213
continue;
1314
}
14-
const data = line.trim().split(' ; ');
15+
const data = record.split(';').map((field) => field.trim());
1516
const charRange = data[0].replace('..', '-').trim();
1617
const rangeParts = charRange.split('-');
17-
const property = data[1].split('#')[0].trim();
18+
const property = data[1];
1819
if (rangeParts.length === 2) {
1920
utils
2021
.range(parseInt(rangeParts[0], 16), parseInt(rangeParts[1], 16))

‎tests/parser-formats.mjs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { test } from 'node:test';
22
import parseSpecialCasing from '../scripts/parse-special-casing.mjs';
3+
import parseEmoji from '../scripts/parse-emoji.mjs';
4+
import parseEmojiSequences from '../scripts/parse-emoji-sequences.mjs';
5+
import parseBidiBrackets from '../scripts/parse-bidi-brackets.mjs';
36
import utils from '../scripts/utils.mjs';
47

58
test('SpecialCasing accepts optional comments and flexible whitespace', async (t) => {
@@ -25,6 +28,48 @@ FB00 ; FB00 ; 0046 0066 ; 0046 0046 ; # comment with # and ; characters
2528
]);
2629
});
2730

31+
test('Emoji properties accept compact and padded field separators', async (t) => {
32+
let source;
33+
t.mock.method(utils, 'readDataFile', () => source);
34+
for (const separator of [';', ' ; ', '\t;\t', ' ;\t']) {
35+
source = ` # comment\n\t\n231A..231B${separator}Emoji_Presentation # watches\n1F600${separator}Emoji\n`;
36+
const map = await parseEmoji('18.0.0');
37+
t.assert.deepEqual(map.Emoji_Presentation, [0x231a, 0x231b]);
38+
t.assert.deepEqual(map.Emoji, [0x1f600]);
39+
}
40+
});
41+
42+
test('Emoji sequences preserve property names with flexible field spacing', async (t) => {
43+
const sources = {
44+
'emoji-sequences':
45+
'0023 FE0F\t20E3;Emoji_Keycap_Sequence; keycap\n231A..231B;Basic_Emoji; watches\n',
46+
'emoji-zwj-sequences':
47+
'1F469\t200D 1F4BB\t;\tEmoji_ZWJ_Sequence ; technologist\n',
48+
'emoji-test': '0023\tFE0F 20E3;fully-qualified # keycap\n',
49+
};
50+
t.mock.method(utils, 'readDataFile', (version, type) => sources[type]);
51+
const map = await parseEmojiSequences('18.0.0');
52+
t.assert.deepEqual(map.Emoji_Keycap_Sequence, ['#\ufe0f\u20e3']);
53+
t.assert.deepEqual(map.Basic_Emoji, ['\u231a', '\u231b']);
54+
t.assert.deepEqual(map.Emoji_ZWJ_Sequence, ['\u{1f469}\u200d\u{1f4bb}']);
55+
t.assert.deepEqual(map.Emoji_Test, ['#\ufe0f\u20e3']);
56+
t.assert.equal(map.keycap, undefined);
57+
});
58+
59+
test('Bidi brackets do not depend on separator spacing', async (t) => {
60+
let source;
61+
t.mock.method(utils, 'readDataFile', () => source);
62+
for (const separator of [';', ' ; ', '\t;\t']) {
63+
source = ` # comment\n\t\n0028${separator}0029${separator}o\n0029${separator}0028${separator}c\n`;
64+
const map = await parseBidiBrackets('18.0.0');
65+
t.assert.deepEqual(map.Open.toArray(), [0x28]);
66+
t.assert.deepEqual(map.Close.toArray(), [0x29]);
67+
t.assert.ok(!map.None.contains(0x28));
68+
t.assert.ok(!map.None.contains(0x29));
69+
t.assert.ok(map.None.contains(0x41));
70+
}
71+
});
72+
2873
test('SpecialCasing preserves empty mappings and normalizes condition separators', async (t) => {
2974
t.mock.method(
3075
utils,

0 commit comments

Comments
 (0)