Skip to content

Commit dc18217

Browse files
JLHwungmathiasbynens
authored andcommitted
Strip inline comments before parsing simple case mappings
Keep comments out of the final UnicodeData field so empty titlecase mappings retain their defaults. Check comment invariance across every parser and Unicode version. Assisted-By: GPT-6 Issue: #110
1 parent b570656 commit dc18217

3 files changed

Lines changed: 37 additions & 1 deletion

File tree

‎scripts/parse-simple-case-mapping.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ const parseSimpleCaseMapping = async (version) => {
2323
}
2424
const lines = source.split('\n');
2525
for (const line of lines) {
26-
const lineTrimmed = line.trim();
26+
const lineTrimmed = line.split('#')[0].trim();
2727
if (lineTrimmed === '' || lineTrimmed.startsWith('#')) {
2828
continue;
2929
}

‎tests/line-endings.mjs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,20 @@ for (const file of fs.readdirSync(new URL('../scripts/', import.meta.url))) {
1717

1818
const formats = [
1919
['CRLF', (source) => source.replace(/\r?\n/g, '\r\n')],
20+
[
21+
'inline comments',
22+
(source) =>
23+
source
24+
.split('\n')
25+
.map((line) => {
26+
if (line.trim().startsWith('#') || !line.includes(';')) return line;
27+
return (
28+
line.split('#')[0].trimEnd() +
29+
'# explanatory comment; with field delimiters'
30+
);
31+
})
32+
.join('\n'),
33+
],
2034
[
2135
'whitespace-only lines and indented comments',
2236
(source) =>

‎tests/parser-formats.mjs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,28 @@ import {
1515
} from '../scripts/parse-blocks-scripts-properties.mjs';
1616
import utils from '../scripts/utils.mjs';
1717

18+
test('Simple case mappings ignore comments after empty and explicit titlecase fields', async (t) => {
19+
t.mock.method(utils, 'readDataFile', () =>
20+
[
21+
' # A comment; with field delimiters',
22+
'0000;<control>;Cc;0;BN;;;;;N;;;;;# no case mappings; still a comment',
23+
'0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;# lowercase only',
24+
'0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;# null titlecase defaults to uppercase',
25+
'01C6;LATIN SMALL LETTER DZ WITH CARON;Ll;0;L;<compat> 0064 017E;;;;N;;;01C4;;01C5# explicit titlecase; # extra comment marker',
26+
].join('\n'),
27+
);
28+
for (const version of ['4.0.0', '18.0.0']) {
29+
const map = await parseSimpleCaseMapping(version);
30+
t.assert.equal(map.Uppercase.get(0x61), 0x41);
31+
t.assert.equal(map.Lowercase.get(0x41), 0x61);
32+
t.assert.equal(map.Titlecase.get(0x61), 0x41);
33+
t.assert.equal(map.Titlecase.get(0x1c6), 0x1c5);
34+
for (const mappings of Object.values(map)) {
35+
t.assert.ok(!mappings.has(0));
36+
}
37+
}
38+
});
39+
1840
test('Script_Extensions accepts whitespace runs for singletons and ranges', async (t) => {
1941
const read = utils.readDataFile;
2042
let source;

0 commit comments

Comments
 (0)