Skip to content

Commit 417700a

Browse files
committed
Normalize Unicode data line endings before parsing
Assisted-By: GPT-6
1 parent 250d435 commit 417700a

3 files changed

Lines changed: 54 additions & 1 deletion

File tree

‎scripts/utils.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ const readDataFile = (version, type) => {
325325
);
326326
try {
327327
const source = fs.readFileSync(sourceFile, 'utf-8');
328-
return source;
328+
return source.replace(/\r\n?/g, '\n');
329329
} catch {
330330
return;
331331
}

‎tests/line-endings.mjs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import fs from 'node:fs';
2+
import { test } from 'node:test';
3+
import resources from '../data/resources.mjs';
4+
import * as properties from '../scripts/parse-blocks-scripts-properties.mjs';
5+
6+
const parsers = Object.entries(properties);
7+
for (const file of fs.readdirSync(new URL('../scripts/', import.meta.url))) {
8+
if (
9+
!/^parse-.*\.mjs$/.test(file) ||
10+
file === 'parse-blocks-scripts-properties.mjs'
11+
) {
12+
continue;
13+
}
14+
const { default: parse } = await import(`../scripts/${file}`);
15+
parsers.push([file, parse]);
16+
}
17+
18+
test('Every parser preserves results with CRLF input', async (t) => {
19+
const read = fs.readFileSync;
20+
let useCRLF = false;
21+
t.mock.method(fs, 'readFileSync', (...args) => {
22+
const result = read(...args);
23+
return useCRLF && typeof result === 'string'
24+
? result.replace(/\r?\n/g, '\r\n')
25+
: result;
26+
});
27+
for (const { version } of resources) {
28+
for (const [name, parse] of parsers) {
29+
const kinds =
30+
name === 'parse-grapheme-word-sentence-break.mjs'
31+
? ['grapheme-cluster-break', 'word-break', 'sentence-break']
32+
: [undefined];
33+
for (const kind of kinds) {
34+
const run = async () =>
35+
parse(
36+
version,
37+
name === 'parse-script-extensions.mjs'
38+
? await properties.parseScripts(version)
39+
: kind,
40+
);
41+
useCRLF = false;
42+
const expected = await run();
43+
useCRLF = true;
44+
t.assert.deepEqual(
45+
await run(),
46+
expected,
47+
`${version}: ${name} ${kind ?? ''}`,
48+
);
49+
}
50+
}
51+
}
52+
});

‎tests/tests.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { suite, test } from 'node:test';
22
import './parsers.mjs';
3+
import './line-endings.mjs';
34
import resources from '../data/resources.mjs';
45

56
const oldest = resources[0].version;

0 commit comments

Comments
 (0)