Skip to content

Commit 9948735

Browse files
committed
Merge branch 'main' into fix/unicode-parser-correctness
2 parents 4e2015d + 039f190 commit 9948735

19 files changed

Lines changed: 455 additions & 54 deletions

‎bootstrap.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ for dir in $(find unicode-* -maxdepth 0 -type d | sort -r); do
1111
cd "${dir}";
1212
echo "Taking care of ${dir}…";
1313
git add -A;
14-
git commit -m 'Release v2.0.8';
14+
git commit -m 'Release v2.0.9';
1515
# When tagging a release, update `templates/package.json` accordingly.
16-
git tag v2.0.8;
16+
git tag v2.0.9;
1717
git push;
1818
git push --tags;
1919
cd ..;

‎index.mjs‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -338,24 +338,34 @@ const generateData = async (version) => {
338338
);
339339
fs.writeFileSync(
340340
path.resolve(__dirname, `output/unicode-${version}/index.d.mts`),
341+
// `index.mjs` has a single default export: an object mapping each
342+
// property type to the list of its directories.
341343
[
342344
'declare const data: {',
343-
...Object.keys(dirMap).map((key) => `\t${key}: string[];`),
345+
...Object.keys(dirMap).map((key) => `\treadonly ${key}: string[];`),
344346
'};',
345347
'export default data;',
346348
'',
347349
].join('\n'),
348350
);
349351
fs.writeFileSync(
350352
path.resolve(__dirname, `output/unicode-${version}/package.json`),
351-
compilePackage({ version: version }),
353+
compilePackage({
354+
version: version,
355+
typesVersions: JSON.stringify(utils.typesVersions(dirMap), null, '\t')
356+
// Indent to match the surrounding JSON.
357+
.replace(/\n/g, '\n\t'),
358+
}),
352359
);
353360
fs.mkdirSync(
354361
path.resolve(__dirname, `output/unicode-${version}/.github/workflows`),
355362
{
356363
recursive: true,
357364
},
358365
);
366+
fs.mkdirSync(path.resolve(__dirname, `output/unicode-${version}/types`), {
367+
recursive: true,
368+
});
359369
const staticFiles = [
360370
'.github/workflows/publish-on-tag.yml',
361371
'.gitattributes',
@@ -365,6 +375,13 @@ const generateData = async (version) => {
365375
'decode-property-map.d.mts',
366376
'decode-ranges.mjs',
367377
'decode-ranges.d.mts',
378+
'mapping-to-symbols.mjs',
379+
'mapping-to-symbols.d.mts',
380+
'decode-names.mjs',
381+
'decode-names.d.mts',
382+
'decode-mapping.mjs',
383+
'decode-mapping.d.mts',
384+
...utils.SHARED_DECLARATIONS.map((file) => `types/${file}.d.mts`),
368385
];
369386
for (const file of staticFiles) {
370387
fs.copyFileSync(

‎scripts/encode-ranges.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,4 @@ const encodeRegenerate = (regenerateSet) => {
5757
return encodeDeltas(deltas);
5858
};
5959

60-
export { encodeRanges, encodeRegenerate };
60+
export { base64enc, encodeDeltas, encodeRanges, encodeRegenerate };

‎scripts/utils.mjs‎

Lines changed: 156 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,49 @@ import { fileURLToPath } from 'node:url';
44
import zlib from 'node:zlib';
55
import jsesc from 'jsesc';
66
import regenerate from 'regenerate';
7-
import { encodeRanges, encodeRegenerate } from './encode-ranges.mjs';
7+
import {
8+
base64enc,
9+
encodeDeltas,
10+
encodeRanges,
11+
encodeRegenerate,
12+
} from './encode-ranges.mjs';
813

914
const __dirname = path.dirname(fileURLToPath(import.meta.url));
1015

11-
const gzipInline = (data) => {
16+
// Brotli-compress `text` and return a JS expression that decompresses it into
17+
// a Buffer at runtime.
18+
const brotliInlineBuffer = (text) => {
19+
const brotliBuffer = zlib.brotliCompressSync(text, {
20+
params: {
21+
[zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY,
22+
[zlib.constants.BROTLI_PARAM_LGWIN]:
23+
zlib.constants.BROTLI_MAX_WINDOW_BITS,
24+
[zlib.constants.BROTLI_PARAM_SIZE_HINT]: Buffer.byteLength(text),
25+
},
26+
});
27+
const str = brotliBuffer.toString('base64');
28+
return `brotliDecompressSync(Buffer.from('${str}', 'base64'))`;
29+
};
30+
31+
const brotliInline = (data) => {
1232
if (data instanceof Map) {
13-
return `new Map(${gzipInline([...data])})`;
33+
return `new Map(${brotliInline([...data])})`;
1434
}
15-
const json = JSON.stringify(data);
16-
const gzipBuffer = zlib.gzipSync(json);
17-
const str = gzipBuffer.toString('base64');
18-
return `JSON.parse(gunzipSync(Buffer.from('${str}', 'base64')))`;
35+
return `JSON.parse(${brotliInlineBuffer(JSON.stringify(data))})`;
1936
};
2037

38+
// Modules whose declarations are shared via `types/*.d.mts` and
39+
// `typesVersions` instead of being repeated in every directory.
40+
const SHARED_DECLARATIONS = ['ranges', 'regex', 'code-points', 'symbols'];
41+
42+
// Directories under these types declare their own `code-points`/`symbols` types.
43+
const MAPPING_TYPES = [
44+
'Case_Folding',
45+
'Simple_Case_Mapping',
46+
'Special_Casing',
47+
'Decomposition_Mapping',
48+
];
49+
2150
const range = (start, stop) => {
2251
// Inclusive, e.g. `range(1, 3)` → `[1, 2, 3]`.
2352
const result = [];
@@ -76,6 +105,59 @@ const samePropertyRuns = (codePointProperties) => {
76105
return result;
77106
};
78107

108+
// Front-code the names in the flat runs produced by `samePropertyRuns`.
109+
// See static/decode-names.mjs for the format and the decoder.
110+
const encodeNames = (flatRuns) => {
111+
const runs = [];
112+
const lines = [];
113+
let previous = '';
114+
for (let i = 0; i < flatRuns.length; i += 3) {
115+
runs.push(flatRuns[i], flatRuns[i + 1]);
116+
const name = flatRuns[i + 2];
117+
let shared = 0;
118+
while (
119+
shared < 63 &&
120+
shared < name.length &&
121+
shared < previous.length &&
122+
name.charCodeAt(shared) === previous.charCodeAt(shared)
123+
) {
124+
shared++;
125+
}
126+
lines.push(base64enc[shared] + name.slice(shared));
127+
previous = name;
128+
}
129+
return JSON.stringify(runs) + '\n' + lines.join('\n');
130+
};
131+
132+
// Zigzag encoding maps signed integers to unsigned ones: 0 → 0, -1 → 1, 1 → 2,
133+
// -2 → 3, … so that small differences in either direction stay small.
134+
const zigzag = (value) => ((value << 1) ^ (value >> 31)) >>> 0;
135+
136+
// Delta-encode a mapping from code points to code point(s), e.g. a case
137+
// mapping or a decomposition mapping, preserving entry order and whether
138+
// each entry maps to a single code point or to an array.
139+
// See static/decode-mapping.mjs for the format and the decoder.
140+
const encodeMapping = (codePoints) => {
141+
const output = [];
142+
let previousFrom = 0;
143+
for (const [from, to] of codePoints) {
144+
output.push(zigzag(from - previousFrom));
145+
previousFrom = from;
146+
let previous = from;
147+
if (Array.isArray(to)) {
148+
console.assert(to.length > 0, `empty mapping for U+${from.toString(16)}`);
149+
output.push(to.length);
150+
for (const codePoint of to) {
151+
output.push(zigzag(codePoint - previous));
152+
previous = codePoint;
153+
}
154+
} else {
155+
output.push(0, zigzag(to - previous));
156+
}
157+
}
158+
return encodeDeltas(output);
159+
};
160+
79161
// Only write the file if its content has changed.
80162
// Reading files is cheaper than writing files, and most files don’t change between builds.
81163
const writeFileIfChanged = (filePath, content) => {
@@ -111,11 +193,7 @@ const writeFiles = (options) => {
111193
const codePoints = map[item];
112194
const type =
113195
typeof options.type === 'function' ? options.type(item) : options.type;
114-
const isCodePointMapping =
115-
type === 'Case_Folding' ||
116-
type === 'Simple_Case_Mapping' ||
117-
type === 'Special_Casing' ||
118-
type === 'Decomposition_Mapping';
196+
const isCodePointMapping = MAPPING_TYPES.includes(type);
119197
const isNamesCanon = type === 'Names' && !subType;
120198
const isNameAliases = type === 'Names' && subType === 'name-aliases';
121199
const subdir = isNameAliases
@@ -157,7 +235,7 @@ const writeFiles = (options) => {
157235
// Sequence properties are special.
158236
if (type === 'Sequence_Property' || isNameAliases) {
159237
const sequences = codePoints;
160-
const output = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(sequences)};\n`;
238+
const output = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(sequences)};\n`;
161239
writeFileIfChanged(path.resolve(dir, 'index.mjs'), output);
162240
writeFileIfChanged(
163241
path.resolve(dir, 'index.d.mts'),
@@ -182,44 +260,41 @@ const writeFiles = (options) => {
182260
path.resolve(dir, 'ranges.mjs'),
183261
`import decodeRanges from '../../decode-ranges.mjs';\n\nexport default decodeRanges('${encodedRanges}');\n`,
184262
);
185-
writeFileIfChanged(
186-
path.resolve(dir, 'ranges.d.mts'),
187-
'import type { UnicodeRange } from "../../decode-ranges.mjs";\n\ndeclare const ranges: UnicodeRange[];\nexport default ranges;\n',
188-
);
189263
writeFileIfChanged(
190264
path.resolve(dir, 'regex.mjs'),
191265
'export default /' + regenerate(codePoints).toString() + '/;\n',
192266
);
193-
writeFileIfChanged(
194-
path.resolve(dir, 'regex.d.mts'),
195-
'declare const regex: RegExp;\nexport default regex;\n',
196-
);
197267
if (codePointsSizeLt(codePoints, 10)) {
198268
const codePointsAsArray =
199269
codePoints instanceof regenerate ? codePoints.toArray() : codePoints;
200270
codePointsFileContent = `export default ${jsesc(codePointsAsArray)};\n`;
201271
symbolsFileContent = `export default ${jsesc(codePointsAsArray.map((cp) => String.fromCodePoint(cp)))};\n`;
202272
}
203273
} else {
204-
const symbols = new Map();
205274
const valueTypes = new Set();
206-
for (let [from, to] of codePoints) {
207-
from = String.fromCodePoint(from);
208-
if (Array.isArray(to)) {
209-
valueTypes.add('number[]');
210-
to = String.fromCodePoint.apply(null, to);
211-
} else {
212-
valueTypes.add('number');
213-
to = String.fromCodePoint(to);
214-
}
215-
symbols.set(from, to);
275+
for (const to of codePoints.values()) {
276+
valueTypes.add(Array.isArray(to) ? 'number[]' : 'number');
216277
}
217278
if (codePoints.size < 10) {
279+
const symbols = new Map();
280+
for (let [from, to] of codePoints) {
281+
from = String.fromCodePoint(from);
282+
if (Array.isArray(to)) {
283+
to = String.fromCodePoint.apply(null, to);
284+
} else {
285+
to = String.fromCodePoint(to);
286+
}
287+
symbols.set(from, to);
288+
}
218289
codePointsFileContent = `export default ${jsesc(codePoints)};\n`;
219290
symbolsFileContent = `export default ${jsesc(symbols)};\n`;
220291
} else {
221-
codePointsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(codePoints)};\n`;
222-
symbolsFileContent = `import { gunzipSync } from 'node:zlib';\n\nexport default ${gzipInline(symbols)};\n`;
292+
codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\nimport decodeMapping from '../../decode-mapping.mjs';\n\nexport default decodeMapping(${brotliInlineBuffer(
293+
encodeMapping(codePoints),
294+
)}.toString());\n`;
295+
// The symbols map is derived from the code points map at runtime
296+
// rather than shipped as a second copy of the same data.
297+
symbolsFileContent = `import codePoints from './code-points.mjs';\nimport mappingToSymbols from '../../mapping-to-symbols.mjs';\n\nexport default mappingToSymbols(codePoints);\n`;
223298
}
224299
codePointsType = `Map<number, ${[...valueTypes].sort().join(' | ') || 'number'}>`;
225300
symbolsType = 'Map<string, string>';
@@ -228,15 +303,26 @@ const writeFiles = (options) => {
228303
path.resolve(dir, 'code-points.mjs'),
229304
codePointsFileContent,
230305
);
231-
writeFileIfChanged(
232-
path.resolve(dir, 'code-points.d.mts'),
233-
`declare const codePoints: ${codePointsType};\nexport default codePoints;\n`,
234-
);
235306
writeFileIfChanged(path.resolve(dir, 'symbols.mjs'), symbolsFileContent);
236-
writeFileIfChanged(
237-
path.resolve(dir, 'symbols.d.mts'),
238-
`declare const symbols: ${symbolsType};\nexport default symbols;\n`,
239-
);
307+
if (isCodePointMapping) {
308+
// Mapping directories declare their own types, since the value type
309+
// differs per directory (`Map<number, number>` vs. `Map<number, number[]>`).
310+
writeFileIfChanged(
311+
path.resolve(dir, 'code-points.d.mts'),
312+
`declare const codePoints: ${codePointsType};\nexport default codePoints;\n`,
313+
);
314+
writeFileIfChanged(
315+
path.resolve(dir, 'symbols.d.mts'),
316+
`declare const symbols: ${symbolsType};\nexport default symbols;\n`,
317+
);
318+
} else {
319+
// All other directories share the declarations in `types/`, selected
320+
// through `typesVersions` in package.json. Remove per-directory
321+
// declarations left over from earlier builds.
322+
for (const file of SHARED_DECLARATIONS) {
323+
fs.rmSync(path.resolve(dir, file + '.d.mts'), { force: true });
324+
}
325+
}
240326
}
241327
if (options.type === 'Bidi_Mirroring_Glyph') {
242328
const type = options.type;
@@ -272,7 +358,7 @@ const writeFiles = (options) => {
272358
bidiMirroringGlyphFlatPairs,
273359
)}.map((v, i, a) => pair(i & 1, a[i ^ 1], v))`;
274360
// `Bidi_Mirroring_Glyph/index.mjs`
275-
// Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t gzip.
361+
// Note: `Bidi_Mirroring_Glyph` doesn’t have repeated strings; don’t compress.
276362
const output = [
277363
`const chr = String.fromCodePoint;`,
278364
`const pair = (t, u, v) => [t ? u + v : v, chr(t ? u : u + v)];`,
@@ -305,9 +391,14 @@ const writeFiles = (options) => {
305391
// or `bidi-brackets/index.mjs`
306392
// or `Names/index.mjs`
307393
const flatRuns = samePropertyRuns(auxMap[type]);
308-
const output = `import { gunzipSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${gzipInline(
309-
flatRuns,
310-
)});\n`;
394+
const output =
395+
type === 'Names'
396+
? `import { brotliDecompressSync } from 'node:zlib';\nimport decodeNames from '../decode-names.mjs';\n\nexport default decodeNames(${brotliInlineBuffer(
397+
encodeNames(flatRuns),
398+
)}.toString());\n`
399+
: `import { brotliDecompressSync } from 'node:zlib';\nimport decodePropertyMap from '../decode-property-map.mjs';\n\nexport default decodePropertyMap(${brotliInline(
400+
flatRuns,
401+
)});\n`;
311402
writeFileIfChanged(path.resolve(dir, 'index.mjs'), output);
312403
writeFileIfChanged(
313404
path.resolve(dir, 'index.d.mts'),
@@ -361,7 +452,26 @@ const isVersionBefore = (version, target) => {
361452
return false;
362453
};
363454

455+
// The `typesVersions` map for package.json. TypeScript picks the most specific
456+
// matching pattern, so mapping directories keep their own declarations while
457+
// everything else resolves to the shared ones in `types/`.
458+
const typesVersions = (dirMap) => {
459+
const map = {};
460+
for (const type of MAPPING_TYPES) {
461+
if (hasKey(dirMap, type)) {
462+
map[`${type}/*/code-points.mjs`] = [`${type}/*/code-points.d.mts`];
463+
map[`${type}/*/symbols.mjs`] = [`${type}/*/symbols.d.mts`];
464+
}
465+
}
466+
for (const file of SHARED_DECLARATIONS) {
467+
map[`*/${file}.mjs`] = [`types/${file}.d.mts`];
468+
}
469+
return { '*': map };
470+
};
471+
364472
export default {
473+
SHARED_DECLARATIONS,
474+
typesVersions,
365475
range,
366476
append,
367477
extend,

‎static/decode-mapping.d.mts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* Decode a delta-encoded mapping from code points to code point(s), e.g. a
3+
* case mapping or a decomposition mapping.
4+
*/
5+
declare function decodeMapping(input: string): Map<number, number | number[]>;
6+
7+
export default decodeMapping;

0 commit comments

Comments
 (0)