perf: delta-encode case and decomposition mappings

The `code-points.mjs` of Case_Folding, Simple_Case_Mapping,
Special_Casing and Decomposition_Mapping stored the map as compressed
JSON `[from, to]` pairs. Mapped code points are almost always close to
their keys, so the map is now written as a stream of variable-length
integers using the same base64 scheme as `ranges.mjs`: per entry, the
zigzag-encoded difference between its key and the previous key, a count,
and the zigzag-encoded differences between consecutive mapped code
points. A count of 0 marks an entry mapping to a single code point, so
files mixing scalar and array values (e.g. Unicode 3.0.1 Case_Folding/E)
round-trip exactly. The stream is brotli-compressed as before and decoded
by the new `decode-mapping.mjs`; `decode-ranges.mjs` now also exports
`decodeDeltas` for it. All 1,699 maps across the 35 versions are
identical to the previous output, entry order included.

For Unicode 18.0.0, `Simple_Case_Mapping/Uppercase/code-points.mjs`
shrinks from 4.1 KB to 1.1 KB, and the package from 390 KB to 378 KB
packed (1.39 MB to 1.38 MB unpacked).

Assisted-By: Claude Fable 5.1
Issue: #109

Author: JLHwung

index.mjs

@@ -355,6 +355,8 @@ const generateData = async (version) => {
 		'mapping-to-symbols.d.mts',
 		'decode-names.mjs',
 		'decode-names.d.mts',
+		'decode-mapping.mjs',
+		'decode-mapping.d.mts',
 	];
 	for (const file of staticFiles) {
 		fs.copyFileSync(

scripts/encode-ranges.mjs

@@ -57,4 +57,4 @@ const encodeRegenerate = (regenerateSet) => {
 	return encodeDeltas(deltas);
 };
 
-export { base64enc, encodeRanges, encodeRegenerate };
+export { base64enc, encodeDeltas, encodeRanges, encodeRegenerate };

scripts/utils.mjs

@@ -4,7 +4,12 @@ import { fileURLToPath } from 'node:url';
 import zlib from 'node:zlib';
 import jsesc from 'jsesc';
 import regenerate from 'regenerate';
-import { base64enc, encodeRanges, encodeRegenerate } from './encode-ranges.mjs';
+import {
+	base64enc,
+	encodeDeltas,
+	encodeRanges,
+	encodeRegenerate,
+} from './encode-ranges.mjs';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -112,6 +117,35 @@ const encodeNames = (flatRuns) => {
 	return JSON.stringify(runs) + '\n' + lines.join('\n');
 };
 
+// Zigzag encoding maps signed integers to unsigned ones: 0 → 0, -1 → 1, 1 → 2,
+// -2 → 3, … so that small differences in either direction stay small.
+const zigzag = (value) => ((value << 1) ^ (value >> 31)) >>> 0;
+
+// Delta-encode a mapping from code points to code point(s), e.g. a case
+// mapping or a decomposition mapping, preserving entry order and whether
+// each entry maps to a single code point or to an array.
+// See static/decode-mapping.mjs for the format and the decoder.
+const encodeMapping = (codePoints) => {
+	const output = [];
+	let previousFrom = 0;
+	for (const [from, to] of codePoints) {
+		output.push(zigzag(from - previousFrom));
+		previousFrom = from;
+		let previous = from;
+		if (Array.isArray(to)) {
+			console.assert(to.length > 0, `empty mapping for U+${from.toString(16)}`);
+			output.push(to.length);
+			for (const codePoint of to) {
+				output.push(zigzag(codePoint - previous));
+				previous = codePoint;
+			}
+		} else {
+			output.push(0, zigzag(to - previous));
+		}
+	}
+	return encodeDeltas(output);
+};
+
 // Only write the file if its content has changed.
 // Reading files is cheaper than writing files, and most files don’t change between builds.
 const writeFileIfChanged = (filePath, content) => {
@@ -258,7 +292,9 @@ const writeFiles = (options) => {
 				codePointsFileContent = `export default ${jsesc(codePoints)};\n`;
 				symbolsFileContent = `export default ${jsesc(symbols)};\n`;
 			} else {
-				codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\n\nexport default ${brotliInline(codePoints)};\n`;
+				codePointsFileContent = `import { brotliDecompressSync } from 'node:zlib';\nimport decodeMapping from '../../decode-mapping.mjs';\n\nexport default decodeMapping(${brotliInlineBuffer(
+					encodeMapping(codePoints),
+				)}.toString());\n`;
 				// The symbols map is derived from the code points map at runtime
 				// rather than shipped as a second copy of the same data.
 				symbolsFileContent = `import codePoints from './code-points.mjs';\nimport mappingToSymbols from '../../mapping-to-symbols.mjs';\n\nexport default mappingToSymbols(codePoints);\n`;

static/decode-mapping.d.mts

@@ -0,0 +1,7 @@
+/**
+ * Decode a delta-encoded mapping from code points to code point(s), e.g. a
+ * case mapping or a decomposition mapping.
+ */
+declare function decodeMapping(input: string): Map<number, number | number[]>;
+
+export default decodeMapping;

static/decode-mapping.mjs

@@ -0,0 +1,46 @@
+/**
+ * This file ships to end users.
+ * See `encodeMapping` in scripts/utils.mjs for the encoder.
+ */
+
+import { decodeDeltas } from './decode-ranges.mjs';
+
+// Undo zigzag encoding: 0 → 0, 1 → -1, 2 → 1, 3 → -2, …
+const unzigzag = (value) => (value >>> 1) ^ -(value & 1);
+
+/**
+ * Decode a delta-encoded mapping from code points to code point(s), e.g. a
+ * case mapping or a decomposition mapping.
+ *
+ * `input` is a base64 stream of variable-length integers (see
+ * `decodeDeltas`). Each entry consists of the zigzag-encoded difference
+ * between its key and the previous key, a count, and then `count` values
+ * (or one value, if `count` is 0). Each value is the zigzag-encoded difference
+ * from the previous code point in the entry, starting from the key. A count
+ * of 0 means the entry maps to a single code point rather than to an array.
+ * @param {string} input
+ * @returns {Map<number, number | number[]>}
+ */
+const decodeMapping = (input) => {
+	const deltas = decodeDeltas(input);
+	const map = new Map();
+	for (let from = 0, i = 0; i < deltas.length;) {
+		from += unzigzag(deltas[i++]);
+		const count = deltas[i++];
+		let codePoint = from;
+		if (count === 0) {
+			codePoint += unzigzag(deltas[i++]);
+			map.set(from, codePoint);
+		} else {
+			const codePoints = [];
+			for (let j = 0; j < count; j++) {
+				codePoint += unzigzag(deltas[i++]);
+				codePoints.push(codePoint);
+			}
+			map.set(from, codePoints);
+		}
+	}
+	return map;
+};
+
+export default decodeMapping;

static/decode-ranges.d.mts

@@ -17,4 +17,10 @@ declare namespace decodeRanges {
  */
 declare function decodeRanges(input: string): decodeRanges.UnicodeRange[];
 
+/**
+ * Base64 decode variable-length deltas (5/10/15/21-bit).
+ */
+declare function decodeDeltas(input: string): number[];
+
+export { decodeDeltas };
 export default decodeRanges;

static/decode-ranges.mjs

@@ -82,4 +82,5 @@ const decodeRanges = (input) => {
 	return ranges;
 };
 
+export { decodeDeltas };
 export default decodeRanges;