Apply the AL default for unlisted assigned Line_Break code points

Before Unicode 4.0, LineBreak.txt gives unlisted assigned characters the
value AL, and only unassigned code points XX. The parser gave every
unlisted code point XX, so the private-use code points in Unicode 3.1.0
through 3.2.0 had Line_Break=Unknown instead of Alphabetic.

Assisted-By: Claude Opus 5.5

Author: JLHwung

scripts/parse-line-break.mjs

@@ -1,6 +1,7 @@
 import valueAliases from 'unicode-property-value-aliases';
 import utils from './utils.mjs';
 import regenerate from 'regenerate';
+import { parseDerivedGeneralCategory } from './parse-blocks-scripts-properties.mjs';
 
 const aliases = valueAliases.get('Line_Break');
 
@@ -24,30 +25,37 @@ const parseLineBreak = async (version) => {
 		// are given the value `XX`.
 		Unknown: regenerate().addRange(0, 0x10ffff),
 	};
+	// Before Unicode 4.0, only unassigned code points default to `XX`:
+	// “Assigned characters that are not listed explicitly are given the
+	// value "AL".”
+	const assignedDefault =
+		/Assigned characters that are not listed explicitly are given the value\s*#?\s*"(\w+)"/.exec(
+			source,
+		)?.[1];
 	const lines = source.split('\n');
 	for (const line of lines) {
 		const trimmed = line.trim();
 		if (!trimmed || trimmed.startsWith('#')) {
 			continue;
 		}
 		const data = trimmed.split(';');
-		const charRange = data[0].replace('..', '-').trim();
-		const rangeParts = charRange.split('-');
 		const value = data[1].split('#')[0].trim();
 		const canonicalName = findCanonicalName(value);
+		const [from, to = from] = data[0]
+			.trim()
+			.split('..')
+			.map((codePoint) => parseInt(codePoint, 16));
 		map[canonicalName] ??= regenerate();
-		if (rangeParts.length === 2) {
-			const [from, to] = [
-				parseInt(rangeParts[0], 16),
-				parseInt(rangeParts[1], 16),
-			];
-			map['Unknown'].removeRange(from, to);
-			map[canonicalName].addRange(from, to);
-		} else {
-			const codePoint = parseInt(charRange, 16);
-			map['Unknown'].remove(codePoint);
-			map[canonicalName].add(codePoint);
-		}
+		map['Unknown'].removeRange(from, to);
+		map[canonicalName].addRange(from, to);
+	}
+	if (assignedDefault) {
+		const canonicalName = findCanonicalName(assignedDefault);
+		const { Assigned } = await parseDerivedGeneralCategory(version);
+		const unlistedAssigned = map['Unknown'].clone().intersection(Assigned);
+		map['Unknown'].remove(unlistedAssigned);
+		map[canonicalName] ??= regenerate();
+		map[canonicalName].add(unlistedAssigned);
 	}
 
 	return map;

tests/parsers.mjs

@@ -9,6 +9,7 @@ import parseIndicConjunctBreak from '../scripts/parse-indic-conjunct-break.mjs';
 import parseJoiningType from '../scripts/parse-joining-type.mjs';
 import parseSimpleCaseMapping from '../scripts/parse-simple-case-mapping.mjs';
 import parseNameAliases from '../scripts/parse-name-aliases.mjs';
+import parseLineBreak from '../scripts/parse-line-break.mjs';
 import {
 	parseBlocks,
 	parseDerivedCoreProperties,
@@ -387,6 +388,24 @@ test('Historical Script values use canonical names', async (t) => {
 	}
 });
 
+test('Early Line_Break data gives unlisted assigned code points AL', async (t) => {
+	for (const version of ['3.1.0', '3.1.1', '3.2.0']) {
+		const lineBreak = await parseLineBreak(version);
+		for (const cp of [0xe000, 0xf8ff, 0xf0000, 0x10fffd]) {
+			const message = `${version}: ${cp.toString(16)}`;
+			t.assert.ok(lineBreak.Alphabetic.contains(cp), message);
+			t.assert.ok(!lineBreak.Unknown.contains(cp), message);
+		}
+		// Unassigned code points keep XX.
+		t.assert.ok(lineBreak.Unknown.contains(0x0378), version);
+		t.assert.ok(lineBreak.Unknown.contains(0xfffff), version);
+	}
+	// Unicode 4.0 gives every unlisted code point XX.
+	const lineBreak = await parseLineBreak('4.0.0');
+	t.assert.ok(lineBreak.Unknown.contains(0xe000));
+	t.assert.ok(!lineBreak.Alphabetic.contains(0xe000));
+});
+
 test('Unicode 3.1.1 uses its corrected property list', async (t) => {
 	const v310 = await parseProperties('3.1.0');
 	const v311 = await parseProperties('3.1.1');