web(build): the build id covers everything that ships under it

A release puts everything under /_/<id>/ and tells browsers it never changes. The id was a hash of the client, the
page and the extension's top-level .js files, so a changed scratch-workspace extension, shim, media file or skill
kept its id — and would have been served stale, from cache, for a year. The hash now covers the extension's whole
tree, its browser build, the scratch workspace and the page's configuration code.

Author: ndemianc

scripts/build-web.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // Build LevelCode in your browser: one static directory, dist-web/, that any static host can serve.
 //
-//   node scripts/build-web.mjs [--vscode vscode] [--out dist-web] [--account https://levelcode.ai]
+//   node scripts/build-web.mjs [--vscode vscode] [--out dist-web] [--account https://levelcode.ai] [--api-url <origin>]
 //                              [--webview-origin 'https://{{uuid}}.view.example.com']
 //                              [--ext-host-origin 'https://{{uuid}}.ext.example.com']
 //                              [--static <prebuilt vscode-web dir>] [--id <build id>]
@@ -23,6 +23,7 @@ import { fileURLToPath } from 'node:url';
 import { webConfig, renderIndex } from '../web/lib/config.mjs';
 import { stageDeclarativeExtensions, copyExtension } from '../web/lib/extensions.mjs';
 import { policy, toHeadersFile, toNginx } from '../web/lib/headers.mjs';
+import { hashTree } from '../web/lib/fingerprint.mjs';
 
 const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const WEB = path.join(REPO, 'web');
@@ -34,6 +35,8 @@ const args = Object.fromEntries(process.argv.slice(2).reduce((a, v, i, all) => {
 const vscodeDir = path.resolve(REPO, args.vscode || 'vscode');
 const outDir = path.resolve(REPO, args.out || 'dist-web');
 const account = String(args.account || process.env.LEVELCODE_WEB_ACCOUNT || 'https://levelcode.ai').replace(/\/+$/, '');
+// Only for a deployment whose API is not on the account origin (production has one host for both).
+const apiUrl = String(args['api-url'] || process.env.LEVELCODE_WEB_API_URL || '').replace(/\/+$/, '');
 const webviewOrigin = args['webview-origin'] || process.env.LEVELCODE_WEB_WEBVIEW_ORIGIN || '';
 const extHostOrigin = args['ext-host-origin'] || process.env.LEVELCODE_WEB_EXTHOST_ORIGIN || '';
 
@@ -65,8 +68,13 @@ for (const must of ['out/vs/workbench/workbench.web.main.internal.js', 'out/vs/w
 const fingerprint = createHash('sha256');
 fingerprint.update(fs.readFileSync(path.join(staticDir, 'out/vs/workbench/workbench.web.main.internal.js')));
 for (const f of ['main.js', 'boot.css', 'index.html']) { fingerprint.update(fs.readFileSync(path.join(WEB, f))); }
+// Everything that goes into /_/<id>/ and is not the web client itself: the extension (its sources, media and skills),
+// its browser build, the scratch workspace, and the page's own configuration code.
 const aiDir = path.join(REPO, 'extensions', 'levelcode-ai');
-for (const f of fs.readdirSync(aiDir)) { if (f.endsWith('.js')) { fingerprint.update(fs.readFileSync(path.join(aiDir, f))); } }
+hashTree(fingerprint, aiDir, { skip: ['test', 'node_modules', 'scripts', '.DS_Store'] });
+hashTree(fingerprint, path.join(WEB, 'ai-extension'), { skip: ['assets.generated.js', 'node_modules', '.DS_Store'] });
+hashTree(fingerprint, path.join(WEB, 'workspace'), { skip: ['node_modules', '.DS_Store'] });
+hashTree(fingerprint, path.join(WEB, 'lib'), { skip: ['.DS_Store'] });
 const id = String(args.id || fingerprint.digest('hex').slice(0, 12));
 let gitHead = 'unknown';
 try { gitHead = execFileSync('git', ['-C', REPO, 'rev-parse', '--short=10', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { /* not a checkout */ }
@@ -124,6 +132,7 @@ if (webviewOrigin) { productConfiguration.webviewContentExternalBaseUrlTemplate
 if (extHostOrigin) { productConfiguration.webEndpointUrlTemplate = `${extHostOrigin}${prefix}/static`; }
 const config = webConfig({
 	account,
+	apiUrl,
 	extensions,
 	base: prefix,
 	staticBase: `${prefix}/static`,
@@ -142,7 +151,7 @@ fs.writeFileSync(path.join(outDir, '_headers'), toHeadersFile(pol));
 fs.mkdirSync(path.join(outDir, 'deploy'), { recursive: true });
 fs.writeFileSync(path.join(outDir, 'deploy', 'nginx.conf'), toNginx(pol));
 const info = {
-	id, levelcode: gitHead, builtAt: new Date().toISOString(), account,
+	id, levelcode: gitHead, builtAt: new Date().toISOString(), account, apiUrl: apiUrl || '(the account origin)',
 	webviewOrigin: webviewOrigin || '(Code-OSS default: *.vscode-cdn.net)',
 	extHostOrigin: extHostOrigin || '(same origin as the editor)',
 	extensions,

web/lib/fingerprint.mjs

@@ -0,0 +1,29 @@
+// What identifies a build. A release puts everything under /_/<id>/ and tells browsers it never changes, so the id has to
+// change whenever anything under it does: a file of the extension that was edited but not named here would be served
+// stale, from cache, for a year.
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Feed every file under `dir` — its path relative to `dir`, then its bytes — into `hash`, in a stable order.
+ * Files and folders whose NAME is in `skip` are left out wherever they are.
+ * @param {import('node:crypto').Hash} hash
+ * @param {string} dir
+ * @param {{ skip?: string[] }} [opts]
+ * @returns {import('node:crypto').Hash}
+ */
+export function hashTree(hash, dir, { skip = [] } = {}) {
+	const skipped = new Set(skip);
+	const walk = (d, rel) => {
+		const entries = fs.readdirSync(d, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+		for (const e of entries) {
+			if (skipped.has(e.name)) { continue; }
+			const full = path.join(d, e.name);
+			const r = rel ? rel + '/' + e.name : e.name;
+			if (e.isDirectory()) { walk(full, r); }
+			else if (e.isFile()) { hash.update('\0' + r + '\0'); hash.update(fs.readFileSync(full)); }
+		}
+	};
+	walk(dir, '');
+	return hash;
+}

web/test/unit/fingerprint.test.js

@@ -0,0 +1,67 @@
+/*---------------------------------------------------------------------------------------------
+ *  web/lib/fingerprint.mjs — run: node web/test/unit/fingerprint.test.js
+ *
+ *  A release is cached for a year under /_/<id>/. The id is a hash of what goes into it, so a changed file that the
+ *  hash did not see is a stale file for a year. These tests are about what the hash sees.
+ *--------------------------------------------------------------------------------------------*/
+// @ts-check
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { createHash } = require('crypto');
+const { pathToFileURL } = require('url');
+
+let n = 0;
+async function test(name, fn) { await fn(); n++; console.log('  ok - ' + name); }
+const write = (root, rel, text) => { const f = path.join(root, rel); fs.mkdirSync(path.dirname(f), { recursive: true }); fs.writeFileSync(f, text); };
+
+(async () => {
+	const { hashTree } = await import(pathToFileURL(path.join(__dirname, '..', '..', 'lib', 'fingerprint.mjs')).href);
+	const digest = (dir, opts) => hashTree(createHash('sha256'), dir, opts).digest('hex');
+	const make = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-fp-')); write(d, 'a.js', 'A'); write(d, 'sub/b.json', '{"b":1}'); write(d, 'sub/deep/c.txt', 'C'); return d; };
+
+	await test('the same files give the same hash, wherever they are and in whatever order they were written', () => {
+		const one = make();
+		const two = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-fp-'));
+		write(two, 'sub/deep/c.txt', 'C'); write(two, 'sub/b.json', '{"b":1}'); write(two, 'a.js', 'A');
+		assert.strictEqual(digest(one), digest(two));
+	});
+	await test('a changed byte in any file, at any depth, changes it', () => {
+		const d = make();
+		const before = digest(d);
+		for (const rel of ['a.js', 'sub/b.json', 'sub/deep/c.txt']) {
+			const original = fs.readFileSync(path.join(d, rel), 'utf8');
+			fs.writeFileSync(path.join(d, rel), original + 'x');
+			assert.notStrictEqual(digest(d), before, rel + ' was not seen');
+			fs.writeFileSync(path.join(d, rel), original);
+		}
+		assert.strictEqual(digest(d), before);
+	});
+	await test('a new file, a removed file and a renamed file change it', () => {
+		const d = make();
+		const before = digest(d);
+		write(d, 'new.js', '');
+		assert.notStrictEqual(digest(d), before);
+		fs.rmSync(path.join(d, 'new.js'));
+		assert.strictEqual(digest(d), before);
+		fs.renameSync(path.join(d, 'a.js'), path.join(d, 'z.js'));
+		assert.notStrictEqual(digest(d), before, 'the same bytes under another name are another build');
+	});
+	await test('where the bytes end and the name begins cannot be confused', () => {
+		const one = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-fp-')); write(one, 'ab', 'c');
+		const two = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-fp-')); write(two, 'a', 'bc');
+		assert.notStrictEqual(digest(one), digest(two));
+	});
+	await test('skipped names are left out wherever they are, and nothing else is', () => {
+		const d = make();
+		write(d, 'test/x.test.js', 't'); write(d, 'sub/node_modules/y.js', 'y');
+		const without = digest(d, { skip: ['test', 'node_modules'] });
+		write(d, 'test/x.test.js', 'changed'); write(d, 'sub/node_modules/y.js', 'changed');
+		assert.strictEqual(digest(d, { skip: ['test', 'node_modules'] }), without);
+		assert.notStrictEqual(digest(d), without);
+	});
+	console.log(`\n${n} tests passed`);
+})().catch((e) => { console.error(e); process.exit(1); });