Repository navigation
Expand file tree
/
Copy pathbuild-web.mjs
More file actions
211 lines (194 loc) · 14.3 KB
/
Copy pathbuild-web.mjs
File metadata and controls
211 lines (194 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
#!/usr/bin/env node
// Build LevelCode in your browser: one static directory, dist-web/, that any static host can serve.
//
// node scripts/build-web.mjs [--vscode vscode] [--out dist-web] [--account https://levelcode.ai] [--api-url <origin>]
// [--webview-origin 'https://{{uuid}}.view.example.com']
// [--ext-host-origin 'https://{{uuid}}.ext.example.com']
// [--static <prebuilt vscode-web dir>] [--id <build id>]
//
// What it does, in order (docs/WEB.md has the reasoning):
// 1. builds Code-OSS's web client from the bootstrapped checkout (gulp vscode-web-min-ci, about a
// minute), or takes a prebuilt one with --static;
// 2. bundles levelcode-ai for the web worker extension host (web/ai-extension), copies the scratch
// workspace extension (web/workspace) and every declarative built-in (grammars, themes, icons);
// 3. names the build after a hash of everything it staged, lays that out under the build-addressed
// prefix (/_/<id>/...), renders index.html with the configuration baked in, and writes the HTTP
// policy for the host (_headers, nginx) and, for Cloudflare Pages, a _redirects.
//
// It changes nothing outside --out, except that building the client from the checkout leaves gulp's own output,
// vscode-web/, next to it (pass --static to use a client built earlier and skip that). It does not deploy.
import { spawnSync, execFileSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { webConfig, renderIndex } from '../web/lib/config.mjs';
import { stageDeclarativeExtensions, copyExtension, withoutConfigurationDefaults } from '../web/lib/extensions.mjs';
import { policy, toHeadersFile, toNginx } from '../web/lib/headers.mjs';
import { hashTree } from '../web/lib/fingerprint.mjs';
import { NODE_MODULES_COPY, toRedirectsFile } from '../web/lib/pages.mjs';
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const WEB = path.join(REPO, 'web');
const args = Object.fromEntries(process.argv.slice(2).reduce((a, v, i, all) => {
if (v.startsWith('--')) { a.push([v.slice(2), all[i + 1] && !all[i + 1].startsWith('--') ? all[i + 1] : 'true']); }
return a;
}, []));
const vscodeDir = path.resolve(REPO, args.vscode || 'vscode');
const outDir = path.resolve(REPO, args.out || 'dist-web');
const account = String(args.account || process.env.LEVELCODE_WEB_ACCOUNT || 'https://levelcode.ai').replace(/\/+$/, '');
// Only for a deployment whose API is not on the account origin (production has one host for both).
const apiUrl = String(args['api-url'] || process.env.LEVELCODE_WEB_API_URL || '').replace(/\/+$/, '');
const webviewOrigin = args['webview-origin'] || process.env.LEVELCODE_WEB_WEBVIEW_ORIGIN || '';
const extHostOrigin = args['ext-host-origin'] || process.env.LEVELCODE_WEB_EXTHOST_ORIGIN || '';
const log = (m) => console.log('[build-web] ' + m);
const die = (m) => { console.error('[build-web] ' + m); process.exit(1); };
const sizeOf = (p) => { let n = 0; const walk = (d) => { for (const e of fs.readdirSync(d, { withFileTypes: true })) { const f = path.join(d, e.name); e.isDirectory() ? walk(f) : (n += fs.statSync(f).size); } }; walk(p); return n; };
const mb = (n) => (n / 1048576).toFixed(1) + ' MB';
/** Every folder under `dir` named `name`, not looking inside one that is. */
const directoriesNamed = (dir, name, found = []) => {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (!e.isDirectory()) { continue; }
const f = path.join(dir, e.name);
if (e.name === name) { found.push(f); } else { directoriesNamed(f, name, found); }
}
return found;
};
/* ----- 1. Code-OSS's web client ---------------------------------------------------------------- */
let staticDir;
if (args.static) {
staticDir = path.resolve(args.static);
log('using the prebuilt web client at ' + staticDir);
} else {
if (!fs.existsSync(path.join(vscodeDir, 'node_modules'))) { die('no bootstrapped checkout at ' + vscodeDir + ' — run scripts/bootstrap.sh first'); }
const product = JSON.parse(fs.readFileSync(path.join(vscodeDir, 'product.json'), 'utf8'));
if (product.nameShort !== 'LevelCode') { die('the checkout is not branded (product.json nameShort is "' + product.nameShort + '") — run scripts/apply-branding.mjs'); }
log('building the web client (gulp vscode-web-min-ci) in ' + vscodeDir);
const r = spawnSync('npm', ['run', 'gulp', '--', 'vscode-web-min-ci'], { cwd: vscodeDir, stdio: 'inherit', env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=8192' } });
if (r.status !== 0) { die('the web client build failed'); }
staticDir = path.resolve(vscodeDir, '..', 'vscode-web');
}
for (const must of ['out/vs/workbench/workbench.web.main.internal.js', 'out/vs/workbench/workbench.web.main.internal.css', 'out/nls.messages.js',
'out/vs/workbench/services/extensions/worker/webWorkerExtensionHostIframe.html', 'out/vs/workbench/contrib/webview/browser/pre/index.html']) {
if (!fs.existsSync(path.join(staticDir, must))) { die('the web client is missing ' + must); }
}
/* ----- 2. assemble, under a name the build does not have yet ------------------------------------- */
// Everything under /_/<id>/ is served as never changing, so the id is a hash of exactly what is there: it is put
// together first, in a staging folder, and named after it. A list of inputs to hash missed what it did not name —
// a theme, a grammar, the web client's CSS or a worker — and a changed file was then served stale for a year.
// Nothing staged depends on the id; only the page (index.html, outside it) names it.
fs.rmSync(outDir, { recursive: true, force: true });
const staging = path.join(outDir, '_', '.staging');
fs.mkdirSync(staging, { recursive: true });
let root = staging;
log('copying the web client');
fs.cpSync(staticDir, path.join(root, 'static'), { recursive: true, filter: (src) => !/\.(js|css)\.map$/.test(src) });
// The client loads its terminal (xterm), its syntax colouring (vscode-textmate, vscode-oniguruma) and more from
// static/node_modules. Cloudflare Pages is never sent a folder by that name (wrangler leaves out **/node_modules): the
// same files are staged again as static/node-modules, and the release's _redirects (written below) serves them from
// there at the address the client asks for. Any other host serves node_modules itself and ignores _redirects.
const clientModules = path.join(root, 'static', 'node_modules');
if (fs.existsSync(clientModules)) { fs.cpSync(clientModules, path.join(root, 'static', NODE_MODULES_COPY), { recursive: true }); }
const baked = new Set(fs.existsSync(path.join(staticDir, 'extensions')) ? fs.readdirSync(path.join(staticDir, 'extensions')) : []);
const extOut = path.join(root, 'extensions');
fs.mkdirSync(extOut, { recursive: true });
log('bundling levelcode-ai for the web worker extension host');
const r = spawnSync(process.execPath, [path.join(WEB, 'ai-extension', 'build.mjs'), '--out', extOut, '--vscode', vscodeDir, '--minify'], { stdio: 'inherit' });
if (r.status !== 0) { die('the levelcode-ai browser build failed'); }
log('bundling levelcode-shell (the shell extension and its engine worker)');
const rShell = spawnSync(process.execPath, [path.join(WEB, 'shell', 'build.mjs'), '--out', extOut, '--vscode', vscodeDir, '--minify'], { stdio: 'inherit' });
if (rShell.status !== 0) { die('the levelcode-shell build failed'); }
log('staging the scratch workspace and the declarative built-ins');
const scratchOut = path.join(extOut, 'levelcode-web');
copyExtension(path.join(WEB, 'workspace'), scratchOut);
fs.writeFileSync(path.join(scratchOut, 'package.nls.json'), '{}\n');
const extensionsSource = path.join(vscodeDir, 'extensions');
if (!fs.existsSync(extensionsSource)) { die('no extensions/ in ' + vscodeDir + ' to take the declarative built-ins from'); }
const declarative = stageDeclarativeExtensions(extensionsSource, extOut, { exclude: [...baked, 'levelcode-ai', 'levelcode-web', 'levelcode-shell'] });
// A checkout that has not had LevelCode's extensions copied in still builds; the themes are then absent. A client built
// from a checkout that has them has them baked in, and a second copy here would be skipped with an error for each of
// its translations.
if (!declarative.includes('levelcode-themes') && !baked.has('levelcode-themes')) {
const themes = path.join(REPO, 'extensions', 'levelcode-themes');
if (fs.existsSync(themes)) { copyExtension(themes, path.join(extOut, 'levelcode-themes')); declarative.push('levelcode-themes'); }
}
// The browser edition's own colour themes. The desktop's default theme is pinned by levelcode-themes' manifest
// (an extension's defaults outrank the page's), so that one is dropped here; its themes stay selectable.
const themeOut = path.join(extOut, 'levelcode-web-theme');
copyExtension(path.join(WEB, 'theme', 'levelcode-web-theme'), themeOut);
fs.writeFileSync(path.join(themeOut, 'package.nls.json'), '{}\n');
const desktopThemes = path.join(extOut, 'levelcode-themes', 'package.json');
if (fs.existsSync(desktopThemes)) { withoutConfigurationDefaults(desktopThemes); }
const extensions = ['levelcode-ai', 'levelcode-web', 'levelcode-web-theme', 'levelcode-shell', ...declarative.sort()];
log(`${extensions.length} extensions: levelcode-ai, levelcode-web, levelcode-web-theme, levelcode-shell and ${declarative.length} declarative (${declarative.slice(0, 6).join(', ')}, ...)`);
// main.js and boot.css are addressed by the build id like everything else; the page names them.
for (const f of ['main.js', 'boot.css', 'chrome.css']) { fs.copyFileSync(path.join(WEB, f), path.join(root, f)); }
// Nothing else may be in a folder named node_modules: Pages would not have it, and nothing would serve it from elsewhere.
for (const dir of directoriesNamed(staging, 'node_modules')) {
if (dir !== clientModules) { die(path.relative(staging, dir) + ' is a node_modules folder, which Cloudflare Pages is never sent'); }
}
/* ----- 3. identity of this build: what was staged ------------------------------------------------- */
const id = String(args.id || hashTree(createHash('sha256'), staging, { skip: ['.DS_Store'] }).digest('hex').slice(0, 12));
const prefix = `/_/${id}`;
root = path.join(outDir, '_', id);
fs.renameSync(staging, root);
let gitHead = 'unknown';
try { gitHead = execFileSync('git', ['-C', REPO, 'rev-parse', '--short=10', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { /* not a checkout */ }
log('build id ' + id + ' (levelcode ' + gitHead + ')');
log('writing the page');
const productConfiguration = {
// No gallery: nothing is installed from the Internet into the page's own origin.
extensionsGallery: null,
};
if (!webviewOrigin) {
// Code-OSS publishes each stable commit's webview host pages to its CDN. product.json pins an older insider commit,
// whose service worker is a version behind this build's; the template below names this build's own commit.
productConfiguration.webviewContentExternalBaseUrlTemplate = 'https://{{uuid}}.vscode-cdn.net/{{quality}}/{{commit}}/out/vs/workbench/contrib/webview/browser/pre/';
console.warn('[build-web] note: no --webview-origin, so the chat panel loads its host page from Code-OSS\'s CDN (*.vscode-cdn.net, this build\'s own commit).\n'
+ ' That works with no setup. To serve it yourself give a wildcard origin, e.g. --webview-origin \'https://{{uuid}}.view.example.com\'\n'
+ ' (each webview needs its own subdomain; see docs/WEB.md).');
}
if (webviewOrigin) { productConfiguration.webviewContentExternalBaseUrlTemplate = `${webviewOrigin}${prefix}/static/out/vs/workbench/contrib/webview/browser/pre/`; }
if (extHostOrigin) { productConfiguration.webEndpointUrlTemplate = `${extHostOrigin}${prefix}/static`; }
const config = webConfig({
account,
apiUrl,
extensions,
base: prefix,
staticBase: `${prefix}/static`,
extensionsBase: `${prefix}/extensions`,
productConfiguration,
});
fs.writeFileSync(path.join(outDir, 'index.html'), renderIndex(config));
for (const f of ['callback.html', 'callback.js', 'callback.css', 'icon.svg', 'icon-192.png', 'icon-512.png', 'manifest.webmanifest']) {
fs.copyFileSync(path.join(WEB, f), path.join(outDir, f));
}
/* ----- 4. the host's half ------------------------------------------------------------------------ */
const toPattern = (o) => (o ? o.replace('{{uuid}}', '*') : '');
const pol = policy({ account, webviewOrigin: toPattern(webviewOrigin), extHostOrigin: toPattern(extHostOrigin), base: '/_' });
fs.writeFileSync(path.join(outDir, '_headers'), toHeadersFile(pol));
if (fs.existsSync(path.join(root, 'static', NODE_MODULES_COPY))) {
fs.writeFileSync(path.join(outDir, '_redirects'), toRedirectsFile());
}
fs.mkdirSync(path.join(outDir, 'deploy'), { recursive: true });
fs.writeFileSync(path.join(outDir, 'deploy', 'nginx.conf'), toNginx(pol));
const info = {
id, levelcode: gitHead, builtAt: new Date().toISOString(), account, apiUrl: apiUrl || '(the account origin)',
webviewOrigin: webviewOrigin || '(Code-OSS default: *.vscode-cdn.net)',
extHostOrigin: extHostOrigin || '(same origin as the editor)',
extensions,
};
fs.writeFileSync(path.join(outDir, 'build.json'), JSON.stringify(info, null, 2) + '\n');
/* ----- 5. check what was written ----------------------------------------------------------------- */
const html = fs.readFileSync(path.join(outDir, 'index.html'), 'utf8');
const m = html.match(/<script type="application\/json" id="levelcode-web-config">([\s\S]*?)<\/script>/);
if (!m) { die('index.html has no configuration'); }
JSON.parse(m[1]);
if (/\{\{[A-Z]+\}\}/.test(html)) { die('index.html still has an unfilled placeholder'); }
for (const must of [`${prefix}/main.js`, `${prefix}/boot.css`, `${prefix}/static/out/nls.messages.js`, `${prefix}/extensions/levelcode-ai/extension.web.js`,
`${prefix}/extensions/levelcode-ai/package.json`, `${prefix}/extensions/levelcode-web/extension.js`,
`${prefix}/chrome.css`, `${prefix}/extensions/levelcode-web-theme/package.json`,
`${prefix}/extensions/levelcode-shell/package.json`, `${prefix}/extensions/levelcode-shell/extension.js`, `${prefix}/extensions/levelcode-shell/engine.worker.js`]) {
if (!fs.existsSync(path.join(outDir, must))) { die('missing ' + must); }
}
log(`done: ${outDir} ${mb(sizeOf(outDir))} (client ${mb(sizeOf(path.join(root, 'static')))}, extensions ${mb(sizeOf(path.join(root, 'extensions')))})`);
log('serve it with: node web/serve.mjs --dist ' + path.relative(process.cwd(), outDir));