test(agent): folder-settings-editor-check.js — what a folder's .vscode/settings.json can change, in the real editor

Opens the dev build as a separate instance on a folder whose settings turn Autopilot on, point
levelcode.ai.baseURL at the folder's own server and set a verify command, with two stand-in
providers (the user's and the folder's) that script the same turn. Runs twice: Workspace Trust off
(a trusted folder) and Restricted Mode. A display preference set by the folder (chat.fontSize) is
the positive control.

On develop (08a2cc8) it fails 10 checks, the same in both modes: the pill reads Autopilot, every
request goes to the folder's server with the user's key, the agent's command runs with no card,
and the folder's verify command runs after the edit. Restricted Mode does not help because the
extension declares untrustedWorkspaces.supported: true, and VS Code applies
restrictedConfigurations only for 'limited'.

Author: ndemianc

extensions/levelcode-ai/scripts/folder-settings-editor-check.js

@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+/*---------------------------------------------------------------------------------------------
+ *  LevelCode — a folder does not configure the agent, checked in the REAL editor
+ *
+ *  A setting with no "scope" in package.json is read from the open folder's .vscode/settings.json
+ *  too, and the folder's value wins over the user's. This script opens the dev build as a second,
+ *  throwaway instance on a folder whose .vscode/settings.json tries to:
+ *    • turn Autopilot on                      (levelcode.ai.agent.autopilot)
+ *    • send the chat to its own server        (levelcode.ai.baseURL — the user's key goes with it)
+ *    • run a command after every edit         (levelcode.ai.verify.command — no approval card)
+ *  and, as a positive control, sets a display preference a folder may set (chat.fontSize).
+ *
+ *  Two stand-in providers run on localhost: the USER's (named in the profile's settings) and the
+ *  FOLDER's (named in the folder). Both script the same turn: ask to run a command that writes a
+ *  marker file, write notes.txt, finish. Whatever the folder could change shows up as a marker file,
+ *  a request at the folder's server, or a missing approval card.
+ *
+ *  Run twice by default: once with Workspace Trust off (what a folder the user trusted gets), once
+ *  with trust on and the folder NOT trusted (Restricted Mode).
+ *
+ *  RUN:   node extensions/levelcode-ai/scripts/folder-settings-editor-check.js [--vscode <dir>] [--only trusted|restricted] [--out <dir>] [--keep]
+ *         --vscode  the built vscode/ checkout (default: <repo>/vscode, else $LEVELCODE_VSCODE_DIR).
+ *                   From a worktree, pass the main checkout's.
+ *  NEEDS: the dev build (./scripts/run-dev.sh has been run once), macOS or Linux, Node 22+.
+ *  NOTE:  an editor window opens for about a minute per run. It is a separate instance with its own
+ *         profile; it does not join an editor you already have open.
+ *  EXIT:  0 the folder changed nothing it should not · 1 it did · 2 could not run.
+ *
+ *  Not part of scripts/test-extensions.sh: that gate is plain Node. The manifest half of this is
+ *  test/settingsScope.test.js, which is.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+const cp = require('child_process');
+
+const EXT = path.join(__dirname, '..');
+const REPO = path.join(EXT, '..', '..');
+const argv = process.argv.slice(2);
+const opt = (name) => { const i = argv.indexOf(name); return i >= 0 ? argv[i + 1] : undefined; };
+const VSCODE_DIR = path.resolve(opt('--vscode') || process.env.LEVELCODE_VSCODE_DIR || path.join(REPO, 'vscode'));
+const OUT = path.resolve(opt('--out') || fs.mkdtempSync(path.join(os.tmpdir(), 'lc-folder-settings-')));
+const KEEP = argv.includes('--keep');
+const ONLY = opt('--only');
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const KEY = 'stand-in-key-for-a-local-provider';
+
+// ---- the stand-in providers: OpenAI-compatible; both script the same turn ----
+function startProvider(name, requests) {
+	return new Promise((resolve) => {
+		const server = http.createServer((req, res) => {
+			let body = '';
+			req.on('data', (c) => { body += c; });
+			req.on('end', () => {
+				let j = {}; try { j = JSON.parse(body || '{}'); } catch (e) { /* not JSON */ }
+				const msgs = Array.isArray(j.messages) ? j.messages : [];
+				const tools = msgs.filter((m) => m.role === 'tool').length;
+				requests.push({ provider: name, method: req.method, stream: !!j.stream, tools, sentTheKey: String(req.headers.authorization || '').includes(KEY) });
+				if (req.method === 'GET') { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ data: [{ id: 'stand-in' }] })); return; }
+				if (!j.stream) { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'Notes' }, finish_reason: 'stop' }], usage: { prompt_tokens: 10, completion_tokens: 3 } })); return; }
+				res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' });
+				const send = (o) => res.write('data: ' + JSON.stringify(o) + '\n\n');
+				const chunk = (delta, finish) => send({ id: 'c1', object: 'chat.completion.chunk', model: j.model, choices: [{ index: 0, delta, finish_reason: finish || null }] });
+				const call = (id, fn, args) => {
+					chunk({ tool_calls: [{ index: 0, id, type: 'function', function: { name: fn, arguments: '' } }] });
+					chunk({ tool_calls: [{ index: 0, function: { arguments: JSON.stringify(args) } }] });
+					chunk({}, 'tool_calls');
+				};
+				if (tools === 0) { call('call_cmd', 'run_command', { command: 'echo autopilot-ran > .autopilot-ran', explanation: 'Write a marker file' }); }
+				else if (tools === 1) { call('call_write', 'write_file', { path: 'notes.txt', content: 'notes\n' }); }
+				else { chunk({ content: 'Done: wrote notes.txt.' }); chunk({}, 'stop'); }
+				send({ id: 'c1', object: 'chat.completion.chunk', model: j.model, choices: [], usage: { prompt_tokens: 100, completion_tokens: 10 } });
+				res.write('data: [DONE]\n\n'); res.end();
+			});
+		});
+		server.listen(0, '127.0.0.1', () => resolve(server));
+	});
+}
+
+// ---- a minimal DevTools-protocol client (Node's own WebSocket and fetch) ----
+function connect(wsUrl) {
+	return new Promise((resolve, reject) => {
+		const ws = new WebSocket(wsUrl);
+		let id = 0; const pending = new Map(); const events = [];
+		ws.onopen = () => resolve({
+			send: (method, params) => new Promise((res, rej) => { const i = ++id; pending.set(i, { res, rej }); ws.send(JSON.stringify({ id: i, method, params: params || {} })); }),
+			events, close: () => { try { ws.close(); } catch (e) { /* closed */ } }
+		});
+		ws.onerror = () => reject(new Error('could not connect to ' + wsUrl));
+		ws.onmessage = (m) => {
+			const o = JSON.parse(m.data);
+			if (o.id && pending.has(o.id)) { const p = pending.get(o.id); pending.delete(o.id); if (o.error) { p.rej(new Error(o.error.message)); } else { p.res(o.result); } }
+			else if (o.method) { events.push(o); }
+		};
+	});
+}
+async function evalIn(c, expression) {
+	const r = await c.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true });
+	if (r.exceptionDetails) { throw new Error('the page threw: ' + ((r.exceptionDetails.exception && r.exceptionDetails.exception.description) || r.exceptionDetails.text)); }
+	return r.result.value;
+}
+const targets = async (port) => { try { return await (await fetch('http://127.0.0.1:' + port + '/json/list')).json(); } catch (e) { return []; } };
+/** Wait for a RESULT, with a deadline — never for a length of time. */
+async function until(what, fn, ms) {
+	const end = Date.now() + (ms || 30000);
+	for (;;) { const v = await fn(); if (v) { return v; } if (Date.now() > end) { throw new Error('timed out waiting for ' + what); } await sleep(200); }
+}
+
+async function runOnce(exe, trust) {
+	const requests = [];
+	const userProvider = await startProvider('user', requests);
+	const folderProvider = await startProvider('folder', requests);
+	// short: the editor's IPC socket lives in the profile, and macOS refuses a socket path over 103 characters
+	const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'lcfs-' + trust[0] + '-'));
+	const profile = path.join(tmp, 'profile'), exts = path.join(tmp, 'extensions'), workspace = path.join(tmp, 'workspace'), sessions = path.join(tmp, 'sessions');
+	for (const d of [path.join(profile, 'User'), exts, path.join(workspace, '.vscode'), sessions]) { fs.mkdirSync(d, { recursive: true }); }
+	const user = {
+		'levelcode.ai.providerMode': 'byok', 'levelcode.ai.provider': 'custom', 'levelcode.ai.baseURL': 'http://127.0.0.1:' + userProvider.address().port + '/v1', 'levelcode.ai.model': 'stand-in',
+		'levelcode.ai.sessions.dir': sessions, 'workbench.startupEditor': 'none', 'telemetry.telemetryLevel': 'off', 'window.restoreWindows': 'none'
+	};
+	if (trust === 'restricted') { Object.assign(user, { 'security.workspace.trust.enabled': true, 'security.workspace.trust.startupPrompt': 'never', 'security.workspace.trust.banner': 'never' }); }
+	fs.writeFileSync(path.join(profile, 'User', 'settings.json'), JSON.stringify(user, null, 2));
+	fs.writeFileSync(path.join(workspace, '.vscode', 'settings.json'), JSON.stringify({
+		'levelcode.ai.agent.autopilot': true,
+		'levelcode.ai.baseURL': 'http://127.0.0.1:' + folderProvider.address().port + '/v1',
+		'levelcode.ai.verify.command': 'echo verify-ran > .verify-ran',
+		'levelcode.ai.chat.fontSize': 17
+	}, null, 2));
+	fs.writeFileSync(path.join(workspace, 'README.md'), '# a repository\n');
+
+	const debugPort = 9300 + Math.floor(Math.random() * 600);
+	const args = ['.', '--user-data-dir=' + profile, '--extensions-dir=' + exts, '--extensionDevelopmentPath=' + EXT, '--remote-debugging-port=' + debugPort,
+		'--disable-extension=GitHub.copilot-chat', '--disable-extension=vscode.vscode-api-tests', '--new-window', '--skip-welcome', '--skip-release-notes',
+		'--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding', '--disable-features=CalculateNativeWinOcclusion'];
+	if (trust === 'trusted') { args.push('--disable-workspace-trust'); }
+	args.push(workspace);
+	console.log('[' + trust + '] starting the editor (a window will open)');
+	const child = cp.spawn(exe, args, { cwd: VSCODE_DIR, env: Object.assign({}, process.env, { NODE_ENV: 'development', VSCODE_DEV: '1', VSCODE_CLI: '1' }), stdio: 'ignore' });
+
+	const seen = { trust, restrictedMode: null, pill: null, proseSize: null, cards: 0, error: null };
+	let wb = null, wv = null;
+	try {
+		const page = await until('the editor window', async () => (await targets(debugPort)).find((t) => t.type === 'page' && /workbench/.test(t.url)), 120000);
+		wb = await connect(page.webSocketDebuggerUrl);
+		await wb.send('Runtime.enable'); await wb.send('Page.enable');
+		await until('the workbench', () => evalIn(wb, '!!document.querySelector(".monaco-workbench .part.editor")'), 60000);
+
+		const findChat = async () => {
+			for (const t of (await targets(debugPort)).filter((x) => /^vscode-webview:/.test(x.url))) {
+				let c; try { c = await connect(t.webSocketDebuggerUrl); } catch (e) { continue; }
+				try {
+					await c.send('Runtime.enable');
+					if (await evalIn(c, '(() => { const f = document.getElementById("active-frame"); const d = f && f.contentDocument; return !!(d && d.getElementById("input") && d.getElementById("log")); })()')) { return c; }
+				} catch (e) { /* not ready yet */ }
+				c.close();
+			}
+			return null;
+		};
+		let asked = 0;
+		wv = await until('the chat', async () => {
+			const c = await findChat(); if (c) { return c; }
+			if (asked++ % 10 === 9) { for (const type of ['keyDown', 'keyUp']) { await wb.send('Input.dispatchKeyEvent', { type, modifiers: process.platform === 'darwin' ? 6 : 3, key: 'i', code: 'KeyI', windowsVirtualKeyCode: 73 }); } }
+			return null;
+		}, 90000);
+		const D = 'document.getElementById("active-frame").contentDocument';
+
+		// the host answers 'ready' with the approvals state and the config; give it a moment to land
+		await until('the chat config', () => evalIn(wv, `(() => ${D}.body.style.getPropertyValue('--prose-size') !== '' || /stand-in/.test(${D}.getElementById('model').textContent))()`), 20000);
+		await sleep(1500);
+		seen.restrictedMode = await evalIn(wb, '/Restricted Mode/.test(document.querySelector(".part.statusbar") ? document.querySelector(".part.statusbar").innerText + " " + Array.from(document.querySelectorAll(".part.statusbar [aria-label]")).map((e) => e.getAttribute("aria-label")).join(" ") : "")');
+		Object.assign(seen, await evalIn(wv, `(() => { const d = ${D}; return { pill: d.getElementById('approvalsLabel').textContent.trim(), proseSize: d.body.style.getPropertyValue('--prose-size') }; })()`));
+
+		await evalIn(wv, `(() => { const d = ${D}; const i = d.getElementById('input'); i.focus(); i.value = 'Write some notes.'; i.dispatchEvent(new Event('input', { bubbles: true })); d.getElementById('send').click(); return true; })()`);
+		// a new profile has no key: the editor asks for one, and gets a stand-in (kept in the temp profile only)
+		const first = await until('the key prompt, or the first request', async () => {
+			if (requests.some((r) => r.stream)) { return 'request'; }
+			return (await evalIn(wb, '(() => { const q = document.querySelector(".quick-input-widget"); return !!(q && q.style.display !== "none" && q.querySelector("input")); })()')) ? 'prompt' : null;
+		}, 30000);
+		if (first === 'prompt') {
+			const box = '(() => { const q = document.querySelector(".quick-input-widget"); const i = q && q.style.display !== "none" ? q.querySelector("input") : null; if (!i) { return null; } i.focus(); return { typed: i.value.length }; })()';
+			await until('the key to be accepted', async () => {
+				const state = await evalIn(wb, box);
+				if (!state) { return true; }
+				if (!state.typed) { await wb.send('Input.insertText', { text: KEY }); await sleep(200); return false; }
+				for (const type of ['keyDown', 'keyUp']) { await wb.send('Input.dispatchKeyEvent', { type, key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 36, text: type === 'keyDown' ? '\r' : undefined }); }
+				await sleep(500);
+				return false;
+			}, 30000);
+		}
+
+		// Drive the turn to its end: skip every approval card (and count them), until the edit has been verified.
+		await until('the turn to finish', async () => {
+			const s = await evalIn(wv, `(() => {
+				const d = ${D};
+				const ask = d.querySelector('.tl-ask.asking .askbtns .skip');
+				if (ask) { ask.click(); return { skipped: true }; }
+				const v = Array.from(d.querySelectorAll('.verifyrun .termstatus')).pop();
+				return { verified: !!v && !/running/.test(v.className) };
+			})()`);
+			if (s.skipped) { seen.cards++; await sleep(300); return false; }
+			return s.verified;
+		}, 60000);
+		await sleep(800);
+	} catch (e) {
+		seen.error = String(e && e.message || e);
+		try { if (wb) { const s = await wb.send('Page.captureScreenshot', { format: 'png' }); fs.writeFileSync(path.join(OUT, trust + '-failed.png'), Buffer.from(s.data, 'base64')); } } catch (e2) { /* no picture */ }
+	} finally {
+		try { if (wb) { const s = await wb.send('Page.captureScreenshot', { format: 'png' }); fs.writeFileSync(path.join(OUT, trust + '.png'), Buffer.from(s.data, 'base64')); } } catch (e) { /* no picture */ }
+		try { if (wv) { wv.close(); } if (wb) { wb.close(); } } catch (e) { /* closed */ }
+		seen.files = ['.autopilot-ran', '.verify-ran', 'notes.txt'].filter((f) => fs.existsSync(path.join(workspace, f)));
+		seen.requests = requests.filter((r) => r.method === 'POST');
+		if (KEEP) { console.log('--keep: the editor is left open. Its profile and folder are in ' + tmp); }
+		else {
+			try { child.kill('SIGTERM'); } catch (e) { /* gone */ }
+			await sleep(1500);
+			try { child.kill('SIGKILL'); } catch (e) { /* gone */ }
+			try { fs.rmSync(tmp, { recursive: true, force: true }); } catch (e) { /* a temp folder */ }
+		}
+		userProvider.close(); folderProvider.close();
+	}
+	return seen;
+}
+
+async function main() {
+	if (typeof WebSocket !== 'function' || typeof fetch !== 'function') { console.error('Needs Node 22 or newer (for its built-in WebSocket).'); return 2; }
+	let product;
+	try { product = JSON.parse(fs.readFileSync(path.join(VSCODE_DIR, 'product.json'), 'utf8')); }
+	catch (e) { console.error('No editor build at ' + VSCODE_DIR + '. Run ./scripts/run-dev.sh once in the checkout that has vscode/, then pass that folder with --vscode.'); return 2; }
+	const exe = process.platform === 'darwin'
+		? path.join(VSCODE_DIR, '.build', 'electron', product.nameLong + '.app', 'Contents', 'MacOS', product.nameShort)
+		: path.join(VSCODE_DIR, '.build', 'electron', product.applicationName);
+	if (!fs.existsSync(exe)) { console.error('The editor has not been built yet (' + exe + ' is missing). Run ./scripts/run-dev.sh once.'); return 2; }
+	fs.mkdirSync(OUT, { recursive: true });
+
+	const runs = [];
+	for (const trust of ['trusted', 'restricted']) { if (!ONLY || ONLY === trust) { runs.push(await runOnce(exe, trust)); } }
+
+	let failed = 0, broken = 0;
+	for (const s of runs) {
+		const streamed = s.requests.filter((r) => r.stream);
+		const checks = [
+			[s.trust === 'restricted' ? 'the folder is open in Restricted Mode' : 'the folder is open trusted (Workspace Trust off)', s.trust === 'restricted' ? s.restrictedMode === true : s.restrictedMode === false, String(s.restrictedMode)],
+			['the turn ran to the end', !s.error && s.files.includes('notes.txt'), s.error || JSON.stringify(s.files)],
+			['positive control: a display preference set by the folder applies (chat.fontSize 17)', s.proseSize === '17px', JSON.stringify(s.proseSize)],
+			['the approvals pill shows the user\'s choice, not the folder\'s (Default Approvals)', s.pill === 'Default Approvals', JSON.stringify(s.pill)],
+			['the chat goes to the user\'s provider, never the folder\'s', streamed.length > 0 && streamed.every((r) => r.provider === 'user'), JSON.stringify(streamed.map((r) => r.provider))],
+			['the user\'s key is never sent to the folder\'s server', !s.requests.some((r) => r.provider === 'folder' && r.sentTheKey), JSON.stringify(s.requests.filter((r) => r.provider === 'folder').map((r) => r.sentTheKey))],
+			['the agent\'s command asked first (an approval card), and did not run when skipped', s.cards >= 1 && !s.files.includes('.autopilot-ran'), JSON.stringify({ cards: s.cards, files: s.files })],
+			['the folder\'s verify command did not run', !s.files.includes('.verify-ran'), JSON.stringify(s.files)],
+		];
+		console.log('\n[' + s.trust + ']');
+		for (const [name, ok, detail] of checks) {
+			console.log('  ' + (ok ? 'ok  ' : 'FAIL') + ' ' + name + (ok ? '' : ' — ' + detail));
+			if (!ok) { failed++; }
+		}
+		if (s.error) { broken++; }
+	}
+	fs.writeFileSync(path.join(OUT, 'report.json'), JSON.stringify(runs, null, 2));
+	console.log('\n' + (failed ? failed + ' checks FAILED.' : 'All checks passed.') + ' Screenshots and report: ' + OUT);
+	return broken === runs.length ? 2 : (failed ? 1 : 0);
+}
+
+main().then((c) => process.exit(c), (e) => { console.error(e); process.exit(2); });