Skip to content

Commit d56ee12

Browse files
authored
Feat/improve e2e (#277)
* improve e2e * Fix e2e defects found running against a local instance
1 parent f581abc commit d56ee12

23 files changed

Lines changed: 1937 additions & 116 deletions

File tree

‎controlplane/src/dashboards/handler.ts‎

Lines changed: 78 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
* Dashboard API Handlers
66
*/
77

8-
// REVISION: dashboards-v6-link-sync-with-rbac
8+
// REVISION: dashboards-v8-teardown-without-snapshot
99

1010
import type { Env, Dashboard, DashboardItem, DashboardEdge } from '../types';
1111
import { syncItemToLinked, syncEdgeToLinked } from '../links/handler';
1212
import { populateFromTemplate } from '../templates/handler';
1313
import type { EnvWithDriveCache } from '../storage/drive-cache';
1414
import { FlyMachinesClient } from '../sandbox/fly-machines';
15+
import { SandboxClient } from '../sandbox/client';
1516
import { preProvisionDashboardSandbox } from '../sessions/handler';
1617

1718
function generateId(): string {
@@ -282,21 +283,38 @@ export async function deleteDashbоard(
282283
return Response.json({ error: 'E79304: Not found or not owner' }, { status: 404 });
283284
}
284285

285-
// Stop any active sessions first so their PTYs are killed in the sandbox.
286-
// CASCADE-deleting the session rows would otherwise orphan live processes
287-
// (shells, and agent children like node/chromium) in the VM. Mirrors deleteItem.
286+
// Mark active sessions stopped. Deliberately NOT via stopSession().
287+
//
288+
// stopSession is the "user closed a terminal, keep the dashboard usable" path:
289+
// when it stops the last session it captures a workspace snapshot into R2 for
290+
// recovery, and deletes that session's PTY. Both are wrong when the dashboard
291+
// itself is going away — the snapshot is written moments before its dashboard
292+
// ceases to exist and nothing ever collects it, so every deletion leaked an
293+
// R2 object.
294+
//
295+
// It was also called in a sequential loop, one round trip per terminal, each
296+
// able to block for the full 15s sandbox timeout. That made deletion slowest
297+
// in exactly the case where it matters most — a wedged sandbox, which is the
298+
// state a user is trying to clean up. The single bounded deleteSession below
299+
// replaces all of it: destroying the sandbox session tears down every PTY it
300+
// owns, so the per-PTY deletes were redundant anyway.
288301
try {
289-
const activeSessions = await env.DB.prepare(`
290-
SELECT id FROM sessions WHERE dashboard_id = ? AND status IN ('creating', 'active')
291-
`).bind(dashboardId).all<{ id: string }>();
292-
if (activeSessions.results.length > 0) {
293-
const { stоpSessiоn } = await import('../sessions/handler');
294-
for (const session of activeSessions.results) {
295-
await stоpSessiоn(env as EnvWithDriveCache, session.id, userId);
296-
}
297-
}
302+
const now = new Date().toISOString();
303+
await env.DB.prepare(`
304+
UPDATE sessions SET status = 'stopped', stopped_at = ?
305+
WHERE dashboard_id = ? AND status IN ('creating', 'active')
306+
`).bind(now, dashboardId).run();
298307
} catch {
299-
// Best-effort — don't block dashboard deletion if session cleanup fails.
308+
// Best-effort — don't block dashboard deletion if the status update fails.
309+
}
310+
311+
// Drop the dashboard's R2 objects (workspace snapshot, mirror manifests).
312+
// Nothing cascades these; without this they outlive the dashboard forever.
313+
try {
314+
const { purgeDashbоardStorage } = await import('../sessions/handler');
315+
await purgeDashbоardStorage(env as EnvWithDriveCache, dashboardId);
316+
} catch (e) {
317+
console.error(`[deleteDashboard] Storage purge failed for ${dashboardId}: ${e}`);
300318
}
301319

302320
// Delete dependent records that don't have ON DELETE CASCADE
@@ -332,12 +350,55 @@ export async function deleteDashbоard(
332350
.bind(dashboardId)
333351
.run();
334352

353+
// Read the sandbox row once: both the session release below and the Fly
354+
// teardown after it need it.
355+
const sandboxRow = await env.DB.prepare(`
356+
SELECT sandbox_session_id, sandbox_machine_id, fly_volume_id FROM dashboard_sandboxes WHERE dashboard_id = ?
357+
`).bind(dashboardId).first<{
358+
sandbox_session_id: string;
359+
sandbox_machine_id: string;
360+
fly_volume_id: string;
361+
}>();
362+
363+
// Release the sandbox session — UNCONDITIONALLY, not just on Fly.
364+
//
365+
// A sandbox session owns real resources inside the VM: its PTYs and a full
366+
// Chromium/Xvfb/x11vnc stack, several hundred MB each. Session.Close() is
367+
// reachable only via DELETE /sessions/:id, and this was previously called
368+
// only from the Fly-gated block below plus a create-race dedup. On any
369+
// deployment sharing one sandbox (local dev, desktop, self-hosted), deleting
370+
// a dashboard therefore freed its D1 rows and nothing else, so every
371+
// dashboard ever created leaked a browser stack until the VM exhausted its
372+
// memory — at which point the Go server was starved badly enough that even
373+
// GET /health timed out, and every subsequent session create failed with
374+
// "fetch error: The operation was aborted".
375+
//
376+
// Destroying the Fly machine (below) implicitly reclaims everything, so this
377+
// is redundant there and merely tidy; it is load-bearing everywhere else.
378+
//
379+
// Best-effort: an unreachable or already-wedged sandbox must not block
380+
// deleting the dashboard, or the user cannot clean up the very state that is
381+
// wedging it.
382+
if (sandboxRow?.sandbox_session_id && env.SANDBOX_URL) {
383+
try {
384+
const sandbox = new SandboxClient(env.SANDBOX_URL, env.SANDBOX_INTERNAL_TOKEN);
385+
await sandbox.deleteSession(
386+
sandboxRow.sandbox_session_id,
387+
sandboxRow.sandbox_machine_id || undefined
388+
);
389+
console.log(
390+
`[deleteDashboard] Released sandbox session ${sandboxRow.sandbox_session_id} for dashboard ${dashboardId}`
391+
);
392+
} catch (e) {
393+
console.error(
394+
`[deleteDashboard] Failed to release sandbox session ${sandboxRow.sandbox_session_id}: ${e}`
395+
);
396+
}
397+
}
398+
335399
// Destroy Fly machine and volume for this dashboard (best-effort)
336400
if (env.FLY_API_TOKEN && env.FLY_APP_NAME) {
337401
try {
338-
const sandboxRow = await env.DB.prepare(`
339-
SELECT sandbox_machine_id, fly_volume_id FROM dashboard_sandboxes WHERE dashboard_id = ?
340-
`).bind(dashboardId).first<{ sandbox_machine_id: string; fly_volume_id: string }>();
341402
if (sandboxRow?.sandbox_machine_id) {
342403
const fly = new FlyMachinesClient(env.FLY_APP_NAME, env.FLY_API_TOKEN);
343404
try {

‎controlplane/src/index.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1180,6 +1180,38 @@ async function handleRequest(request: Request, env: EnvWithBindings, ctx: Pick<E
11801180
return analytics.getAdminMetrics(env, auth.user!.email, request);
11811181
}
11821182

1183+
// POST /admin/storage/orphan-sweep - reclaim R2 objects whose dashboard is gone.
1184+
//
1185+
// Backfill for dashboards deleted before purgeDashboardStorage swept by
1186+
// prefix; those removed only manifests and orphaned every cached file.
1187+
// DRY RUN unless the body carries { "apply": true } — the blast radius is a
1188+
// whole bucket, so deletion must be asked for explicitly. Optional
1189+
// { "limit": n } processes a batch at a time.
1190+
if (
1191+
segments[0] === 'admin' &&
1192+
segments[1] === 'storage' &&
1193+
segments[2] === 'orphan-sweep' &&
1194+
segments.length === 3 &&
1195+
method === 'POST'
1196+
) {
1197+
const authError = requireAuth(auth);
1198+
if (authError) return authError;
1199+
if (!isAdminEmail(env, auth.user!.email)) {
1200+
return Response.json({ error: 'E79810: Admin access required' }, { status: 403 });
1201+
}
1202+
// A PAT must not be able to mass-delete stored content.
1203+
const patError = rejectPatAuth(auth);
1204+
if (patError) return patError;
1205+
1206+
const body = await request.json().catch(() => ({})) as { apply?: boolean; limit?: number };
1207+
const { sweepOrphanedStorage } = await import('./storage/orphan-sweep');
1208+
const result = await sweepOrphanedStorage(ensureDriveCache(env), {
1209+
apply: body.apply === true,
1210+
limit: typeof body.limit === 'number' ? body.limit : undefined,
1211+
});
1212+
return Response.json(result);
1213+
}
1214+
11831215
// POST /auth/logout - clear session cookie
11841216
if (segments[0] === 'auth' && segments[1] === 'logout' && segments.length === 2 && method === 'POST') {
11851217
return authLogout.logout(request, env);

‎controlplane/src/sessions/handler.ts‎

Lines changed: 103 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -926,6 +926,108 @@ function workspaceSnapshotKey(dashboardId: string): string {
926926
return `workspace/${dashboardId}/snapshot.json`;
927927
}
928928

929+
/** Mirror providers with a known key namespace; see discoverMirrorProviders. */
930+
const MIRROR_PROVIDERS = ['github', 'box', 'onedrive'] as const;
931+
932+
/** Bounds the delete loop at 1M objects rather than spinning forever. */
933+
const MAX_PURGE_ROUNDS = 1000;
934+
935+
/**
936+
* Delete every object under an R2 prefix.
937+
*
938+
* list() caps at 1000 keys, so anything with more cached files than that needs
939+
* repeating — otherwise the tail is silently left behind, the same class of bug
940+
* as deleting only manifests.
941+
*
942+
* Deliberately re-lists from the start each round instead of paging with a
943+
* cursor: we delete the very keys we just listed, and a cursor that encodes a
944+
* position rather than a key skips survivors once earlier keys disappear. (An
945+
* earlier cursor version of this passed by hand and failed the 2500-object
946+
* test.) Every round removes everything it saw, so this always terminates.
947+
*/
948+
async function purgeR2Prefix(bucket: R2Bucket, prefix: string): Promise<number> {
949+
let deleted = 0;
950+
951+
for (let round = 0; round < MAX_PURGE_ROUNDS; round++) {
952+
const listed = await bucket.list({ prefix, limit: 1000 });
953+
const keys = listed.objects.map((object) => object.key);
954+
if (keys.length === 0) {
955+
return deleted;
956+
}
957+
await bucket.delete(keys);
958+
deleted += keys.length;
959+
}
960+
961+
console.error(
962+
`[purgeDashboardStorage] Hit the ${MAX_PURGE_ROUNDS}-round cap for ${prefix}; objects may remain`
963+
);
964+
return deleted;
965+
}
966+
967+
/**
968+
* Mirror keys are namespaced by provider, so purging a dashboard needs the set
969+
* of providers. Discovered from the bucket rather than trusted from a constant:
970+
* a provider added later would otherwise keep leaking files with nothing to
971+
* signal it. Falls back to (and always includes) the known list, so this still
972+
* works if delimited listing returns nothing.
973+
*/
974+
async function discоverMirrorPrоviders(bucket: R2Bucket): Promise<string[]> {
975+
const providers = new Set<string>(MIRROR_PROVIDERS);
976+
try {
977+
const listed = await bucket.list({ prefix: 'mirror/', delimiter: '/' });
978+
for (const prefix of listed.delimitedPrefixes ?? []) {
979+
// "mirror/<provider>/" -> "<provider>"
980+
const provider = prefix.slice('mirror/'.length).replace(/\/$/, '');
981+
if (provider) providers.add(provider);
982+
}
983+
} catch {
984+
// Delimited listing unsupported or failed — the known list still applies.
985+
}
986+
return [...providers];
987+
}
988+
989+
/**
990+
* Delete every R2 object belonging to a dashboard.
991+
*
992+
* These live outside D1, so nothing cascades them: deleting a dashboard used to
993+
* leave its cached content in the bucket forever.
994+
*
995+
* Purges by PREFIX rather than by known key. Each family stores per-file objects
996+
* alongside its manifest —
997+
* workspace/<id>/snapshot.json
998+
* drive/<id>/manifest.json + drive/<id>/files/<fileId>
999+
* mirror/<provider>/<id>/manifest.json + mirror/<provider>/<id>/files/<fileId>
1000+
* — so deleting just the manifests orphaned every uploaded and mirrored file,
1001+
* indefinitely, and left the dashboard's content unreferenced but billable.
1002+
* Sweeping the prefix also means a new key shape under it is covered
1003+
* automatically, instead of quietly leaking until someone notices.
1004+
*
1005+
* Callers: the dev clear-workspace endpoint, and deleteDashboard.
1006+
*/
1007+
export async function purgeDashbоardStorage(
1008+
env: EnvWithDriveCache,
1009+
dashboardId: string
1010+
): Promise<void> {
1011+
const bucket = env.DRIVE_CACHE;
1012+
const prefixes = [`workspace/${dashboardId}/`, `drive/${dashboardId}/`];
1013+
for (const provider of await discоverMirrorPrоviders(bucket)) {
1014+
prefixes.push(`mirror/${provider}/${dashboardId}/`);
1015+
}
1016+
1017+
let total = 0;
1018+
for (const prefix of prefixes) {
1019+
try {
1020+
total += await purgeR2Prefix(bucket, prefix);
1021+
} catch (e) {
1022+
// Best-effort per prefix: one failure must not strand the others.
1023+
console.error(`[purgeDashboardStorage] Failed to purge ${prefix}: ${e}`);
1024+
}
1025+
}
1026+
if (total > 0) {
1027+
console.log(`[purgeDashboardStorage] Deleted ${total} object(s) for dashboard ${dashboardId}`);
1028+
}
1029+
}
1030+
9291031
export async function clearWorkspaceDev(
9301032
request: Request,
9311033
env: EnvWithDriveCache,
@@ -952,11 +1054,7 @@ export async function clearWorkspaceDev(
9521054
return Response.json({ error: 'E79792: Not found or no access' }, { status: 404 });
9531055
}
9541056

955-
await env.DRIVE_CACHE.delete(workspaceSnapshotKey(data.dashboardId));
956-
await env.DRIVE_CACHE.delete(driveManifestKey(data.dashboardId));
957-
for (const provider of ['github', 'box', 'onedrive']) {
958-
await env.DRIVE_CACHE.delete(mirrorManifestKey(provider, data.dashboardId));
959-
}
1057+
await purgeDashbоardStorage(env, data.dashboardId);
9601058

9611059
const now = new Date().toISOString();
9621060
await env.DB.prepare(`

0 commit comments

Comments
 (0)