fix: reconcile stale backup runs

Author: sametbasbug

scripts/orbit-slice1-test-worker.ts

@@ -489,6 +489,16 @@ async function testRoute(request: Request, env: TestEnv): Promise<Response | nul
     const bucket = new MemoryR2();
     const testKey = btoa(String.fromCharCode(...new Uint8Array(32).fill(7)))
       .replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/u, '');
+    await env.DB.batch([
+      env.DB.prepare(`
+        INSERT INTO backup_runs (id, backup_kind, status, started_at)
+        VALUES ('slice5-stale-backup', 'daily', 'running', ?)
+      `).bind(now - 31 * 60 * 1000),
+      env.DB.prepare(`
+        INSERT INTO backup_runs (id, backup_kind, status, started_at)
+        VALUES ('slice5-fresh-backup', 'daily', 'running', ?)
+      `).bind(now - 60 * 1000),
+    ]);
     const result = await runR2Backup({
       ...env,
       BACKUPS: bucket,
@@ -503,10 +513,17 @@ async function testRoute(request: Request, env: TestEnv): Promise<Response | nul
       SELECT status, object_key, manifest_checksum, error_code
       FROM backup_runs WHERE id = ?
     `).bind(result.runId).first();
+    const reconciled = await env.DB.prepare(`
+      SELECT id, status, error_code, completed_at
+      FROM backup_runs
+      WHERE id IN ('slice5-stale-backup', 'slice5-fresh-backup')
+      ORDER BY id
+    `).all();
     return Response.json({
       objectCount: bucket.objects.size,
       retention,
       run: runs,
+      reconciled: reconciled.results,
       objectKeyIsSafe: !result.objectKey.includes('nyx') && !result.objectKey.includes('samet'),
       checksumLength: result.objectChecksum.length,
     });

scripts/orbit-slice5-tests.ts

@@ -903,6 +903,12 @@ describe('Orbit V6 Slice 5 dashboard and platform core', { concurrency: false },
         manifest_checksum: string;
         error_code: string | null;
       };
+      reconciled: Array<{
+        id: string;
+        status: string;
+        error_code: string | null;
+        completed_at: number | null;
+      }>;
       objectKeyIsSafe: boolean;
       checksumLength: number;
     };
@@ -914,6 +920,20 @@ describe('Orbit V6 Slice 5 dashboard and platform core', { concurrency: false },
     assert.ok(proof.run.object_key.startsWith('orbit-v6/daily/'));
     assert.ok(proof.run.manifest_checksum.length >= 40);
     assert.ok(proof.checksumLength >= 40);
+    assert.deepEqual(proof.reconciled, [
+      {
+        id: 'slice5-fresh-backup',
+        status: 'running',
+        error_code: null,
+        completed_at: null,
+      },
+      {
+        id: 'slice5-stale-backup',
+        status: 'failed',
+        error_code: 'backup_run_stale_timeout',
+        completed_at: NOW,
+      },
+    ]);
   });
 
   test('backup failures are owner-visible without exposing encryption material', async () => {

src/server/backup/r2-backup.ts

@@ -18,6 +18,8 @@ const RETENTION: Record<Exclude<BackupKind, 'manual'>, number> = {
   weekly: 8,
   monthly: 6,
 };
+const STALE_BACKUP_RUN_MS = 30 * 60 * 1000;
+const STALE_BACKUP_ERROR_CODE = 'backup_run_stale_timeout';
 
 function requireBackupBindings(env: OrbitBindings): { bucket: R2BucketLike; encryptionKey: string } {
   if (env.ORBIT_BACKUP_ENABLED !== 'true') {
@@ -68,6 +70,18 @@ export async function runR2Backup(
   actorAccountId: string | null = null,
 ): Promise<{ runId: string; objectKey: string; manifestChecksum: string; objectChecksum: string }> {
   const repository = new D1PlatformRepository(env.DB);
+  const reconciledRuns = await repository.failStaleBackupRuns({
+    before: now - STALE_BACKUP_RUN_MS,
+    errorCode: STALE_BACKUP_ERROR_CODE,
+    now,
+  });
+  if (reconciledRuns > 0) {
+    console.log(JSON.stringify({
+      event: 'backup.reconciliation',
+      status: 'failed_stale_runs',
+      count: reconciledRuns,
+    }));
+  }
   const runId = createEntityId();
   const startedAt = Date.now();
   await repository.startBackupRun({ id: runId, kind, actorAccountId, now });

src/server/repositories/d1/d1-platform-repository.ts

@@ -199,6 +199,16 @@ export class D1PlatformRepository implements PlatformRepository {
     `).bind(input.errorCode.slice(0, 160), input.now, input.id).run();
   }
 
+  async failStaleBackupRuns(
+    input: Parameters<PlatformRepository['failStaleBackupRuns']>[0],
+  ): Promise<number> {
+    const result = await this.#db.prepare(`
+      UPDATE backup_runs SET status = 'failed', error_code = ?, completed_at = ?
+      WHERE status = 'running' AND started_at <= ?
+    `).bind(input.errorCode.slice(0, 160), input.now, input.before).run();
+    return result.meta?.changes ?? 0;
+  }
+
   async listBackupRuns(limit: number): Promise<BackupRunView[]> {
     const result = await this.#db.prepare(`
       SELECT id, backup_kind, status, object_key, manifest_checksum, schema_version,

src/server/repositories/platform-repository.ts

@@ -50,5 +50,6 @@ export interface PlatformRepository {
   startBackupRun(input: { id: string; kind: BackupRunView['backupKind']; actorAccountId: string | null; now: number }): Promise<void>;
   finishBackupRun(input: { id: string; objectKey: string; manifestChecksum: string; schemaVersion: number; counts: Record<string, number>; now: number }): Promise<void>;
   failBackupRun(input: { id: string; errorCode: string; now: number }): Promise<void>;
+  failStaleBackupRuns(input: { before: number; errorCode: string; now: number }): Promise<number>;
   listBackupRuns(limit: number): Promise<BackupRunView[]>;
 }