Skip to content

Commit dce739b

Browse files
authored
Fix scheduled backup recovery (#43)
## Summary - retain expired idempotency rows that immutable media transform claims still reference - isolate scheduled maintenance tasks so one failure cannot terminate backup work - add a 04:00 UTC reconciliation trigger for backup runs stale beyond 30 minutes ## Verification - 168 D1 tests passed - Astro diagnostics: 0 errors, 0 warnings, 0 hints - production config: 54 assertions passed - Actions scope tests passed - production Worker build and dry-run passed
1 parent 0cb45b7 commit dce739b

11 files changed

Lines changed: 203 additions & 21 deletions

‎scripts/orbit-production-config-tests.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ const expectedResources = {
124124
},
125125
},
126126
triggers: {
127-
crons: ['17 3 * * *'],
127+
crons: ['17 3 * * *', '0 4 * * *'],
128128
},
129129
};
130130

‎scripts/orbit-slice1-test-worker.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,56 @@ async function testRoute(request: Request, env: TestEnv): Promise<Response | nul
657657
return Response.json({ ok: true });
658658
}
659659

660+
if (url.pathname === '/__test/seed-referenced-idempotency') {
661+
try {
662+
const id = String(body.id);
663+
const accountId = '019f64d2-0109-7644-9a4e-a0d25df888e2';
664+
const monthUtc = new Date(now).toISOString().slice(0, 7);
665+
const usageDay = new Date(now).toISOString().slice(0, 10);
666+
await env.DB.batch([
667+
env.DB.prepare(`
668+
INSERT INTO idempotency_keys (
669+
id, principal_type, principal_id, key_digest, operation,
670+
request_digest, response_status, resource_type, resource_id,
671+
created_at, expires_at, response_json, state, completed_at
672+
) VALUES (?, 'account', ?, ?, 'POST /v1/account/avatar', ?, 0,
673+
'media_upload', NULL, ?, ?, '{}', 'in_progress', NULL)
674+
`).bind(id, accountId, id, id, now - 1000, now - 1),
675+
env.DB.prepare(`
676+
INSERT INTO media_transform_usage_monthly (
677+
month_utc, attempted_count, succeeded_count, failed_count, updated_at
678+
) VALUES (?, 0, 0, 0, ?)
679+
ON CONFLICT(month_utc) DO NOTHING
680+
`).bind(monthUtc, now),
681+
env.DB.prepare(`
682+
INSERT INTO media_transform_claims (
683+
id, month_utc, profile, actor_type, actor_id,
684+
source_content_type, source_byte_size, status, created_at,
685+
usage_day, target_type, target_id, idempotency_id
686+
) VALUES (?, ?, 'avatar', 'account', ?, 'image/png', 128,
687+
'reserved', ?, ?, 'account', ?, NULL)
688+
`).bind(`${id}-claim`, monthUtc, accountId, now, usageDay, accountId),
689+
]);
690+
await env.DB.prepare(`
691+
UPDATE media_transform_claims SET idempotency_id = ? WHERE id = ?
692+
`).bind(id, `${id}-claim`).run();
693+
await env.DB.prepare(`
694+
UPDATE idempotency_keys
695+
SET state = 'completed', response_status = 201,
696+
response_json = '{"ok":true}', completed_at = ?
697+
WHERE id = ?
698+
`).bind(now, id).run();
699+
const claim = await env.DB.prepare(`
700+
SELECT idempotency_id FROM media_transform_claims WHERE id = ?
701+
`).bind(`${id}-claim`).first<{ idempotency_id: string | null }>();
702+
return Response.json({ ok: true, idempotencyId: claim?.idempotency_id ?? null });
703+
} catch (error) {
704+
return Response.json({
705+
error: error instanceof Error ? error.message : 'seed_referenced_idempotency_failed',
706+
}, { status: 500 });
707+
}
708+
}
709+
660710
return Response.json({ error: 'not_found' }, { status: 404 });
661711
}
662712

‎scripts/orbit-slice1-tests.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1556,9 +1556,15 @@ let firstCredentialToken = '';
15561556
assert.equal(expired.status, 401);
15571557
});
15581558

1559-
test('daily cleanup removes retained OAuth/session rows but leaves audit evidence', async () => {
1559+
test('daily cleanup retains referenced media idempotency while removing other expired rows', async () => {
15601560
const cleanupAt = NOW + 62 * 24 * 60 * 60 * 1000;
15611561
await postJson('/__test/seed-idempotency', { id: 'cleanup-key' }, {}, cleanupAt);
1562+
const referencedSeed = await postJson('/__test/seed-referenced-idempotency', {
1563+
id: 'cleanup-referenced-key',
1564+
}, {}, cleanupAt);
1565+
assert.equal(referencedSeed.status, 200, await referencedSeed.clone().text());
1566+
assert.equal((await referencedSeed.json() as { idempotencyId: string }).idempotencyId,
1567+
'cleanup-referenced-key');
15621568
const beforeResponse = await postJson('/__test/state', {}, {}, cleanupAt);
15631569
const before = await beforeResponse.json() as {
15641570
counts: { audit_events: number; idempotency_keys: number };
@@ -1573,12 +1579,12 @@ let firstCredentialToken = '';
15731579
assert.ok(body.oauthFlows > 0);
15741580
assert.ok(body.sessions > 0);
15751581
assert.ok(before.counts.idempotency_keys > 0);
1576-
assert.equal(body.idempotencyKeys, before.counts.idempotency_keys);
1582+
assert.equal(body.idempotencyKeys, before.counts.idempotency_keys - 1);
15771583
const afterResponse = await postJson('/__test/state', {}, {}, cleanupAt);
15781584
const after = await afterResponse.json() as {
15791585
counts: { audit_events: number; idempotency_keys: number };
15801586
};
1581-
assert.equal(after.counts.idempotency_keys, 0);
1587+
assert.equal(after.counts.idempotency_keys, 1);
15821588
assert.equal(after.counts.audit_events, before.counts.audit_events);
15831589
});
15841590
});

‎scripts/orbit-slice1-worker-tests.ts‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import assert from 'node:assert/strict';
22
import { describe, test } from 'node:test';
3-
import worker from '../src/worker';
3+
import worker, {
4+
BACKUP_CRON,
5+
BACKUP_RECONCILIATION_CRON,
6+
runScheduledMaintenance,
7+
} from '../src/worker';
48
import {
59
assertDeploymentBindings,
610
assertIdentityBindings,
@@ -331,4 +335,54 @@ describe('Orbit V6 deployment-mode contract', () => {
331335
assert.equal(response.headers.get('x-robots-tag'), null);
332336
}
333337
});
338+
339+
test('waits for independent scheduled tasks before reporting one task failure', async () => {
340+
const calls: string[] = [];
341+
const scheduledTime = 1_785_727_021_000;
342+
await assert.rejects(
343+
runScheduledMaintenance(
344+
{ cron: BACKUP_CRON, scheduledTime },
345+
productionBindings('live'),
346+
{
347+
async runIdentityCleanup(_env, now) {
348+
assert.equal(now, scheduledTime);
349+
calls.push('identity_cleanup');
350+
throw new Error('simulated_cleanup_failure');
351+
},
352+
async runScheduledBackups(_env, now) {
353+
assert.equal(now, scheduledTime);
354+
calls.push('backup');
355+
},
356+
async cleanupMedia(_env, now) {
357+
assert.equal(now, scheduledTime);
358+
calls.push('media_cleanup');
359+
},
360+
async reconcileStaleBackupRuns() {
361+
calls.push('reconciliation');
362+
},
363+
},
364+
),
365+
/scheduled_maintenance_failed:identity_cleanup/u,
366+
);
367+
assert.deepEqual(calls.sort(), ['backup', 'identity_cleanup', 'media_cleanup']);
368+
});
369+
370+
test('routes the follow-up cron only to stale backup reconciliation', async () => {
371+
const calls: string[] = [];
372+
const scheduledTime = 1_785_729_600_000;
373+
await runScheduledMaintenance(
374+
{ cron: BACKUP_RECONCILIATION_CRON, scheduledTime },
375+
productionBindings('live'),
376+
{
377+
async runIdentityCleanup() { calls.push('identity_cleanup'); },
378+
async runScheduledBackups() { calls.push('backup'); },
379+
async cleanupMedia() { calls.push('media_cleanup'); },
380+
async reconcileStaleBackupRuns(_env, now) {
381+
assert.equal(now, scheduledTime);
382+
calls.push('reconciliation');
383+
},
384+
},
385+
);
386+
assert.deepEqual(calls, ['reconciliation']);
387+
});
334388
});

‎src/server/backup/r2-backup.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,10 @@ export async function enforceBackupRetention(bucket: R2BucketLike): Promise<Reco
6363
return deleted;
6464
}
6565

66-
export async function runR2Backup(
66+
export async function reconcileStaleBackupRuns(
6767
env: OrbitBindings,
68-
kind: BackupKind,
6968
now = Date.now(),
70-
actorAccountId: string | null = null,
71-
): Promise<{ runId: string; objectKey: string; manifestChecksum: string; objectChecksum: string }> {
69+
): Promise<number> {
7270
const repository = new D1PlatformRepository(env.DB);
7371
const reconciledRuns = await repository.failStaleBackupRuns({
7472
before: now - STALE_BACKUP_RUN_MS,
@@ -82,6 +80,17 @@ export async function runR2Backup(
8280
count: reconciledRuns,
8381
}));
8482
}
83+
return reconciledRuns;
84+
}
85+
86+
export async function runR2Backup(
87+
env: OrbitBindings,
88+
kind: BackupKind,
89+
now = Date.now(),
90+
actorAccountId: string | null = null,
91+
): Promise<{ runId: string; objectKey: string; manifestChecksum: string; objectChecksum: string }> {
92+
const repository = new D1PlatformRepository(env.DB);
93+
await reconcileStaleBackupRuns(env, now);
8594
const runId = createEntityId();
8695
const startedAt = Date.now();
8796
await repository.startBackupRun({ id: runId, kind, actorAccountId, now });

‎src/server/repositories/d1/d1-identity-repository.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,12 @@ export class D1IdentityRepository implements IdentityRepository {
528528
OR (MIN(idle_expires_at, absolute_expires_at) <= ?)
529529
`).bind(sessionCutoff, sessionCutoff),
530530
this.#db.prepare(`
531-
DELETE FROM idempotency_keys WHERE expires_at <= ?
531+
DELETE FROM idempotency_keys
532+
WHERE expires_at <= ?
533+
AND NOT EXISTS (
534+
SELECT 1 FROM media_transform_claims claim
535+
WHERE claim.idempotency_id = idempotency_keys.id
536+
)
532537
`).bind(now),
533538
]);
534539
return {

‎src/worker.ts‎

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ import {
55
} from './server/identity/bindings';
66
import { handleApiRequest, runIdentityCleanup, type ApiDependencies } from './server/http/api';
77
import { dashboardAssetResponse } from './server/dashboard/response';
8-
import { runScheduledBackups } from './server/backup/r2-backup';
8+
import {
9+
reconcileStaleBackupRuns,
10+
runScheduledBackups,
11+
} from './server/backup/r2-backup';
912
import {
1013
bumpPublicCacheEpoch,
1114
mutationInvalidatesPublicCache,
@@ -25,6 +28,32 @@ interface ExecutionContextLike {
2528
waitUntil(promise: Promise<unknown>): void;
2629
}
2730

31+
interface ScheduledControllerLike {
32+
cron?: string;
33+
scheduledTime?: number;
34+
}
35+
36+
interface ScheduledDependencies {
37+
runIdentityCleanup(env: OrbitBindings, now: number): Promise<unknown>;
38+
runScheduledBackups(env: OrbitBindings, now: number): Promise<unknown>;
39+
cleanupMedia(env: OrbitBindings, now: number): Promise<unknown>;
40+
reconcileStaleBackupRuns(env: OrbitBindings, now: number): Promise<unknown>;
41+
}
42+
43+
export const BACKUP_CRON = '17 3 * * *';
44+
export const BACKUP_RECONCILIATION_CRON = '0 4 * * *';
45+
46+
const scheduledDependencies: ScheduledDependencies = {
47+
runIdentityCleanup,
48+
runScheduledBackups,
49+
cleanupMedia: async (env, now) => await cleanupMedia(
50+
env,
51+
new D1MediaRepository(env.DB),
52+
now,
53+
),
54+
reconcileStaleBackupRuns,
55+
};
56+
2857
interface WorkerDependencies extends Omit<ApiDependencies, 'requestId'> {
2958
publicRepository?: PublicRepository;
3059
agentRepository?: AgentRepository;
@@ -140,17 +169,46 @@ export async function handleWorkerRequest(
140169
return protectFromIndexing(response, env);
141170
}
142171

172+
export async function runScheduledMaintenance(
173+
controller: ScheduledControllerLike,
174+
env: OrbitBindings,
175+
dependencies: ScheduledDependencies = scheduledDependencies,
176+
): Promise<void> {
177+
const now = controller.scheduledTime ?? Date.now();
178+
if (controller.cron === BACKUP_RECONCILIATION_CRON) {
179+
await dependencies.reconcileStaleBackupRuns(env, now);
180+
return;
181+
}
182+
183+
const tasks = [
184+
['identity_cleanup', () => dependencies.runIdentityCleanup(env, now)],
185+
['backup', () => dependencies.runScheduledBackups(env, now)],
186+
['media_cleanup', () => dependencies.cleanupMedia(env, now)],
187+
] as const;
188+
const results = await Promise.allSettled(tasks.map(([, run]) => Promise.resolve().then(run)));
189+
const failedTasks: string[] = [];
190+
results.forEach((result, index) => {
191+
if (result.status === 'fulfilled') return;
192+
const task = tasks[index]?.[0] ?? 'unknown';
193+
failedTasks.push(task);
194+
console.error(JSON.stringify({
195+
event: 'scheduled.task',
196+
task,
197+
status: 'failed',
198+
}));
199+
});
200+
if (failedTasks.length > 0) {
201+
throw new Error(`scheduled_maintenance_failed:${failedTasks.join(',')}`);
202+
}
203+
}
204+
143205
export default {
144206
async fetch(request: Request, env: OrbitBindings): Promise<Response> {
145207
return await handleWorkerRequest(request, env);
146208
},
147209

148-
scheduled(_controller: unknown, env: OrbitBindings, ctx: ExecutionContextLike): void {
210+
scheduled(controller: ScheduledControllerLike, env: OrbitBindings, ctx: ExecutionContextLike): void {
149211
assertDeploymentBindings(env);
150-
ctx.waitUntil(Promise.all([
151-
runIdentityCleanup(env),
152-
runScheduledBackups(env),
153-
cleanupMedia(env, new D1MediaRepository(env.DB)),
154-
]));
212+
ctx.waitUntil(runScheduledMaintenance(controller, env));
155213
},
156214
};

‎wrangler.jsonc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@
2727
"enabled": false
2828
},
2929
"triggers": {
30-
"crons": ["17 3 * * *"]
30+
"crons": ["17 3 * * *", "0 4 * * *"]
3131
}
3232
}

‎wrangler.production.dark-launch.jsonc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,6 @@
5656
}
5757
},
5858
"triggers": {
59-
"crons": ["17 3 * * *"]
59+
"crons": ["17 3 * * *", "0 4 * * *"]
6060
}
6161
}

‎wrangler.production.live.jsonc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,6 @@
5656
}
5757
},
5858
"triggers": {
59-
"crons": ["17 3 * * *"]
59+
"crons": ["17 3 * * *", "0 4 * * *"]
6060
}
6161
}

0 commit comments

Comments
 (0)