Skip to content

Commit 999a027

Browse files
refactor(scripts): scrub-data writes its config root through gitsheets
Replace writeConfigRootCommit's hash-object / temp index / write-tree / commit-tree plumbing with one orphan transact({ parent: null }) that stages the source's .gitsheets/ files with tx.writeFile (gitsheets 2.9, #305). The final snapshot commit stays on raw commit-tree + update-ref: 2.9's tx.sheet() loads configs from the transaction's parent (HEAD for an orphan), never from files staged with tx.writeFile in the same transaction. So records can't be upserted straight into an orphan, and gitsheets can't point a branch at an existing commit. Each remaining raw call is commented with its reason. Tests now pin the snapshot shape: a parentless commit carrying the source's configs and contracts byte-for-byte plus records, HEAD on the snapshot branch, the tag, a no-op same-seed re-run, and no refs after a failed verification. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011Q2tgjZEUrtvVk6k8SpdXJ
1 parent 63ac793 commit 999a027

2 files changed

Lines changed: 102 additions & 52 deletions

File tree

‎apps/api/scripts/scrub-data.ts‎

Lines changed: 57 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,32 @@
2727
* would land on another record's path — fails the run instead of being
2828
* silently collapsed.
2929
*
30+
* The source's `.gitsheets/` files (configs + vendored contracts) are copied
31+
* with `tx.writeFile` into a detached gitsheets orphan commit that the record
32+
* transaction builds on; the staged tree is then re-committed as the
33+
* parentless snapshot on `snapshot-<seed>-scrubbed`, HEAD pointed at it and a
34+
* same-named tag added (`.github/workflows/snapshot.yml` pushes HEAD's
35+
* branch).
36+
*
3037
* If verification fails the script exits non-zero and no target ref moves
3138
* (the staged objects are left unreferenced).
3239
*/
3340

3441
import { createHash } from 'node:crypto';
3542
import { execFile } from 'node:child_process';
36-
import { mkdir, mkdtemp, readdir, readFile, rm } from 'node:fs/promises';
37-
import { tmpdir } from 'node:os';
38-
import { dirname, join, resolve } from 'node:path';
43+
import { mkdir, readdir, readFile, rm } from 'node:fs/promises';
44+
import { join, resolve } from 'node:path';
3945
import { promisify } from 'node:util';
4046

4147
import { faker } from '@faker-js/faker';
42-
import { getFormat, openRepo, RECORD_PATH_KEY, type BlobHandle, type UpsertResult } from 'gitsheets';
48+
import {
49+
getFormat,
50+
openRepo,
51+
RECORD_PATH_KEY,
52+
type BlobHandle,
53+
type Repository,
54+
type UpsertResult,
55+
} from 'gitsheets';
4356
import { SHEET_REFERENCES } from '@cfp/shared';
4457

4558
const exec = promisify(execFile);
@@ -387,6 +400,10 @@ interface ScrubbedRecord {
387400
readonly attachments: Record<string, BlobHandle> | null;
388401
}
389402

403+
/**
404+
* Open the target repo, `git init`-ing it first when absent. Raw git:
405+
* gitsheets opens existing repositories but has no init.
406+
*/
390407
async function initOrOpenTargetRepo(targetPath: string): Promise<GitRunner> {
391408
await mkdir(targetPath, { recursive: true });
392409
const git: GitRunner = (...args) => exec('git', args, { cwd: targetPath });
@@ -403,50 +420,29 @@ async function initOrOpenTargetRepo(targetPath: string): Promise<GitRunner> {
403420
return git;
404421
}
405422

406-
async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
407-
const chunks: Buffer[] = [];
408-
for await (const chunk of stream) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
409-
return Buffer.concat(chunks);
410-
}
411-
412423
/**
413-
* Write a parentless commit holding only the source's `.gitsheets/` configs,
414-
* for the record transaction to build on.
424+
* Write a parentless commit holding only the source's `.gitsheets/` files
425+
* (configs plus vendored contracts), for the record transaction to build on:
426+
* an orphan `transact({ parent: null })` staging each file with
427+
* `tx.writeFile`. No branch is named, so the commit is detached and no ref
428+
* moves.
415429
*
416-
* Stays on raw git plumbing (hash-object / mktree / commit-tree): gitsheets
417-
* 2.7 has no public API for writing non-record files (`tx.writeFile` is
418-
* internal) or for orphan commits. Swap once upstream exposes them.
430+
* The configs can't go in the record transaction itself: a transaction's
431+
* sheets load their configs from its parent commit (from HEAD for an orphan),
432+
* never from files staged with `tx.writeFile` in the same transaction.
419433
*/
420434
async function writeConfigRootCommit(
421-
targetPath: string,
422-
configs: ReadonlyArray<{ name: string; content: Buffer }>,
435+
repo: Repository,
436+
configs: ReadonlyArray<{ path: string; content: Buffer }>,
423437
): Promise<string> {
424-
// A throwaway index builds the nested `.gitsheets/` tree (configs plus
425-
// `contracts/<host>/<path>.toml`) without touching any real index.
426-
const indexFile = join(await mkdtemp(join(tmpdir(), 'cfp-scrub-index-')), 'index');
427-
const indexEnv = { ...process.env, GIT_INDEX_FILE: indexFile };
428-
for (const { name, content } of configs) {
429-
const hash = await new Promise<string>((resolvePromise, reject) => {
430-
const child = execFile(
431-
'git',
432-
['hash-object', '-w', '--stdin'],
433-
{ cwd: targetPath },
434-
(err, stdout) => (err ? reject(err) : resolvePromise(stdout.trim())),
435-
);
436-
child.stdin?.end(content);
437-
});
438-
await exec('git', ['update-index', '--add', '--cacheinfo', `100644,${hash},.gitsheets/${name}`], {
439-
cwd: targetPath,
440-
env: indexEnv,
441-
});
442-
}
443-
const rootTree = (await exec('git', ['write-tree'], { cwd: targetPath, env: indexEnv })).stdout.trim();
444-
await rm(dirname(indexFile), { recursive: true, force: true });
445-
const { stdout } = await exec('git', ['commit-tree', rootTree, '-m', 'scrub-data: sheet configs'], {
446-
cwd: targetPath,
447-
env: { ...process.env, ...gitIdentityEnv(new Date().toISOString()) },
448-
});
449-
return stdout.trim();
438+
const { commitHash } = await repo.transact(
439+
{ parent: null, message: 'scrub-data: sheet configs', author: SNAPSHOT_AUTHOR },
440+
async (tx) => {
441+
for (const { path, content } of configs) tx.writeFile(path, content);
442+
},
443+
);
444+
if (commitHash === null) throw new Error('scrub-data: internal: config root transaction produced no commit');
445+
return commitHash;
450446
}
451447

452448
function gitIdentityEnv(timestamp: string): Record<string, string> {
@@ -556,24 +552,23 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
556552
// moves; its tree is re-committed as the orphan snapshot in step 9.
557553
// -------------------------------------------------------------------------
558554
const git = await initOrOpenTargetRepo(target);
555+
// Raw git: resolves a non-bare target's `.git` for openRepo.
559556
const targetGitDir = (await git('rev-parse', '--absolute-git-dir')).stdout.trim();
560557
const targetRepo = await openRepo({ gitDir: targetGitDir });
561558

562559
// Everything under the source's `.gitsheets/`: sheet configs plus the
563560
// vendored schema contracts (`contracts/…`) that configs may declare.
564561
const sourceRef = sourceHeadHash ?? 'HEAD';
562+
// Raw git: gitsheets reads a blob at a path but has no subtree listing.
565563
const configPaths = (
566564
await exec('git', ['ls-tree', '-r', '--name-only', sourceRef, '.gitsheets/'], { cwd: source })
567565
).stdout
568566
.split('\n')
569567
.filter((p) => p.length > 0);
570568
const configs = await Promise.all(
571-
configPaths.map(async (path) => ({
572-
name: path.slice('.gitsheets/'.length),
573-
content: await streamToBuffer(await sourceRepo.readBlobStream(sourceRef, path)),
574-
})),
569+
configPaths.map(async (path) => ({ path, content: await (await sourceRepo.readBlob(sourceRef, path)).read() })),
575570
);
576-
const configRoot = await writeConfigRootCommit(target, configs);
571+
const configRoot = await writeConfigRootCommit(targetRepo, configs);
577572

578573
const writtenFiles: SnapshotFile[] = [];
579574
const scrubCounts: Record<string, number> = {};
@@ -649,13 +644,22 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
649644
// -------------------------------------------------------------------------
650645
// 9. Commit: orphan squashed commit of the staged tree
651646
//
652-
// Stays on raw git (commit-tree / update-ref / symbolic-ref / tag):
653-
// gitsheets 2.7 can't write a parentless commit. The snapshot workflow
654-
// pushes HEAD and tags it, so HEAD must point at the new branch.
647+
// Stays on raw git, each step for want of a gitsheets API:
648+
// - commit-tree: re-committing the staged tree as a parentless commit.
649+
// gitsheets 2.9's orphan transact starts from an empty tree, and its
650+
// sheets can't see configs staged in the same transaction (see
651+
// writeConfigRootCommit), so the records can't be written straight
652+
// into an orphan.
653+
// - update-ref: gitsheets can't point a branch at an existing commit
654+
// (only commit onto one), and a same-seed re-run overwrites the branch.
655+
// - symbolic-ref HEAD: gitsheets never moves HEAD. The snapshot workflow
656+
// pushes the branch HEAD is on, so HEAD must point at the new branch.
657+
// - tag: gitsheets has no tag API.
655658
// -------------------------------------------------------------------------
656659
const timestamp = new Date().toISOString();
657660
const branchName = `snapshot-${seed.replace(/[^a-z0-9]/gi, '-')}-scrubbed`;
658661

662+
// Raw git: HEAD's tree hash, which gitsheets doesn't expose.
659663
const currentTree = await git('rev-parse', '--verify', '--quiet', 'HEAD^{tree}')
660664
.then((r) => r.stdout.trim())
661665
.catch(() => null);
@@ -686,7 +690,8 @@ See apps/api/scripts/scrub-data.ts and specs/behaviors/storage.md.`;
686690
await git('symbolic-ref', 'HEAD', `refs/heads/${branchName}`);
687691
await exec('git', ['tag', '-f', branchName, commitHash], { cwd: target, env: commitEnv });
688692

689-
// Bring a non-bare target's working tree in line with the new HEAD.
693+
// Bring a non-bare target's working tree in line with the new HEAD. Raw
694+
// git: gitsheets works bare-style and never touches a working tree.
690695
if ((await git('rev-parse', '--is-bare-repository')).stdout.trim() !== 'true') {
691696
for (const entry of await readdir(target, { withFileTypes: true })) {
692697
if (entry.name === '.git') continue;

‎apps/api/tests/scrub-data.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,49 @@ describe('scrubRepo', () => {
483483
INTEGRATION_TIMEOUT,
484484
);
485485

486+
it(
487+
'commits a parentless snapshot holding configs, contracts and records, on HEAD + tag',
488+
async () => {
489+
sourceTmp = await makeSourceRepo({
490+
people: [
491+
{ id: uuid(1), slug: 'dana', fullName: 'Dana Gray', accountLevel: 'user', createdAt: NOW, updatedAt: NOW },
492+
],
493+
projects: [
494+
{ id: uuid(2), slug: 'proj', title: 'Proj', stage: 'bootstrapping', createdAt: NOW, updatedAt: NOW },
495+
],
496+
});
497+
targetTmp = await makeTmp();
498+
const git = async (cwd: string, ...args: string[]) => (await exec('git', args, { cwd })).stdout.trim();
499+
500+
const result = await scrubRepo({ source: sourceTmp.path, target: targetTmp.path, seed: 'orphan-test' });
501+
const commit = result.commitHash!;
502+
expect(result.branchName).toBe('snapshot-orphan-test-scrubbed');
503+
504+
// Parentless, on the snapshot branch, HEAD pointing at it, and tagged.
505+
expect(await git(targetTmp.path, 'rev-list', '--parents', '-n', '1', commit)).toBe(commit);
506+
expect(await git(targetTmp.path, 'symbolic-ref', 'HEAD')).toBe('refs/heads/snapshot-orphan-test-scrubbed');
507+
expect(await git(targetTmp.path, 'rev-parse', 'HEAD')).toBe(commit);
508+
expect(await git(targetTmp.path, 'rev-parse', 'refs/tags/snapshot-orphan-test-scrubbed')).toBe(commit);
509+
510+
// `.gitsheets/` (configs + vendored contracts) is byte-identical to the source's.
511+
const sourceConfigs = await git(sourceTmp.path, 'ls-tree', '-r', 'HEAD', '.gitsheets/');
512+
expect(sourceConfigs).toContain('.gitsheets/contracts/');
513+
expect(await git(targetTmp.path, 'ls-tree', '-r', commit, '.gitsheets/')).toBe(sourceConfigs);
514+
515+
// ...alongside the scrubbed records.
516+
const files = (await git(targetTmp.path, 'ls-tree', '-r', '--name-only', commit)).split('\n');
517+
const pseudo = buildSlugMap(['dana'], 'orphan-test').get('dana')!;
518+
expect(files).toContain(`people/${pseudo}.toml`);
519+
expect(files).toContain('projects/proj.toml');
520+
521+
// A same-seed re-run with unchanged content is a no-op.
522+
const again = await scrubRepo({ source: sourceTmp.path, target: targetTmp.path, seed: 'orphan-test' });
523+
expect(again.commitHash).toBeNull();
524+
expect(await git(targetTmp.path, 'rev-parse', 'HEAD')).toBe(commit);
525+
},
526+
INTEGRATION_TIMEOUT,
527+
);
528+
486529
it(
487530
'record counts match source vs snapshot',
488531
async () => {
@@ -540,6 +583,8 @@ describe('scrubRepo', () => {
540583
await expect(
541584
scrubRepo({ source: path2, target: targetTmp.path, seed: 'leak-test' }),
542585
).rejects.toThrow(/Verification failed/);
586+
// No ref moved: the target has no branches or tags at all.
587+
expect((await exec('git', ['for-each-ref'], { cwd: targetTmp.path })).stdout.trim()).toBe('');
543588
},
544589
INTEGRATION_TIMEOUT,
545590
);

0 commit comments

Comments
 (0)