Skip to content

Commit 10c9778

Browse files
Merge pull request #258 from CodeForPhilly/chore/gitsheets-2.11
refactor: gitsheets 2.11 — sync afterMove, fetch, single-transaction snapshots
2 parents c1f5cc5 + 3cfccc2 commit 10c9778

11 files changed

Lines changed: 443 additions & 283 deletions

File tree

‎apps/api/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"bcryptjs": "^3.0.3",
3737
"better-sqlite3": "^12.10.0",
3838
"fastify": "^5.8.5",
39-
"gitsheets": "^2.9.0",
39+
"gitsheets": "^2.11.0",
4040
"jose": "^6.2.3",
4141
"postmark": "^5.1.0",
4242
"samlify": "^2.13.0",

‎apps/api/scripts/check-sheet-configs.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import { dirname, join, relative } from 'node:path';
2222
import { fileURLToPath } from 'node:url';
2323
import { promisify } from 'node:util';
2424

25+
import { openRepo, type Repository } from 'gitsheets';
26+
2527
const exec = promisify(execFile);
2628

2729
// The data repo is private: SSH with a key that can read it (CI: a deploy key).
@@ -40,6 +42,17 @@ async function readTomlTree(dir: string): Promise<Map<string, string>> {
4042
return out;
4143
}
4244

45+
/** Every file path under `dir` at `ref`, recursively. */
46+
async function listFilesUnder(repo: Repository, ref: string, dir: string): Promise<string[]> {
47+
const out: string[] = [];
48+
for (const entry of (await repo.listTree(ref, dir)) ?? []) {
49+
const path = `${dir}/${entry.name}`;
50+
if (entry.type === 'tree') out.push(...(await listFilesUnder(repo, ref, path)));
51+
else if (entry.type === 'blob') out.push(path);
52+
}
53+
return out;
54+
}
55+
4356
function arg(name: string, fallback: string): string {
4457
const prefix = `--${name}=`;
4558
return process.argv.find((a) => a.startsWith(prefix))?.slice(prefix.length) ?? fallback;
@@ -51,15 +64,18 @@ async function main(): Promise<number> {
5164

5265
const gitDir = await mkdtemp(join(tmpdir(), 'cfp-sheet-configs-'));
5366
try {
54-
const git = (...args: string[]) => exec('git', ['--git-dir', gitDir, ...args]);
67+
// Raw git: gitsheets has no init, and fetches only from a named remote.
5568
await exec('git', ['init', '--bare', '-q', gitDir]);
56-
await git('fetch', '-q', '--depth=1', remote, ref);
69+
await exec('git', ['--git-dir', gitDir, 'remote', 'add', 'origin', remote]);
70+
const repo = await openRepo({ gitDir });
71+
const fetched = await repo.fetch({ remote: 'origin', branch: ref });
72+
if (!fetched.ok) throw new Error(`fetching ${remote}#${ref} failed: ${fetched.error}`);
5773

58-
const { stdout: listing } = await git('ls-tree', '-r', '--name-only', 'FETCH_HEAD', '.gitsheets/');
5974
const upstream = new Map<string, string>();
60-
for (const path of listing.split('\n').filter((p) => p.endsWith('.toml'))) {
61-
const { stdout } = await git('show', `FETCH_HEAD:${path}`);
62-
upstream.set(path.slice('.gitsheets/'.length), stdout);
75+
for (const path of await listFilesUnder(repo, fetched.remoteCommit, '.gitsheets')) {
76+
if (!path.endsWith('.toml')) continue;
77+
const blob = await repo.readBlob(fetched.remoteCommit, path);
78+
upstream.set(path.slice('.gitsheets/'.length), (await blob.read()).toString('utf8'));
6379
}
6480

6581
const vendored = await readTomlTree(VENDORED_DIR);

‎apps/api/scripts/scrub-data.ts‎

Lines changed: 91 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@
2727
* would land on another record's path — fails the run instead of being
2828
* silently collapsed.
2929
*
30-
* The source's `.gitsheets/` files (configs + vendored contracts) are copied
31-
* with `tx.writeFile` into a detached gitsheets orphan commit that the record
32-
* transaction builds on; the staged tree is then re-committed as the
33-
* parentless snapshot on `snapshot-<seed>-scrubbed`, HEAD pointed at it and a
34-
* same-named tag added (`.github/workflows/snapshot.yml` pushes HEAD's
35-
* branch).
30+
* The whole snapshot is one parentless gitsheets transaction on
31+
* `snapshot-<seed>-scrubbed`: it stages the source's `.gitsheets/` files
32+
* (configs + vendored contracts) with `tx.writeFile`, then upserts the
33+
* records, whose sheets read those staged configs. HEAD is pointed at the
34+
* branch and a same-named tag added (`.github/workflows/snapshot.yml` pushes
35+
* HEAD's branch).
3636
*
37-
* If verification fails the script exits non-zero and no target ref moves
38-
* (the staged objects are left unreferenced).
37+
* If verification fails the script exits non-zero and nothing is committed:
38+
* the checks run inside the transaction, so a failure discards it.
3939
*/
4040

4141
import { createHash } from 'node:crypto';
@@ -421,39 +421,18 @@ async function initOrOpenTargetRepo(targetPath: string): Promise<GitRunner> {
421421
}
422422

423423
/**
424-
* Write a parentless commit holding only the source's `.gitsheets/` files
425-
* (configs plus vendored contracts), for the record transaction to build on:
426-
* an orphan `transact({ parent: null })` staging each file with
427-
* `tx.writeFile`. No branch is named, so the commit is detached and no ref
428-
* moves.
429-
*
430-
* The configs can't go in the record transaction itself: a transaction's
431-
* sheets load their configs from its parent commit (from HEAD for an orphan),
432-
* never from files staged with `tx.writeFile` in the same transaction.
424+
* Every file path under `dir` at `ref`, recursively (gitsheets `listTree`
425+
* lists one level).
433426
*/
434-
async function writeConfigRootCommit(
435-
repo: Repository,
436-
configs: ReadonlyArray<{ path: string; content: Buffer }>,
437-
): Promise<string> {
438-
const { commitHash } = await repo.transact(
439-
{ parent: null, message: 'scrub-data: sheet configs', author: SNAPSHOT_AUTHOR },
440-
async (tx) => {
441-
for (const { path, content } of configs) tx.writeFile(path, content);
442-
},
443-
);
444-
if (commitHash === null) throw new Error('scrub-data: internal: config root transaction produced no commit');
445-
return commitHash;
446-
}
447-
448-
function gitIdentityEnv(timestamp: string): Record<string, string> {
449-
return {
450-
GIT_AUTHOR_NAME: SNAPSHOT_AUTHOR.name,
451-
GIT_AUTHOR_EMAIL: SNAPSHOT_AUTHOR.email,
452-
GIT_COMMITTER_NAME: SNAPSHOT_AUTHOR.name,
453-
GIT_COMMITTER_EMAIL: SNAPSHOT_AUTHOR.email,
454-
GIT_AUTHOR_DATE: timestamp,
455-
GIT_COMMITTER_DATE: timestamp,
456-
};
427+
async function listFilesUnder(repo: Repository, ref: string, dir: string): Promise<string[]> {
428+
const entries = (await repo.listTree(ref, dir)) ?? [];
429+
const out: string[] = [];
430+
for (const entry of entries) {
431+
const path = `${dir}/${entry.name}`;
432+
if (entry.type === 'tree') out.push(...(await listFilesUnder(repo, ref, path)));
433+
else if (entry.type === 'blob') out.push(path);
434+
}
435+
return out;
457436
}
458437

459438
/**
@@ -545,11 +524,12 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
545524
}
546525

547526
// -------------------------------------------------------------------------
548-
// 6. Stage the snapshot tree through gitsheets: a transaction on top of a
549-
// config-only root commit upserts every scrubbed record (canonical
527+
// 6. Write the snapshot through gitsheets: one parentless transaction
528+
// stages the `.gitsheets/` files first (its sheets read configs from the
529+
// transaction's own tree), then upserts every scrubbed record (canonical
550530
// serialization, path templates, content-typed formats) and copies
551-
// attachments. The transaction commits on a bare hash parent, so no ref
552-
// moves; its tree is re-committed as the orphan snapshot in step 9.
531+
// attachments. Count parity and verification (steps 7 and 8) run inside
532+
// the handler, so a failure throws before anything commits.
553533
// -------------------------------------------------------------------------
554534
const git = await initOrOpenTargetRepo(target);
555535
// Raw git: resolves a non-bare target's `.git` for openRepo.
@@ -559,26 +539,42 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
559539
// Everything under the source's `.gitsheets/`: sheet configs plus the
560540
// vendored schema contracts (`contracts/…`) that configs may declare.
561541
const sourceRef = sourceHeadHash ?? 'HEAD';
562-
// Raw git: gitsheets reads a blob at a path but has no subtree listing.
563-
const configPaths = (
564-
await exec('git', ['ls-tree', '-r', '--name-only', sourceRef, '.gitsheets/'], { cwd: source })
565-
).stdout
566-
.split('\n')
567-
.filter((p) => p.length > 0);
542+
const configPaths = await listFilesUnder(sourceRepo, sourceRef, '.gitsheets');
568543
const configs = await Promise.all(
569544
configPaths.map(async (path) => ({ path, content: await (await sourceRepo.readBlob(sourceRef, path)).read() })),
570545
);
571-
const configRoot = await writeConfigRootCommit(targetRepo, configs);
546+
547+
const timestamp = new Date().toISOString();
548+
const branchName = `snapshot-${seed.replace(/[^a-z0-9]/gi, '-')}-scrubbed`;
549+
const commitMessage = `snapshot: anonymized data export from ${timestamp}
550+
551+
Generated by apps/api/scripts/scrub-data.ts.
552+
Source revision: ${sourceHeadHash ?? 'unknown'}
553+
Seed: ${seed}
554+
555+
This snapshot is intended for contributor onboarding. It contains no
556+
real names, no emails, no GitHub identities. All personal data has
557+
been pseudonymized via the documented scrubbing rules.
558+
559+
See apps/api/scripts/scrub-data.ts and specs/behaviors/storage.md.`;
560+
561+
// An orphan transaction creates its branch and refuses an existing one.
562+
// A same-seed re-run (the branch exists) commits detached instead and
563+
// moves the branch below only if the tree changed.
564+
const existing = await targetRepo.resolveRef(`refs/heads/${branchName}`);
572565

573566
const writtenFiles: SnapshotFile[] = [];
574567
const scrubCounts: Record<string, number> = {};
575-
const staged = await targetRepo.transact(
568+
const snapshot = await targetRepo.transact(
576569
{
577-
parent: configRoot,
578-
message: 'scrub-data: staged snapshot tree (re-committed as an orphan)',
570+
parent: null,
571+
...(existing === null ? { branch: branchName } : {}),
572+
message: commitMessage,
579573
author: SNAPSHOT_AUTHOR,
580574
},
581575
async (tx) => {
576+
for (const { path, content } of configs) tx.writeFile(path, content);
577+
582578
const pathsBySheet = new Map<string, Map<string, unknown>>();
583579
for (const { sheetName, record, attachments } of scrubbedRecords) {
584580
const sheet = tx.sheet(sheetName);
@@ -613,82 +609,55 @@ export async function scrubRepo(opts: ScrubOptions): Promise<ScrubResult> {
613609
await sheet.setAttachments(record, bytes);
614610
}
615611
}
612+
613+
// ---------------------------------------------------------------------
614+
// 7. Count validation — a record that collapsed onto another's path,
615+
// or failed to write, shows up here.
616+
// ---------------------------------------------------------------------
617+
const countMismatches: string[] = [];
618+
for (const [sheetName, sourceCount] of Object.entries(sourceCounts)) {
619+
const scrubCount = scrubCounts[sheetName] ?? 0;
620+
if (sourceCount !== scrubCount) {
621+
countMismatches.push(`Sheet "${sheetName}": source=${sourceCount} scrubbed=${scrubCount}`);
622+
}
623+
}
624+
if (countMismatches.length > 0) {
625+
throw new Error(`Record count mismatch:\n${countMismatches.join('\n')}`);
626+
}
627+
628+
// ---------------------------------------------------------------------
629+
// 8. Verification pass (before anything commits)
630+
// ---------------------------------------------------------------------
631+
const verification = verifySnapshotFiles(writtenFiles, new Set(allPersonSlugs));
632+
if (!verification.passed) {
633+
throw new Error(`Verification failed:\n${verification.errors.join('\n')}`);
634+
}
616635
},
617636
);
618-
const treeHash = staged.treeHash;
619-
if (treeHash === null) throw new Error('scrub-data: internal: staging transaction produced no tree');
637+
const commitHash = snapshot.commitHash;
638+
if (commitHash === null) throw new Error('scrub-data: internal: snapshot transaction produced no commit');
620639

621640
// -------------------------------------------------------------------------
622-
// 7. Count validation — a record that collapsed onto another's path, or
623-
// failed to write, shows up here.
641+
// 9. Point the branch, HEAD and tag at the snapshot.
624642
// -------------------------------------------------------------------------
625-
const countMismatches: string[] = [];
626-
for (const [sheetName, sourceCount] of Object.entries(sourceCounts)) {
627-
const scrubCount = scrubCounts[sheetName] ?? 0;
628-
if (sourceCount !== scrubCount) {
629-
countMismatches.push(`Sheet "${sheetName}": source=${sourceCount} scrubbed=${scrubCount}`);
643+
if (existing !== null) {
644+
// Same-seed re-run: an unchanged tree is a no-op (the detached commit is
645+
// left unreferenced). Equal root listings mean equal trees.
646+
const [before, after] = await Promise.all([
647+
targetRepo.listTree(existing),
648+
targetRepo.listTree(commitHash),
649+
]);
650+
if (JSON.stringify(before) === JSON.stringify(after)) {
651+
return { sourceCounts, commitHash: null, branchName };
630652
}
653+
// Raw git: gitsheets can't repoint an existing branch at an unrelated
654+
// commit (an orphan transact refuses an existing branch).
655+
await git('update-ref', `refs/heads/${branchName}`, commitHash);
631656
}
632-
if (countMismatches.length > 0) {
633-
throw new Error(`Record count mismatch:\n${countMismatches.join('\n')}`);
634-
}
635-
636-
// -------------------------------------------------------------------------
637-
// 8. Verification pass (before any ref moves)
638-
// -------------------------------------------------------------------------
639-
const verification = verifySnapshotFiles(writtenFiles, new Set(allPersonSlugs));
640-
if (!verification.passed) {
641-
throw new Error(`Verification failed:\n${verification.errors.join('\n')}`);
642-
}
643-
644-
// -------------------------------------------------------------------------
645-
// 9. Commit: orphan squashed commit of the staged tree
646-
//
647-
// Stays on raw git, each step for want of a gitsheets API:
648-
// - commit-tree: re-committing the staged tree as a parentless commit.
649-
// gitsheets 2.9's orphan transact starts from an empty tree, and its
650-
// sheets can't see configs staged in the same transaction (see
651-
// writeConfigRootCommit), so the records can't be written straight
652-
// into an orphan.
653-
// - update-ref: gitsheets can't point a branch at an existing commit
654-
// (only commit onto one), and a same-seed re-run overwrites the branch.
655-
// - symbolic-ref HEAD: gitsheets never moves HEAD. The snapshot workflow
656-
// pushes the branch HEAD is on, so HEAD must point at the new branch.
657-
// - tag: gitsheets has no tag API.
658-
// -------------------------------------------------------------------------
659-
const timestamp = new Date().toISOString();
660-
const branchName = `snapshot-${seed.replace(/[^a-z0-9]/gi, '-')}-scrubbed`;
661-
662-
// Raw git: HEAD's tree hash, which gitsheets doesn't expose.
663-
const currentTree = await git('rev-parse', '--verify', '--quiet', 'HEAD^{tree}')
664-
.then((r) => r.stdout.trim())
665-
.catch(() => null);
666-
if (currentTree === treeHash) {
667-
return { sourceCounts, commitHash: null, branchName };
668-
}
669-
670-
const commitMessage = `snapshot: anonymized data export from ${timestamp}
671-
672-
Generated by apps/api/scripts/scrub-data.ts.
673-
Source revision: ${sourceHeadHash ?? 'unknown'}
674-
Seed: ${seed}
675-
676-
This snapshot is intended for contributor onboarding. It contains no
677-
real names, no emails, no GitHub identities. All personal data has
678-
been pseudonymized via the documented scrubbing rules.
679-
680-
See apps/api/scripts/scrub-data.ts and specs/behaviors/storage.md.`;
681-
682-
const commitEnv = { ...process.env, ...gitIdentityEnv(timestamp) };
683-
const { stdout: commitHashRaw } = await exec('git', ['commit-tree', treeHash, '-m', commitMessage], {
684-
cwd: target,
685-
env: commitEnv,
686-
});
687-
const commitHash = commitHashRaw.trim();
688-
689-
await git('update-ref', `refs/heads/${branchName}`, commitHash);
657+
// Raw git: gitsheets never moves HEAD. The snapshot workflow pushes the
658+
// branch HEAD is on, so HEAD must point at the new branch.
690659
await git('symbolic-ref', 'HEAD', `refs/heads/${branchName}`);
691-
await exec('git', ['tag', '-f', branchName, commitHash], { cwd: target, env: commitEnv });
660+
await targetRepo.tag(branchName, commitHash, { force: true });
692661

693662
// Bring a non-bare target's working tree in line with the new HEAD. Raw
694663
// git: gitsheets works bare-style and never touches a working tree.

0 commit comments

Comments
 (0)