Skip to content

Commit 03bd0fe

Browse files
neonwattyneonwatty
authored
Merge pull request #358 from mean-weasel/codex/reconcile-existing-installations
feat: reconcile existing GitHub installations
2 parents 675d407 + c149722 commit 03bd0fe

8 files changed

Lines changed: 617 additions & 55 deletions

‎docs/installation-usage-counting.md‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,17 @@ The mirror is not created until the installation identity record is visible. If
5353
still propagating, the Durable Object makes at most 1,440 one-minute retry checks without resetting
5454
the original budget. The budget is a non-temporal integer; no submission timestamp is stored. If no
5555
identity arrives, it purges the unanchored counter instead of retaining usage that the scheduled
56-
cleanup cannot discover. This intentionally favors deletion safety over completeness for a
57-
pre-tracking installation or a permanently missed installation-created webhook.
56+
cleanup cannot discover. Before cleanup, the same daily task finds active GitHub App installations
57+
that lack an identity record and creates those missing records using the approved minimal schema. It stores an
58+
aggregate-only audit, and reuses the fetched installation set for cleanup. That idempotent repair
59+
lets installations from before tracking—and installations whose creation webhook was permanently
60+
missed—begin prospective counting without a reinstall. The reconciliation code also supports a
61+
non-writing dry run for controlled verification. Active installations without a supported GitHub
62+
User or Organization identity remain part of cleanup but are skipped by reconciliation and counted
63+
only in the aggregate audit. Apply repairs at most 25 records at a time and reports the remaining
64+
aggregate count so large inventories finish safely over later daily runs. Each repair candidate is
65+
confirmed active immediately after creation; an uninstall racing the repair triggers the
66+
same complete identity-and-usage cleanup as an uninstall webhook.
5867

5968
Delivery uses one opaque event ID across retries, so an ambiguous retry does not increment twice.
6069
It has the same best-effort delivery boundary as the existing anonymous public counter; it is not a

‎src/index.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import api from './routes/api';
55
import githubWebhook from './routes/github-webhook';
66
import { createBoardDogfoodToken } from './lib/boardDogfood';
77
import { sweepInstallationRecords } from './lib/installation-retention';
8+
import { listActiveGitHubInstallations } from './lib/github-installation-inventory';
9+
import { reconcileInstallationRecords } from './lib/installation-reconciliation';
810

911
export { FeedbackCounter } from './lib/feedback-counter';
1012

@@ -151,7 +153,13 @@ export async function scheduled(
151153
env: Env,
152154
_ctx: ExecutionContext
153155
): Promise<void> {
154-
await sweepInstallationRecords(env);
156+
await sweepInstallationRecords(env, {
157+
listActiveInstallationIds: async () => {
158+
const inventory = await listActiveGitHubInstallations(env);
159+
await reconcileInstallationRecords(env, { mode: 'apply', inventory });
160+
return new Set(inventory.installationIds);
161+
},
162+
});
155163
}
156164

157165
export default {

‎src/lib/github-installation-inventory.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import { generateGitHubAppJWT } from './jwt';
2+
import { GITHUB_API, githubHeaders } from './github';
3+
import {
4+
isCanonicalGitHubProfileUrl,
5+
isGitHubAccountLogin,
6+
isInstallationAccountType,
7+
type NewInstallationRecord,
8+
} from './installation-analytics';
9+
import type { Env } from '../types';
10+
11+
const GITHUB_PAGE_SIZE = 100;
12+
export const MAX_GITHUB_INSTALLATION_PAGES = 20;
13+
14+
interface GitHubListOptions {
15+
fetchImpl?: typeof fetch;
16+
createJwt?: (appId: string, privateKey: string) => Promise<string>;
17+
}
18+
19+
export interface GitHubInstallationInventory {
20+
installationIds: number[];
21+
records: NewInstallationRecord[];
22+
skippedCount: number;
23+
pageCount: number;
24+
}
25+
26+
export async function listActiveGitHubInstallations(
27+
env: Env,
28+
options: GitHubListOptions = {}
29+
): Promise<GitHubInstallationInventory> {
30+
if (!env.GITHUB_APP_ID || !env.GITHUB_PRIVATE_KEY) {
31+
throw new Error('GitHub App credentials are required for installation reconciliation');
32+
}
33+
34+
const fetchImpl = options.fetchImpl ?? fetch;
35+
const createJwt = options.createJwt ?? generateGitHubAppJWT;
36+
const jwt = await createJwt(env.GITHUB_APP_ID, env.GITHUB_PRIVATE_KEY);
37+
const installationIds = new Set<number>();
38+
const records: NewInstallationRecord[] = [];
39+
let skippedCount = 0;
40+
41+
for (let page = 1; page <= MAX_GITHUB_INSTALLATION_PAGES; page += 1) {
42+
const response = await fetchImpl(
43+
`${GITHUB_API}/app/installations?per_page=${GITHUB_PAGE_SIZE}&page=${page}`,
44+
{ headers: githubHeaders(jwt) }
45+
);
46+
if (!response.ok) {
47+
throw new Error(`Failed to list GitHub App installations: ${response.status}`);
48+
}
49+
50+
const installations = (await response.json()) as unknown;
51+
if (!Array.isArray(installations)) {
52+
throw new Error('GitHub returned an invalid installation list');
53+
}
54+
for (const value of installations) {
55+
const installation = installationFromGitHub(value);
56+
if (installationIds.has(installation.installationId)) {
57+
throw new Error('GitHub returned a duplicate installation');
58+
}
59+
installationIds.add(installation.installationId);
60+
if (installation.record) records.push(installation.record);
61+
else skippedCount += 1;
62+
}
63+
if (installations.length < GITHUB_PAGE_SIZE) {
64+
return { installationIds: [...installationIds], records, skippedCount, pageCount: page };
65+
}
66+
}
67+
68+
throw new Error('GitHub installation pagination exceeded the safety limit');
69+
}
70+
71+
function installationFromGitHub(value: unknown): {
72+
installationId: number;
73+
record: NewInstallationRecord | null;
74+
} {
75+
if (!value || typeof value !== 'object') {
76+
throw new Error('GitHub returned an invalid installation record');
77+
}
78+
const candidate = value as {
79+
id?: unknown;
80+
account?: { login?: unknown; type?: unknown; html_url?: unknown };
81+
created_at?: unknown;
82+
};
83+
if (
84+
typeof candidate.id !== 'number' ||
85+
!Number.isSafeInteger(candidate.id) ||
86+
candidate.id <= 0
87+
) {
88+
throw new Error('GitHub returned an invalid installation record');
89+
}
90+
const account = candidate.account;
91+
if (!account || !isInstallationAccountType(account.type)) {
92+
return { installationId: candidate.id, record: null };
93+
}
94+
const installedAt = normalizeGitHubDate(candidate.created_at);
95+
if (
96+
typeof account.login !== 'string' ||
97+
!isGitHubAccountLogin(account.login) ||
98+
typeof account.html_url !== 'string' ||
99+
!isCanonicalGitHubProfileUrl(account.html_url, account.login) ||
100+
!installedAt
101+
) {
102+
throw new Error('GitHub returned an invalid installation record');
103+
}
104+
return {
105+
installationId: candidate.id,
106+
record: {
107+
installationId: candidate.id,
108+
account: {
109+
login: account.login,
110+
type: account.type,
111+
profileUrl: account.html_url,
112+
},
113+
installedAt,
114+
},
115+
};
116+
}
117+
118+
function normalizeGitHubDate(value: unknown): string | null {
119+
if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) return null;
120+
return new Date(value).toISOString();
121+
}

‎src/lib/installation-analytics.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,13 @@ export function installationRecordKey(installationId: number): string {
2929
export async function createInstallationRecord(
3030
store: KVNamespace,
3131
installation: NewInstallationRecord
32-
): Promise<void> {
32+
): Promise<boolean> {
3333
const key = installationRecordKey(installation.installationId);
3434
const existing = await store.get(key);
3535
if (existing !== null) {
3636
const record = parseInstallationIdentityRecord(existing, installation.installationId);
3737
assertInstallationIdentityRecord(record, installation.installationId);
38-
return;
38+
return false;
3939
}
4040

4141
const record: InstallationIdentityRecord = {
@@ -46,6 +46,7 @@ export async function createInstallationRecord(
4646
};
4747
assertInstallationIdentityRecord(record, installation.installationId);
4848
await store.put(key, JSON.stringify(record));
49+
return true;
4950
}
5051

5152
function parseInstallationIdentityRecord(value: string, installationId: number): unknown {

‎src/lib/installation-reconciliation.ts‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import { INSTALLATION_RECORD_PREFIX, createInstallationRecord } from './installation-analytics';
2+
import {
3+
confirmGitHubInstallationIsInactive,
4+
deleteInstallationData,
5+
INSTALLATION_SWEEP_PAGE_SIZE,
6+
} from './installation-retention';
7+
import {
8+
listActiveGitHubInstallations,
9+
type GitHubInstallationInventory,
10+
} from './github-installation-inventory';
11+
import type { Env } from '../types';
12+
13+
const STORED_RECORD_PAGE_SIZE = 1000;
14+
const MAX_STORED_RECORD_PAGES = 20;
15+
const INSTALLATION_RECONCILIATION_BATCH_SIZE = 25;
16+
export const MAX_SCHEDULED_GITHUB_REQUESTS = 50;
17+
export const INSTALLATION_RECONCILIATION_AUDIT_KEY =
18+
'operations:installation-reconciliation:last-success';
19+
20+
type ReconciliationMode = 'dry-run' | 'apply';
21+
22+
interface ReconciliationOptions {
23+
mode?: ReconciliationMode;
24+
now?: Date;
25+
inventory?: GitHubInstallationInventory;
26+
confirmInactive?: (env: Env, installationId: number) => Promise<boolean>;
27+
deleteInstallation?: (env: Env, installationId: number) => Promise<void>;
28+
}
29+
30+
interface InstallationReconciliationAudit {
31+
schemaVersion: 1;
32+
mode: ReconciliationMode;
33+
completedAt: string;
34+
activeCount: number;
35+
eligibleCount: number;
36+
skippedCount: number;
37+
existingCount: number;
38+
missingCount: number;
39+
processedCount: number;
40+
createdCount: number;
41+
inactiveCount: number;
42+
remainingCount: number;
43+
}
44+
45+
export async function reconcileInstallationRecords(
46+
env: Env,
47+
options: ReconciliationOptions = {}
48+
): Promise<InstallationReconciliationAudit> {
49+
const store = env.INSTALLATION_ANALYTICS;
50+
if (!store) throw new Error('INSTALLATION_ANALYTICS binding is required for reconciliation');
51+
52+
const mode = options.mode ?? 'dry-run';
53+
const inventory = options.inventory ?? (await listActiveGitHubInstallations(env));
54+
const storedIds = await listStoredInstallationIds(store);
55+
const missing = inventory.records.filter(record => !storedIds.has(record.installationId));
56+
let createdCount = 0;
57+
let inactiveCount = 0;
58+
let processedCount = 0;
59+
60+
if (mode === 'apply') {
61+
const confirmInactive = options.confirmInactive ?? confirmGitHubInstallationIsInactive;
62+
const deleteInstallation = options.deleteInstallation ?? deleteInstallationData;
63+
const batchSize = Math.max(
64+
0,
65+
Math.min(
66+
INSTALLATION_RECONCILIATION_BATCH_SIZE,
67+
MAX_SCHEDULED_GITHUB_REQUESTS - inventory.pageCount - INSTALLATION_SWEEP_PAGE_SIZE
68+
)
69+
);
70+
for (const installation of missing.slice(0, batchSize)) {
71+
processedCount += 1;
72+
const created = await createInstallationRecord(store, installation);
73+
if (await confirmInactive(env, installation.installationId)) {
74+
await deleteInstallation(env, installation.installationId);
75+
inactiveCount += 1;
76+
} else if (created) {
77+
createdCount += 1;
78+
}
79+
}
80+
}
81+
82+
const audit: InstallationReconciliationAudit = {
83+
schemaVersion: 1,
84+
mode,
85+
completedAt: (options.now ?? new Date()).toISOString(),
86+
activeCount: inventory.installationIds.length,
87+
eligibleCount: inventory.records.length,
88+
skippedCount: inventory.skippedCount,
89+
existingCount: inventory.records.length - missing.length,
90+
missingCount: missing.length,
91+
processedCount,
92+
createdCount,
93+
inactiveCount,
94+
remainingCount: missing.length - processedCount,
95+
};
96+
if (mode === 'apply') {
97+
await store.put(INSTALLATION_RECONCILIATION_AUDIT_KEY, JSON.stringify(audit));
98+
}
99+
return audit;
100+
}
101+
102+
async function listStoredInstallationIds(store: KVNamespace): Promise<Set<number>> {
103+
const ids = new Set<number>();
104+
let cursor: string | undefined;
105+
for (let pageNumber = 0; pageNumber < MAX_STORED_RECORD_PAGES; pageNumber += 1) {
106+
const page = await store.list({
107+
prefix: INSTALLATION_RECORD_PREFIX,
108+
limit: STORED_RECORD_PAGE_SIZE,
109+
...(cursor ? { cursor } : {}),
110+
});
111+
for (const key of page.keys) {
112+
const rawId = key.name.slice(INSTALLATION_RECORD_PREFIX.length);
113+
if (!/^\d+$/.test(rawId)) throw new Error('Malformed installation record key');
114+
const id = Number(rawId);
115+
if (!Number.isSafeInteger(id) || id <= 0) {
116+
throw new Error('Malformed installation record key');
117+
}
118+
ids.add(id);
119+
}
120+
if (page.list_complete) return ids;
121+
if (!page.cursor) throw new Error('Installation record pagination omitted its cursor');
122+
cursor = page.cursor;
123+
}
124+
throw new Error('Installation record pagination exceeded the safety limit');
125+
}

‎src/lib/installation-retention.ts‎

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { generateGitHubAppJWT } from './jwt';
22
import { GITHUB_API, githubHeaders } from './github';
33
import { INSTALLATION_RECORD_PREFIX, installationRecordKey } from './installation-analytics';
4+
import { listActiveGitHubInstallations } from './github-installation-inventory';
45
import {
56
deleteInstallationFeedbackCounter,
67
purgeInstallationFeedbackCounter,
@@ -20,8 +21,7 @@ import type { Env } from '../types';
2021

2122
export type { InstallationCleanupAudit } from './installation-cleanup-state';
2223

23-
const GITHUB_PAGE_SIZE = 100;
24-
export const MAX_GITHUB_INSTALLATION_PAGES = 20;
24+
export { MAX_GITHUB_INSTALLATION_PAGES } from './github-installation-inventory';
2525
export const INSTALLATION_SWEEP_PAGE_SIZE = 25;
2626
export const MAX_CONCURRENT_CLEANUP_OPERATIONS = 6;
2727

@@ -90,41 +90,8 @@ export async function listActiveGitHubInstallationIds(
9090
env: Env,
9191
options: GitHubListOptions = {}
9292
): Promise<Set<number>> {
93-
if (!env.GITHUB_APP_ID || !env.GITHUB_PRIVATE_KEY) {
94-
throw new Error('GitHub App credentials are required for installation cleanup');
95-
}
96-
97-
const fetchImpl = options.fetchImpl ?? fetch;
98-
const createJwt = options.createJwt ?? generateGitHubAppJWT;
99-
const jwt = await createJwt(env.GITHUB_APP_ID, env.GITHUB_PRIVATE_KEY);
100-
const activeIds = new Set<number>();
101-
102-
for (let page = 1; page <= MAX_GITHUB_INSTALLATION_PAGES; page += 1) {
103-
const response = await fetchImpl(
104-
`${GITHUB_API}/app/installations?per_page=${GITHUB_PAGE_SIZE}&page=${page}`,
105-
{
106-
headers: githubHeaders(jwt),
107-
}
108-
);
109-
110-
if (!response.ok) {
111-
throw new Error(`Failed to list GitHub App installations: ${response.status}`);
112-
}
113-
114-
const installations = (await response.json()) as unknown;
115-
if (!Array.isArray(installations)) {
116-
throw new Error('GitHub returned an invalid installation list');
117-
}
118-
119-
for (const installation of installations) {
120-
const id = installationIdFromGitHub(installation);
121-
activeIds.add(id);
122-
}
123-
124-
if (installations.length < GITHUB_PAGE_SIZE) return activeIds;
125-
}
126-
127-
throw new Error('GitHub installation pagination exceeded the safety limit');
93+
const inventory = await listActiveGitHubInstallations(env, options);
94+
return new Set(inventory.installationIds);
12895
}
12996

13097
export async function confirmGitHubInstallationIsInactive(
@@ -281,17 +248,6 @@ async function listStoredInstallationPage(
281248
return { ids, ...(page.list_complete ? {} : { cursor: page.cursor }) };
282249
}
283250

284-
function installationIdFromGitHub(value: unknown): number {
285-
if (!value || typeof value !== 'object' || !('id' in value)) {
286-
throw new Error('GitHub returned an invalid installation record');
287-
}
288-
const id = (value as { id: unknown }).id;
289-
if (typeof id !== 'number' || !Number.isSafeInteger(id) || id <= 0) {
290-
throw new Error('GitHub returned an invalid installation ID');
291-
}
292-
return id;
293-
}
294-
295251
function hexToBytes(hex: string): Uint8Array | null {
296252
if (!/^[0-9a-f]{64}$/i.test(hex)) return null;
297253
const bytes = new Uint8Array(hex.length / 2);

0 commit comments

Comments
 (0)