Skip to content

Commit 84daba8

Browse files
feat(promotion): gate promotion out of an org on a passing check
`orgs.<slug>.check: <name>` in promotion.yml names a vapi-checks.yml check that must pass in that org before any transition promotes out of it. The gate runs the same inline check as the PR workflow, built from the source org's files at the promoted commit, using that org's key from VAPI_PROMOTION_TOKENS. - Checks are validated before any transition: the named check must exist and read and run in the gated org. - Transitions with no changes skip the gate; plan-only runs print `check would run <name> in <org> (<n> simulations × <t> targets)` and run nothing. - On --apply the check runs after bindings refresh and before promotionPlanApply writes the target. Any non-pass (failed, incomplete, build error) throws `Promotion out of <org> blocked: check <name> <outcome> (<run url>)`, so the target is untouched and earlier transitions are still committed (previous change). - A pass is reused for later transitions out of the same org in the same run, and dropped once a transition applies into that org. - The "Reconcile configured promotions" step gets timeout-minutes: 90 on the step, not the job, so the always() commit step still runs. - With no check: configured, promotion is unchanged: a test pins the plan output to what the pre-gate code prints, and asserts no check runs and vapi-checks.yml is never read. - Docs: promotion.example.yml and README ("Check before promoting", and a pointer from "PR Checks"). Refs TEST-141 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent c07fed1 commit 84daba8

8 files changed

Lines changed: 555 additions & 2 deletions

File tree

‎.github/workflows/promotion.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ jobs:
5656

5757
- name: Reconcile configured promotions
5858
id: promotion
59+
# On the step, not the job: a job timeout would cancel the
60+
# always() commit step below, and a gate check (orgs.<slug>.check)
61+
# can run up to its timeoutMinutes per gated org.
62+
timeout-minutes: 90
5963
shell: bash
6064
run: |
6165
set -euo pipefail

‎README.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,30 @@ its cleaned state after downstream deletion completes.
439439
See [sync behavior](docs/learnings/sync-behavior.md#cross-org-promotion-deletions)
440440
for the exact lifecycle.
441441

442+
#### Check before promoting (optional)
443+
444+
Gate an org on a [PR check](#pr-checks-simulations-against-your-branch):
445+
nothing is promoted **out of** it unless the check passes there first.
446+
447+
```yaml
448+
# promotion.yml
449+
orgs:
450+
example-staging:
451+
check: staging-core # a vapi-checks.yml check whose org (and runOrg) is example-staging
452+
```
453+
454+
- Plans print `check would run staging-core in example-staging (<n> simulations × <t> targets)`
455+
and run nothing.
456+
- On `--apply`, the check runs against `resources/example-staging/` at the
457+
promoted commit, in example-staging, with that org's key from
458+
`VAPI_PROMOTION_TOKENS`, before any file is written to the destination. A
459+
failure, an incomplete run (timeout, billing) or a build error blocks the
460+
transition with the run link; transitions that already applied are still
461+
committed.
462+
- A pass is reused for later transitions out of the same org in the same run,
463+
until something is promoted into it.
464+
- Transitions with no changes skip the check.
465+
442466
#### Rolling Back a Promotion
443467

444468
Treat a promotion rollback as a new, auditable Git change: revert the source
@@ -588,6 +612,12 @@ Require the **commit status `Vapi Evals`** in branch protection — not the
588612
status, so dispatch again after that. Running one named check by hand
589613
never changes `Vapi Evals`.
590614

615+
### Gate promotion on a check (optional)
616+
617+
Multi-org repos can require a check to pass in an org before anything is
618+
promoted out of it: set `orgs.<org>.check: <name>` in `promotion.yml` (see
619+
[Check before promoting](#check-before-promoting-optional)).
620+
591621
### Dedicated CI org (optional; recommended with `toolMocks: off`)
592622

593623
1. `npm run setup -- my-ci-org --resources none`.

‎promotion.example.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ orgs:
88
baseUrl: https://api.vapi.ai
99
example-staging:
1010
baseUrl: https://api.vapi.ai
11+
# Optional gate: nothing is promoted out of example-staging unless this
12+
# vapi-checks.yml check (whose org and runOrg are example-staging) passes
13+
# there first. Plans print what it would run; --apply runs it.
14+
# check: staging-core
1115
bindings:
1216
credentials:
1317
default: bind

‎src/promote-cmd.ts‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,16 @@ import { execFileSync } from "node:child_process";
22
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
33
import { dirname, resolve } from "node:path";
44
import { fileURLToPath } from "node:url";
5+
import type { CheckDefinition } from "./check-config.ts";
56
import type { OrgConnection } from "./org-connection.ts";
67
import { childRun, connectionLoad, tokensParse } from "./org-connection.ts";
78
import type { PromotionConfig, PromotionPipeline } from "./promotion.ts";
9+
import type { PromotionGateResult } from "./promotion-gate.ts";
10+
import {
11+
promotionChecksLoad,
12+
promotionGatePlanLine,
13+
promotionGateRun,
14+
} from "./promotion-gate.ts";
815
import {
916
promotionConfigParse,
1017
promotionPlanApply,
@@ -42,6 +49,17 @@ export const APPLIED_PATHS_FILE = "tmp/promotion-applied.txt";
4249

4350
export interface PromotionDeps {
4451
childRun: typeof orgScriptRun;
52+
checkRun: (
53+
check: CheckDefinition,
54+
connection: OrgConnection,
55+
) => Promise<PromotionGateResult>;
56+
}
57+
58+
// Gated orgs' checks, and the orgs whose check already passed in this run.
59+
// A pass stays valid until a transition applies into that org.
60+
interface PromotionGates {
61+
checks: Map<string, CheckDefinition>;
62+
passed: Set<string>;
4563
}
4664

4765
function argumentsParse(args: string[]): PromotionArguments {
@@ -215,6 +233,7 @@ async function transitionRun(
215233
tokens: Map<string, string>,
216234
allowEmptySourceDeletion: boolean,
217235
deps: PromotionDeps,
236+
gates: PromotionGates,
218237
): Promise<boolean> {
219238
const run = deps.childRun;
220239
if (apply) {
@@ -247,7 +266,23 @@ async function transitionRun(
247266
for (const change of plan.changes)
248267
console.log(` ${change.kind.padEnd(6)} ${change.path}`);
249268
if (plan.changes.length === 0) console.log(" no changes");
250-
if (!apply || plan.changes.length === 0) return false;
269+
if (plan.changes.length === 0) return false;
270+
const check = gates.checks.get(transition.source);
271+
if (check && !apply)
272+
console.log(await promotionGatePlanLine(ROOT_DIR, check));
273+
if (!apply) return false;
274+
if (check && !gates.passed.has(transition.source)) {
275+
console.log(` check running ${check.name} in ${transition.source}…`);
276+
const result = await deps.checkRun(
277+
check,
278+
orgConnection(config, transition.source, tokens),
279+
);
280+
if (result.outcome !== "passed")
281+
throw new Error(
282+
`Promotion out of ${transition.source} blocked: check ${check.name} ${result.outcome} (${result.url ?? result.reason})`,
283+
);
284+
gates.passed.add(transition.source);
285+
}
251286
await promotionPlanApply(plan);
252287
const changedPaths = plan.changes.map(
253288
(change) => `resources/${transition.target}/${change.path}`,
@@ -259,18 +294,30 @@ async function transitionRun(
259294
["--force", "--allow-new-files", "--resolve=ours", ...changedPaths],
260295
);
261296
appliedPathsRecord(transition.target);
297+
// The target's files just changed, so an earlier pass no longer covers it.
298+
gates.passed.delete(transition.target);
262299
return plan.changes.some((change) => change.kind === "delete");
263300
}
264301

265302
export async function promotionCommandRun(
266303
args = process.argv.slice(2),
267-
deps: PromotionDeps = { childRun: orgScriptRun },
304+
overrides: Partial<PromotionDeps> = {},
268305
): Promise<void> {
306+
const deps: PromotionDeps = {
307+
childRun: orgScriptRun,
308+
checkRun: (check, connection) =>
309+
promotionGateRun(ROOT_DIR, check, connection),
310+
...overrides,
311+
};
269312
const parsed = argumentsParse(args);
270313
const configPath = resolve(ROOT_DIR, "promotion.yml");
271314
if (!existsSync(configPath))
272315
throw new Error("promotion.yml is required at the repository root");
273316
const config = promotionConfigParse(readFileSync(configPath, "utf8"));
317+
const gates: PromotionGates = {
318+
checks: promotionChecksLoad(ROOT_DIR, config),
319+
passed: new Set(),
320+
};
274321
const tokens = parsed.apply
275322
? tokensParse(TOKENS_ENV)
276323
: new Map<string, string>();
@@ -292,6 +339,7 @@ export async function promotionCommandRun(
292339
tokens,
293340
deletionAuthorizedSources.has(sourceKey),
294341
deps,
342+
gates,
295343
);
296344
if (deleted)
297345
deletionAuthorizedSources.add(

‎src/promotion-gate.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// The promotion check gate: `orgs.<slug>.check: <name>` in promotion.yml
2+
// names a vapi-checks.yml check that must pass in that org before any
3+
// transition promotes out of it. The gate runs the same check as the PR
4+
// workflow, built from the source org's files at the promoted commit.
5+
6+
import type { CheckDefinition } from "./check-config.ts";
7+
import { CHECKS_CONFIG_FILE, checksConfigLoad } from "./check-config.ts";
8+
import { checkJobsBuild } from "./check-build.ts";
9+
import type { CheckOutcome } from "./check-run.ts";
10+
import { checkRunAll, MIN_START_MS } from "./check-run.ts";
11+
import type { OrgConnection } from "./org-connection.ts";
12+
import type { PromotionConfig } from "./promotion.ts";
13+
import { userAgentGet } from "./user-agent.ts";
14+
15+
export interface PromotionGateResult {
16+
outcome: CheckOutcome;
17+
reason: string;
18+
url?: string;
19+
}
20+
21+
const DEFAULT_BASE_URL = "https://api.vapi.ai";
22+
const OUTCOME_RANK: Record<CheckOutcome, number> = {
23+
passed: 0,
24+
built: 0,
25+
incomplete: 1,
26+
failed: 2,
27+
error: 3,
28+
};
29+
30+
// The gated orgs' checks, validated up front so a typo fails before any
31+
// transition applies.
32+
export function promotionChecksLoad(
33+
rootDir: string,
34+
config: PromotionConfig,
35+
): Map<string, CheckDefinition> {
36+
const gated = Object.entries(config.orgs).filter(([, org]) => org.check);
37+
const checks = new Map<string, CheckDefinition>();
38+
if (gated.length === 0) return checks;
39+
const checksConfig = checksConfigLoad(rootDir);
40+
if (!checksConfig)
41+
throw new Error(
42+
`promotion.yml gates ${gated.map(([slug]) => slug).join(", ")} on checks, but there is no ${CHECKS_CONFIG_FILE}`,
43+
);
44+
for (const [slug, org] of gated) {
45+
const check = checksConfig.checks[org.check!];
46+
if (!check)
47+
throw new Error(
48+
`orgs.${slug}.check: no check named ${org.check} in ${CHECKS_CONFIG_FILE}`,
49+
);
50+
if (check.org !== slug || check.runOrg !== slug)
51+
throw new Error(
52+
`orgs.${slug}.check: check ${check.name} must read and run in ${slug} (it reads ${check.org} and runs in ${check.runOrg})`,
53+
);
54+
checks.set(slug, check);
55+
}
56+
return checks;
57+
}
58+
59+
// The plan-only line: what the gate would run, built offline.
60+
export async function promotionGatePlanLine(
61+
rootDir: string,
62+
check: CheckDefinition,
63+
): Promise<string> {
64+
const jobs = await checkJobsBuild(rootDir, check);
65+
const broken = jobs.find((job) => !job.result.body);
66+
if (broken)
67+
return ` check would run ${check.name} in ${check.org}, but its payload doesn't build: ${broken.result.errors[0]}`;
68+
const simulations = jobs[0]?.result.body?.simulations.length ?? 0;
69+
return ` check would run ${check.name} in ${check.org} (${simulations} simulation${simulations === 1 ? "" : "s"} × ${jobs.length} target${jobs.length === 1 ? "" : "s"})`;
70+
}
71+
72+
// Run the check live in the source org and reduce it to one result: the
73+
// worst target wins, so anything short of every target passing blocks.
74+
export async function promotionGateRun(
75+
rootDir: string,
76+
check: CheckDefinition,
77+
connection: OrgConnection,
78+
): Promise<PromotionGateResult> {
79+
const jobs = await checkJobsBuild(rootDir, check);
80+
const controller = new AbortController();
81+
const abort = () => controller.abort();
82+
process.once("SIGINT", abort);
83+
process.once("SIGTERM", abort);
84+
try {
85+
const results = await checkRunAll({
86+
jobs,
87+
connectionFor: () => ({
88+
token: connection.token,
89+
baseUrl: check.baseUrl ?? connection.baseUrl ?? DEFAULT_BASE_URL,
90+
userAgent: userAgentGet("check"),
91+
}),
92+
deadline: Date.now() + check.timeoutMinutes * 60_000 + MIN_START_MS,
93+
signal: controller.signal,
94+
});
95+
const worst = results.reduce((a, b) =>
96+
OUTCOME_RANK[b.outcome] > OUTCOME_RANK[a.outcome] ? b : a,
97+
);
98+
for (const result of results)
99+
console.log(
100+
` check ${result.job.label}: ${result.outcome} — ${result.reason}${result.url ? ` (${result.url})` : ""}`,
101+
);
102+
return { outcome: worst.outcome, reason: worst.reason, url: worst.url };
103+
} finally {
104+
process.off("SIGINT", abort);
105+
process.off("SIGTERM", abort);
106+
}
107+
}

‎src/promotion.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ export interface PromotionBindings {
2929
export interface PromotionOrg {
3030
baseUrl?: string;
3131
bindings: PromotionBindings;
32+
// A vapi-checks.yml check that must pass in this org before anything is
33+
// promoted out of it.
34+
check?: string;
3235
}
3336

3437
export interface PromotionPipeline {
@@ -152,9 +155,17 @@ export function promotionConfigParse(content: string): PromotionConfig {
152155
const org = object(value ?? {}, `org ${slug}`);
153156
if (org.baseUrl !== undefined && typeof org.baseUrl !== "string")
154157
throw new Error(`org ${slug}.baseUrl must be a string`);
158+
if (
159+
org.check !== undefined &&
160+
(typeof org.check !== "string" || !SLUG_RE.test(org.check))
161+
)
162+
throw new Error(
163+
`org ${slug}.check must be a check name from vapi-checks.yml`,
164+
);
155165
orgs[slug] = {
156166
baseUrl: typeof org.baseUrl === "string" ? org.baseUrl : undefined,
157167
bindings: promotionBindingsParse(org.bindings),
168+
...(typeof org.check === "string" ? { check: org.check } : {}),
158169
};
159170
}
160171
const pipelinesRaw = object(raw.pipelines, "promotion.yml pipelines");

0 commit comments

Comments
 (0)