|
| 1 | +// The promotion check gate: `orgs.<slug>.check: <name>` in promotion.yml |
| 2 | +// names a vapi-checks.yml check that must pass in that org before any |
| 3 | +// transition promotes out of it. The gate runs the same check as the PR |
| 4 | +// workflow, built from the source org's files at the promoted commit. |
| 5 | + |
| 6 | +import type { CheckDefinition } from "./check-config.ts"; |
| 7 | +import { CHECKS_CONFIG_FILE, checksConfigLoad } from "./check-config.ts"; |
| 8 | +import { checkJobsBuild } from "./check-build.ts"; |
| 9 | +import type { CheckOutcome, CheckTargetResult } from "./check-run.ts"; |
| 10 | +import { checkRunAll, MAX_CONCURRENT, MIN_START_MS } from "./check-run.ts"; |
| 11 | +import type { OrgConnection } from "./org-connection.ts"; |
| 12 | +import type { PromotionConfig } from "./promotion.ts"; |
| 13 | +import { userAgentGet } from "./user-agent.ts"; |
| 14 | + |
| 15 | +export interface PromotionGateResult { |
| 16 | + outcome: CheckOutcome; |
| 17 | + reason: string; |
| 18 | + url?: string; |
| 19 | +} |
| 20 | + |
| 21 | +const DEFAULT_BASE_URL = "https://api.vapi.ai"; |
| 22 | +// Gate checks must fit well inside the promotion step's timeout, so a check |
| 23 | +// that can't finish is rejected when the config loads, not killed mid-run. |
| 24 | +export const GATE_BUDGET_MINUTES = 300; |
| 25 | +const OUTCOME_RANK: Record<CheckOutcome, number> = { |
| 26 | + passed: 0, |
| 27 | + built: 0, |
| 28 | + incomplete: 1, |
| 29 | + failed: 2, |
| 30 | + error: 3, |
| 31 | +}; |
| 32 | + |
| 33 | +// The gated orgs' checks, validated up front so a typo fails before any |
| 34 | +// transition applies. |
| 35 | +export function promotionChecksLoad( |
| 36 | + rootDir: string, |
| 37 | + config: PromotionConfig, |
| 38 | +): Map<string, CheckDefinition> { |
| 39 | + const gated = Object.entries(config.orgs).filter(([, org]) => org.check); |
| 40 | + const checks = new Map<string, CheckDefinition>(); |
| 41 | + if (gated.length === 0) return checks; |
| 42 | + const checksConfig = checksConfigLoad(rootDir); |
| 43 | + if (!checksConfig) |
| 44 | + throw new Error( |
| 45 | + `promotion.yml gates ${gated.map(([slug]) => slug).join(", ")} on checks, but there is no ${CHECKS_CONFIG_FILE}`, |
| 46 | + ); |
| 47 | + // An org that is last in every pipeline is never promoted out of. |
| 48 | + const sources = new Set( |
| 49 | + Object.values(config.pipelines).flatMap((pipeline) => |
| 50 | + pipeline.orgs.slice(0, -1), |
| 51 | + ), |
| 52 | + ); |
| 53 | + let budgetMinutes = 0; |
| 54 | + for (const [slug, org] of gated) { |
| 55 | + if (!sources.has(slug)) |
| 56 | + throw new Error( |
| 57 | + `orgs.${slug}.check: nothing is promoted out of ${slug} (it is last in every pipeline), so this check would never run; gate the org before it instead`, |
| 58 | + ); |
| 59 | + const check = checksConfig.checks[org.check!]; |
| 60 | + if (!check) |
| 61 | + throw new Error( |
| 62 | + `orgs.${slug}.check: no check named ${org.check} in ${CHECKS_CONFIG_FILE}`, |
| 63 | + ); |
| 64 | + if (check.org !== slug || check.runOrg !== slug) |
| 65 | + throw new Error( |
| 66 | + `orgs.${slug}.check: check ${check.name} must read and run in ${slug} (it reads ${check.org} and runs in ${check.runOrg})`, |
| 67 | + ); |
| 68 | + // A gate runs in the real org, never a CI org, so it must not reach real |
| 69 | + // systems: no live tools, and no webhooks to the org's own servers. |
| 70 | + if (check.toolMocks === "off") |
| 71 | + throw new Error( |
| 72 | + `orgs.${slug}.check: check ${check.name} sets toolMocks: off, which runs real tools; a gate runs in ${slug} itself, so it must use toolMocks: strict`, |
| 73 | + ); |
| 74 | + if (!check.stripWebhooks) |
| 75 | + throw new Error( |
| 76 | + `orgs.${slug}.check: check ${check.name} sets stripWebhooks: false, which sends simulated calls' webhooks to ${slug}'s real servers; a gate must keep the default`, |
| 77 | + ); |
| 78 | + // The org's token goes only to the host promotion uses for that org. |
| 79 | + if (check.baseUrl && check.baseUrl !== org.baseUrl?.replace(/\/+$/, "")) |
| 80 | + throw new Error( |
| 81 | + `orgs.${slug}.check: check ${check.name} uses ${check.baseUrl}, but promotion.yml uses ${org.baseUrl ?? "the default API"} for ${slug}; set the same baseUrl in both`, |
| 82 | + ); |
| 83 | + budgetMinutes += gateBudgetMinutes(check); |
| 84 | + checks.set(slug, check); |
| 85 | + } |
| 86 | + if (budgetMinutes > GATE_BUDGET_MINUTES) |
| 87 | + throw new Error( |
| 88 | + `promotion.yml's gated checks can take up to ${budgetMinutes} minutes in one run, more than the ${GATE_BUDGET_MINUTES} the promotion step allows; lower their timeoutMinutes or targets`, |
| 89 | + ); |
| 90 | + return checks; |
| 91 | +} |
| 92 | + |
| 93 | +// The longest a gate check can run: targets run MAX_CONCURRENT at a time, |
| 94 | +// and each batch gets a full timeoutMinutes. |
| 95 | +export function gateBudgetMinutes( |
| 96 | + check: Pick<CheckDefinition, "timeoutMinutes"> & { |
| 97 | + targets: readonly unknown[]; |
| 98 | + }, |
| 99 | +): number { |
| 100 | + return ( |
| 101 | + Math.ceil(check.targets.length / MAX_CONCURRENT) * check.timeoutMinutes + |
| 102 | + MIN_START_MS / 60_000 |
| 103 | + ); |
| 104 | +} |
| 105 | + |
| 106 | +export function gateDeadline( |
| 107 | + check: Pick<CheckDefinition, "timeoutMinutes"> & { |
| 108 | + targets: readonly unknown[]; |
| 109 | + }, |
| 110 | + now: number, |
| 111 | +): number { |
| 112 | + return now + gateBudgetMinutes(check) * 60_000; |
| 113 | +} |
| 114 | + |
| 115 | +// Anything short of every target passing blocks: the worst target wins. |
| 116 | +export function gateResultReduce( |
| 117 | + results: Array<Pick<CheckTargetResult, "outcome" | "reason" | "url">>, |
| 118 | +): PromotionGateResult { |
| 119 | + const worst = results.reduce((a, b) => |
| 120 | + OUTCOME_RANK[b.outcome] > OUTCOME_RANK[a.outcome] ? b : a, |
| 121 | + ); |
| 122 | + return { outcome: worst.outcome, reason: worst.reason, url: worst.url }; |
| 123 | +} |
| 124 | + |
| 125 | +// The plan-only line: what the gate would run, built offline. |
| 126 | +export async function promotionGatePlanLine( |
| 127 | + rootDir: string, |
| 128 | + check: CheckDefinition, |
| 129 | +): Promise<string> { |
| 130 | + const jobs = await checkJobsBuild(rootDir, check); |
| 131 | + const broken = jobs.find((job) => !job.result.body); |
| 132 | + if (broken) |
| 133 | + return ` check would run ${check.name} in ${check.org}, but its payload doesn't build: ${broken.result.errors[0]}`; |
| 134 | + const simulations = jobs[0]?.result.body?.simulations.length ?? 0; |
| 135 | + return ` check would run ${check.name} in ${check.org} (${simulations} simulation${simulations === 1 ? "" : "s"} × ${jobs.length} target${jobs.length === 1 ? "" : "s"})`; |
| 136 | +} |
| 137 | + |
| 138 | +// Run the check live in the source org and reduce it to one result: the |
| 139 | +// worst target wins, so anything short of every target passing blocks. |
| 140 | +export async function promotionGateRun( |
| 141 | + rootDir: string, |
| 142 | + check: CheckDefinition, |
| 143 | + connection: OrgConnection, |
| 144 | +): Promise<PromotionGateResult> { |
| 145 | + const jobs = await checkJobsBuild(rootDir, check); |
| 146 | + const controller = new AbortController(); |
| 147 | + const abort = () => controller.abort(); |
| 148 | + process.once("SIGINT", abort); |
| 149 | + process.once("SIGTERM", abort); |
| 150 | + try { |
| 151 | + const results = await checkRunAll({ |
| 152 | + jobs, |
| 153 | + connectionFor: () => ({ |
| 154 | + token: connection.token, |
| 155 | + baseUrl: connection.baseUrl ?? DEFAULT_BASE_URL, |
| 156 | + userAgent: userAgentGet("check"), |
| 157 | + }), |
| 158 | + deadline: gateDeadline(check, Date.now()), |
| 159 | + signal: controller.signal, |
| 160 | + }); |
| 161 | + for (const result of results) |
| 162 | + console.log( |
| 163 | + ` check ${result.job.label}: ${result.outcome} — ${result.reason}${result.url ? ` (${result.url})` : ""}`, |
| 164 | + ); |
| 165 | + return gateResultReduce(results); |
| 166 | + } finally { |
| 167 | + process.off("SIGINT", abort); |
| 168 | + process.off("SIGTERM", abort); |
| 169 | + } |
| 170 | +} |
0 commit comments