feat(check): run checks live and report Vapi Evals statuses

`npm run check -- <check>|--all` now runs each built target payload in
the check's run org: one POST /eval/simulation/run per target, at most
three at once, judged by the same strict verdict as `npm run sim`
(the create/poll/cancel/hydrate loop moves into simRunExecute, which
both share).

- Budget and cancellation: each run's deadline is the earlier of its
  timeoutMinutes and --budget-minutes; no run starts with under five
  minutes left; the deadline, SIGINT and SIGTERM cancel in-flight runs.
- Keys: VAPI_CHECK_TOKENS, then .env.<runOrg>, then VAPI_PRIVATE_API_KEY
  when every selected check runs in one org. --refresh-bindings runs the
  same read-only bindings pull promotion uses.
- Selection: --changed-since <ref> runs only checks affected by changes
  since the merge base (the org's files and state, the run org's, the
  config, the engine, and the check's own paths).
- Statuses (when GITHUB_TOKEN, GITHUB_REPOSITORY and HEAD_SHA are set):
  `Vapi Evals / <check> / <target>` goes pending with the run's link,
  then success/failure/error; --all runs post the aggregate `Vapi Evals`
  from a finally block, so config errors and exceptions report too.
- Report: a markdown summary (also appended to $GITHUB_STEP_SUMMARY) with
  failing evaluations' expected vs extracted values and "unmocked tool
  called" notices from transcripts, plus --json. No PR comments.
- Runs send User-Agent vapi-gitops-check/<version>.

Exit codes: 0 passed, 1 failed, 2 config or build error, 3 incomplete.

Refs TEST-141

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: scott-lowe-vapi

AGENTS.md

@@ -90,7 +90,7 @@ If setup reports the org is "already set up locally", do not delete anything to
 | Push with new resources             | `npm run push -- <org> --allow-new-files` — bypass orphan-YAML gate. **AI agents**: do NOT auto-pass this flag; confirm with the human first (see push section below) |
 | Test a call                         | `npm run call -- <org> -a <assistant-name>` or `-s <squad-name>`                  |
 | Run a simulation suite              | `npm run sim -- <org> --suite <name> --target <assistant-name>`                   |
-| Build PR-check payloads offline     | `npm run check -- <check> --dry-run` (or `--all`); `--print-payload` writes the JSON |
+| Run PR simulation checks            | `npm run check -- <check>` (or `--all`); `--dry-run` builds the payloads offline, `--print-payload` writes them |
 | Migrate a legacy state file         | `npm run migrate` — one-shot, all orgs; required once after upgrading to the hash-store engine |
 
 ---
@@ -904,7 +904,8 @@ npm run validate -- <org>                          # Lint resources locally (fai
 npm run audit -- <org>                             # Read-only drift detector: orphan YAML, state ghosts, content-identical clusters, sibling base-slugs, dashboard orphans, inline model.tools. Exit 1 on findings.
 npm run audit -- <org> --type assistants           # Scope audit to a single resource type
 npm run sim -- <org> --suite <name> --target <name>  # Run a simulation suite against an assistant/squad (exit 0 pass, 1 fail, 3 incomplete; --timeout <min>)
-npm run check -- <check> --dry-run                 # Build a vapi-checks.yml check's inline run payload offline (exit 0 built, 2 config/build error; --print-payload [dir])
+npm run check -- <check>                          # Run a vapi-checks.yml check inline against the local files (exit 0 pass, 1 fail, 2 config/build error, 3 incomplete; --changed-since <ref>, --budget-minutes <n>, --json <path>)
+npm run check -- <check> --dry-run                # Build the check's inline run payloads offline, no key needed (--print-payload [dir])
 npm run rollback -- <org> --to <ISO-timestamp>     # Re-apply a snapshot taken before a push
 npm run rollback -- <org> --list                   # List available snapshots
 

README.md

@@ -104,7 +104,7 @@ Every command works in two modes:
 | `npm run rollback` | — | `npm run rollback -- <org> --list` or `--to <ISO>` | Restore from a snapshot in `.vapi-state.<org>.snapshots/` (one is written before every push/apply). |
 | `npm run call` | ✅ | `npm run call -- <org> -a <name>` or `-s <squad>` | Start an interactive WebSocket call against an assistant or squad. |
 | `npm run sim` | — | `npm run sim -- <org> --suite <name> --target <name> [--timeout <min>]` | Run a simulation suite (or specific simulations) against a deployed assistant/squad. Prints the run link; exits 0 passed, 1 failed, 3 incomplete (timeout, Ctrl-C, missing results). |
-| `npm run check` | — | `npm run check -- <check>\|--all --dry-run [--print-payload [dir]]` | Build each `vapi-checks.yml` check target's inline simulation-run payload from the files on disk, with its tools, handoffs, judges and personalities. Offline: no API key, nothing sent. `--print-payload` writes the JSON (default `tmp/check-payloads/`). Exits 0 when every payload builds, 2 on a config or build error. |
+| `npm run check` | — | `npm run check -- <check>\|--all [--dry-run] [--changed-since <ref>] [--budget-minutes <n>] [--json <path>]` | Run the `vapi-checks.yml` simulation checks against the files on disk: each target is built inline (tools, handoffs, judges, personalities; tools mocked fail-closed, servers dead-ended) and run in one simulation run per target, so nothing is deployed. Posts `Vapi Evals` commit statuses when run by the PR workflow. `--dry-run` builds the payloads offline (no key, nothing sent; `--print-payload` writes them). Exits 0 passed, 1 failed, 2 config or build error, 3 incomplete (timeout, interrupt, budget, billing). |
 | `npm run migrate` | — | `npm run migrate` | One-time, all orgs at once: slim legacy state files to pure `name → uuid` and seed the per-developer `.vapi-state-hash/` baseline store from the old hashes. Required once after upgrading to the hash-store engine — `pull`/`push`/`apply` refuse legacy-shaped state until it runs. Idempotent. |
 | `npm run build` | — | — | Type-check the codebase (`tsc --noEmit`). |
 | `npm test` | — | — | Run regression tests (`node:test`). |

src/check-build.ts

@@ -0,0 +1,80 @@
+// Build every target payload of one check from the files under rootDir.
+// Shared by the dry run and the live run, so both test the same body.
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { CheckDefinition, CheckTarget } from "./check-config.ts";
+import type { CheckPayloadResult } from "./check-payload.ts";
+import { checkPayloadBuild } from "./check-payload.ts";
+import { promotionBindingsResolve, promotionStateParse } from "./promotion.ts";
+import { orgResourcesRead } from "./resource-parse.ts";
+import type { StateFile } from "./types.ts";
+
+export interface CheckJob {
+  check: CheckDefinition;
+  target: CheckTarget;
+  // `<check> / <type>/<id>`
+  label: string;
+  result: CheckPayloadResult;
+}
+
+export function checkTargetLabel(target: CheckTarget): string {
+  return `${target.type}/${target.id}`;
+}
+
+function stateRead(root: string, org: string, warnings: string[]): StateFile {
+  const path = join(root, `.vapi-state.${org}.json`);
+  if (!existsSync(path)) {
+    warnings.push(
+      `no .vapi-state.${org}.json: references by UUID and credential bindings can't resolve`,
+    );
+    return promotionStateParse("{}");
+  }
+  return promotionStateParse(readFileSync(path, "utf8"));
+}
+
+export async function checkJobsBuild(
+  root: string,
+  check: CheckDefinition,
+): Promise<CheckJob[]> {
+  const job = (target: CheckTarget, result: CheckPayloadResult): CheckJob => ({
+    check,
+    target,
+    label: `${check.name} / ${checkTargetLabel(target)}`,
+    result,
+  });
+  if (!existsSync(join(root, "resources", check.org))) {
+    return check.targets.map((target) =>
+      job(target, {
+        errors: [`resources/${check.org}/ does not exist`],
+        warnings: [],
+        bytes: 0,
+      }),
+    );
+  }
+  const warnings: string[] = [];
+  const resources = await orgResourcesRead(root, check.org);
+  const sourceState = stateRead(root, check.org, warnings);
+  const runState =
+    check.runOrg === check.org
+      ? sourceState
+      : stateRead(root, check.runOrg, warnings);
+  const bindingsResolved = await promotionBindingsResolve(
+    root,
+    check.org,
+    check.runOrg,
+    sourceState,
+  );
+  return check.targets.map((target) => {
+    const result = checkPayloadBuild({
+      check,
+      target,
+      resources,
+      sourceState,
+      runState,
+      bindingsResolved,
+    });
+    result.warnings.unshift(...warnings);
+    return job(target, result);
+  });
+}