From 1181f6c27ded6da30d4acdc0bb70dbdc358af7a3 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Fri, 26 Jun 2026 12:47:40 -0300
Subject: [PATCH 01/10] feat(ci): post-deploy verification for managed and
 enterprise orgs

A separate pipeline that runs after the entire prod CI Pipeline finishes
green (workflow_run) and confirms managed and enterprise organizations are
executing workflows correctly on the just-shipped build.

- new HMAC-authed internal endpoint reads recent execution outcomes for the
  managed slugs plus enterprise-plan orgs and flags error-rate regressions
- runs in-cluster (internal routes are WAF-blocked on the public host),
  signing like digest-cron.sh and curling the in-pod service URL
- per-org detail goes to internal observability only; public CI logs and the
  chat alert carry aggregate counts so customer data is not exposed
- extract the managed-client slug list into a shared dependency-free module
---
 .../workflows/post-deploy-verification.yml    | 176 ++++++++++
 .../post-deploy-verification/route.ts         | 327 ++++++++++++++++++
 lib/metrics/db-metrics.ts                     |  14 +-
 lib/orgs/managed-clients.ts                   |  15 +
 4 files changed, 525 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/post-deploy-verification.yml
 create mode 100644 app/api/internal/post-deploy-verification/route.ts
 create mode 100644 lib/orgs/managed-clients.ts

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
new file mode 100644
index 000000000..0c12dfefa
--- /dev/null
+++ b/.github/workflows/post-deploy-verification.yml
@@ -0,0 +1,176 @@
+# Type: standalone | Triggered by the CI Pipeline finishing successfully.
+#
+# This is a SEPARATE pipeline from the deploy. It fires only after the entire
+# "CI Pipeline" run (build -> e2e -> deploy -> release -> docs-sync) completes
+# with conclusion == success on prod -- not after any single step. It then
+# verifies that managed + enterprise orgs are executing workflows correctly on
+# the just-shipped build.
+#
+# It runs INSIDE the cluster: the /api/internal/* routes are HMAC-authed and
+# WAF-blocked through the public hostname, so it signs the request the same way
+# deploy/scripts/digest-cron.sh does and curls the in-pod service URL via a
+# one-shot pod -- mirroring the deploy health check.
+name: post-deploy-verification
+
+on:
+  workflow_run:
+    workflows: ["CI Pipeline"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  id-token: write
+
+# One verification per target at a time. cancel-in-progress is false so a second
+# pipeline's verification queues rather than killing an in-flight check.
+concurrency:
+  group: post-deploy-verification-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    # Only when the whole CI Pipeline went green on prod. A failed/cancelled CI
+    # run, or a successful run on any other branch, is skipped entirely.
+    if: >
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_branch == 'prod'
+    runs-on: ubuntu-latest
+    environment: prod
+    env:
+      REGION: ${{ vars.TO_REGION }}
+      CLUSTER_NAME: ${{ vars.TO_CLUSTER_NAME }}
+      NAMESPACE: keeperhub
+      SERVICE_NAME: keeperhub
+      SSM_ENV_PREFIX: techops-prod
+      # Window and thresholds the verification reads recent executions against.
+      # An org is flagged when it has any system_error, or when it ran at least
+      # MIN_EXECUTIONS and its error rate exceeds MAX_ERROR_RATE over the window.
+      LOOKBACK_MINUTES: "60"
+      MIN_EXECUTIONS: "1"
+      MAX_ERROR_RATE: "0.5"
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ env.REGION }}
+
+      - name: Configure kubectl
+        run: |
+          aws eks update-kubeconfig --name "${CLUSTER_NAME}" --region "${REGION}"
+
+      - name: Fetch internal-service HMAC secret
+        run: |
+          SECRET=$(aws ssm get-parameter \
+            --name "/eks/${SSM_ENV_PREFIX}/keeperhub/internal-service-hmac-secret" \
+            --with-decryption --query "Parameter.Value" --output text \
+            --region "${REGION}")
+          echo "::add-mask::$SECRET"
+          echo "INTERNAL_SERVICE_HMAC_SECRET=$SECRET" >> "$GITHUB_ENV"
+
+      - name: Run post-deploy verification (in-cluster)
+        id: verify
+        run: |
+          set -euo pipefail
+
+          CALLER="scheduler"
+          METHOD="GET"
+          PATHNAME="/api/internal/post-deploy-verification"
+          QUERY="lookbackMinutes=${LOOKBACK_MINUTES}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
+          TIMESTAMP=$(date +%s)
+
+          # The verifier signs over url.pathname only (no query string), so the
+          # thresholds ride in the query without affecting the signature.
+          BODY_DIGEST=$(printf '' | openssl dgst -sha256 | awk '{print $2}')
+          SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP")
+          SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}')
+
+          URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?${QUERY}"
+          echo "Verifying ${PATHNAME} over a ${LOOKBACK_MINUTES}m window (in-cluster)..."
+
+          # The secret never enters the pod -- only the precomputed signature does.
+          RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
+            --namespace "${NAMESPACE}" \
+            --restart=Never \
+            --attach \
+            --rm \
+            --quiet \
+            --image=curlimages/curl:8.11.0 \
+            --env="URL=${URL}" \
+            --env="CALLER=${CALLER}" \
+            --env="TIMESTAMP=${TIMESTAMP}" \
+            --env="SIGNATURE=${SIGNATURE}" \
+            --command -- sh -c '
+              curl -sS --max-time 30 \
+                -H "X-KH-Caller: ${CALLER}" \
+                -H "X-KH-Timestamp: ${TIMESTAMP}" \
+                -H "X-KH-Signature: ${SIGNATURE}" \
+                "$URL"
+            ')
+
+          echo "$RESPONSE" > /tmp/verification.json
+
+          # This is a PUBLIC repo: Actions logs are world-readable. Surface only
+          # non-identifying aggregates here. Per-org slugs, names, and sample
+          # error text stay in the app's internal logs (Loki) -- never printed to
+          # CI or sent to Discord. The raw endpoint error (which could carry
+          # sensitive detail) is also never echoed.
+          OK=$(jq -r '.ok // false' /tmp/verification.json 2>/dev/null || echo false)
+          CHECKED=$(jq -r '.checkedOrgs // "?"' /tmp/verification.json 2>/dev/null || echo "?")
+          PROBLEMS=$(jq -r '.problemCount // "?"' /tmp/verification.json 2>/dev/null || echo "?")
+          echo "Verification: ok=${OK}, checkedOrgs=${CHECKED}, problemCount=${PROBLEMS}"
+
+          {
+            echo "ok=$OK"
+            echo "checked=$CHECKED"
+            echo "problems=$PROBLEMS"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Alert and fail on verification problems
+        if: always() && steps.verify.outputs.ok != 'true'
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_DEPLOY_ALERTS }}
+          CHECKED: ${{ steps.verify.outputs.checked }}
+          PROBLEMS: ${{ steps.verify.outputs.problems }}
+        run: |
+          REPO_URL="https://github.com/${{ github.repository }}"
+          RUN_URL="${REPO_URL}/actions/runs/${{ github.run_id }}"
+
+          # Aggregate-only message. No org slugs/names/error text -- those live in
+          # internal logs (Loki: "Post-deploy verification flagged ..."). Safe to
+          # post to a chat channel without leaking customer data.
+          if [ -n "${PROBLEMS:-}" ] && [ "${PROBLEMS}" != "?" ]; then
+            SUMMARY="Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) erroring on the new build. Details are in internal logs (search: post-deploy verification flagged)."
+          else
+            SUMMARY="Post-deploy verification could not complete. See the Actions run and internal logs."
+          fi
+
+          echo "$SUMMARY"
+
+          if [ -n "${DISCORD_WEBHOOK:-}" ]; then
+            jq -n \
+              --arg title "Post-deploy verification failed" \
+              --arg url "$RUN_URL" \
+              --arg desc "$SUMMARY" \
+              '{
+                username: "KeeperHub Deploy Verifier",
+                embeds: [{
+                  title: $title,
+                  url: $url,
+                  description: $desc,
+                  color: 15548997
+                }]
+              }' > /tmp/discord-payload.json
+
+            HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+              -H "Content-Type: application/json" \
+              -d @/tmp/discord-payload.json \
+              "$DISCORD_WEBHOOK")
+            echo "Discord notification HTTP ${HTTP_STATUS}"
+          else
+            echo "DISCORD_WEBHOOK_DEPLOY_ALERTS not set; skipping Discord notification (failure still surfaces via the red job and internal logs)."
+          fi
+
+          echo "::error::Post-deploy verification flagged managed/enterprise org errors (count only; see internal logs for specifics)."
+          exit 1
diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
new file mode 100644
index 000000000..284c0eaed
--- /dev/null
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -0,0 +1,327 @@
+/**
+ * Post-deployment verification for managed and enterprise organizations.
+ *
+ * Reads recent workflow execution outcomes for the white-glove cohort
+ * (MANAGED_ORG_SLUGS plus any org on an `enterprise` subscription) and reports
+ * whether any of them are erroring after a deploy. Authenticated as an internal
+ * service (HMAC) and invoked from the post-deploy-verification CI job, which
+ * runs inside the cluster against the in-pod service URL (the public hostname
+ * is WAF-blocked for /api/internal/*).
+ *
+ * An org is flagged as a problem when:
+ *   - it produced any `system_error` executions in the window (platform/infra
+ *     faults are the strongest signal of a deploy regression), OR
+ *   - it ran at least `minExecutions` and its error rate exceeds `maxErrorRate`.
+ *
+ * Window and thresholds are query-param overridable so the CI job can tune them
+ * without a redeploy: `lookbackMinutes`, `minExecutions`, `maxErrorRate`.
+ */
+import { and, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { db } from "@/lib/db";
+import {
+  organization,
+  organizationSubscriptions,
+  workflowExecutions,
+  workflows,
+} from "@/lib/db/schema";
+import { ERROR_STATUSES } from "@/lib/errors/execution-status";
+import { HttpStatus } from "@/lib/http-status";
+import { authenticateInternalService } from "@/lib/internal-service-auth";
+import { ErrorCategory, logSystemError, logSystemWarn } from "@/lib/logging";
+import { MANAGED_ORG_SLUGS } from "@/lib/orgs/managed-clients";
+
+const DEFAULT_LOOKBACK_MINUTES = 60;
+const DEFAULT_MIN_EXECUTIONS = 1;
+const DEFAULT_MAX_ERROR_RATE = 0.5;
+const SAMPLE_ERRORS_PER_ORG = 3;
+const MAX_SAMPLE_ERROR_LENGTH = 300;
+
+type TargetOrg = {
+  id: string;
+  slug: string;
+  name: string;
+  plan: string;
+};
+
+type OrgVerification = {
+  organizationId: string;
+  slug: string;
+  name: string;
+  plan: string;
+  total: number;
+  success: number;
+  userErrors: number;
+  systemErrors: number;
+  errorRate: number;
+  isProblem: boolean;
+  reasons: string[];
+  sampleErrors: string[];
+};
+
+function parseNumberParam(value: string | null, fallback: number): number {
+  if (!value) {
+    return fallback;
+  }
+  const parsed = Number.parseFloat(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+async function resolveTargetOrgs(): Promise<TargetOrg[]> {
+  const rows = await db
+    .select({
+      id: organization.id,
+      slug: organization.slug,
+      name: organization.name,
+      plan: organizationSubscriptions.plan,
+    })
+    .from(organization)
+    .leftJoin(
+      organizationSubscriptions,
+      eq(organizationSubscriptions.organizationId, organization.id)
+    )
+    .where(
+      and(
+        isNull(organization.deactivatedAt),
+        or(
+          inArray(organization.slug, [...MANAGED_ORG_SLUGS]),
+          eq(organizationSubscriptions.plan, "enterprise")
+        )
+      )
+    );
+
+  return rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    name: row.name,
+    plan: row.plan ?? "free",
+  }));
+}
+
+async function fetchSampleErrors(
+  problemOrgIds: string[],
+  since: Date
+): Promise<Map<string, string[]>> {
+  const byOrg = new Map<string, string[]>();
+  if (problemOrgIds.length === 0) {
+    return byOrg;
+  }
+
+  const rows = await db
+    .select({
+      organizationId: workflows.organizationId,
+      error: workflowExecutions.error,
+      status: workflowExecutions.status,
+    })
+    .from(workflowExecutions)
+    .innerJoin(workflows, eq(workflows.id, workflowExecutions.workflowId))
+    .where(
+      and(
+        inArray(workflows.organizationId, problemOrgIds),
+        gte(workflowExecutions.startedAt, since),
+        inArray(workflowExecutions.status, [...ERROR_STATUSES])
+      )
+    )
+    .orderBy(desc(workflowExecutions.startedAt))
+    .limit(problemOrgIds.length * SAMPLE_ERRORS_PER_ORG * 4);
+
+  for (const row of rows) {
+    const orgId = row.organizationId;
+    if (!orgId) {
+      continue;
+    }
+    const existing = byOrg.get(orgId) ?? [];
+    if (existing.length >= SAMPLE_ERRORS_PER_ORG) {
+      continue;
+    }
+    const message = (row.error ?? "(no error message)").slice(
+      0,
+      MAX_SAMPLE_ERROR_LENGTH
+    );
+    existing.push(`[${row.status}] ${message}`);
+    byOrg.set(orgId, existing);
+  }
+
+  return byOrg;
+}
+
+export async function GET(request: Request): Promise<NextResponse> {
+  const auth = await authenticateInternalService(request);
+  if (!auth.authenticated) {
+    return NextResponse.json(
+      { error: auth.error ?? "Unauthorized" },
+      { status: auth.status }
+    );
+  }
+
+  const url = new URL(request.url);
+  const lookbackMinutes = parseNumberParam(
+    url.searchParams.get("lookbackMinutes"),
+    DEFAULT_LOOKBACK_MINUTES
+  );
+  const minExecutions = parseNumberParam(
+    url.searchParams.get("minExecutions"),
+    DEFAULT_MIN_EXECUTIONS
+  );
+  const maxErrorRate = parseNumberParam(
+    url.searchParams.get("maxErrorRate"),
+    DEFAULT_MAX_ERROR_RATE
+  );
+
+  try {
+    const since = new Date(Date.now() - lookbackMinutes * 60 * 1000);
+    const targetOrgs = await resolveTargetOrgs();
+
+    if (targetOrgs.length === 0) {
+      return NextResponse.json({
+        ok: true,
+        lookbackMinutes,
+        minExecutions,
+        maxErrorRate,
+        checkedOrgs: 0,
+        problemCount: 0,
+        orgs: [],
+        problems: [],
+        generatedAt: new Date().toISOString(),
+      });
+    }
+
+    const orgIds = targetOrgs.map((org) => org.id);
+    const statsRows = await db
+      .select({
+        organizationId: workflows.organizationId,
+        total: sql<number>`COUNT(*)`,
+        success: sql<number>`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'success')`,
+        userErrors: sql<number>`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'error')`,
+        systemErrors: sql<number>`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'system_error')`,
+      })
+      .from(workflowExecutions)
+      .innerJoin(workflows, eq(workflows.id, workflowExecutions.workflowId))
+      .where(
+        and(
+          inArray(workflows.organizationId, orgIds),
+          gte(workflowExecutions.startedAt, since)
+        )
+      )
+      .groupBy(workflows.organizationId);
+
+    const statsByOrg = new Map(
+      statsRows.map((row) => [
+        row.organizationId,
+        {
+          total: Number(row.total),
+          success: Number(row.success),
+          userErrors: Number(row.userErrors),
+          systemErrors: Number(row.systemErrors),
+        },
+      ])
+    );
+
+    const orgs: OrgVerification[] = targetOrgs.map((org) => {
+      const stats = statsByOrg.get(org.id) ?? {
+        total: 0,
+        success: 0,
+        userErrors: 0,
+        systemErrors: 0,
+      };
+      const errorCount = stats.userErrors + stats.systemErrors;
+      const errorRate = stats.total > 0 ? errorCount / stats.total : 0;
+
+      const reasons: string[] = [];
+      if (stats.systemErrors > 0) {
+        reasons.push(`${stats.systemErrors} system_error execution(s)`);
+      }
+      if (stats.total >= minExecutions && errorRate > maxErrorRate) {
+        reasons.push(
+          `error rate ${(errorRate * 100).toFixed(1)}% exceeds ${(
+            maxErrorRate * 100
+          ).toFixed(1)}% over ${stats.total} execution(s)`
+        );
+      }
+
+      return {
+        organizationId: org.id,
+        slug: org.slug,
+        name: org.name,
+        plan: org.plan,
+        total: stats.total,
+        success: stats.success,
+        userErrors: stats.userErrors,
+        systemErrors: stats.systemErrors,
+        errorRate,
+        isProblem: reasons.length > 0,
+        reasons,
+        sampleErrors: [],
+      };
+    });
+
+    const problemOrgIds = orgs
+      .filter((org) => org.isProblem)
+      .map((org) => org.organizationId);
+    const sampleErrors = await fetchSampleErrors(problemOrgIds, since);
+    for (const org of orgs) {
+      if (org.isProblem) {
+        org.sampleErrors = sampleErrors.get(org.organizationId) ?? [];
+      }
+    }
+
+    const problems = orgs.filter((org) => org.isProblem);
+
+    // The per-org detail (slugs, names, sample error text) is customer data and
+    // must not surface in the public-repo CI logs or an external Discord channel.
+    // Emit it to internal observability (Loki) instead, where operators look up
+    // specifics; the CI job only ever reports aggregate counts.
+    if (problems.length > 0) {
+      logSystemWarn(
+        ErrorCategory.WORKFLOW_ENGINE,
+        "Post-deploy verification flagged managed/enterprise org errors",
+        undefined,
+        {
+          problem_count: String(problems.length),
+          checked_orgs: String(orgs.length),
+          lookback_minutes: String(lookbackMinutes),
+          problems: JSON.stringify(
+            problems.map((org) => ({
+              slug: org.slug,
+              plan: org.plan,
+              total: org.total,
+              userErrors: org.userErrors,
+              systemErrors: org.systemErrors,
+              reasons: org.reasons,
+              sampleErrors: org.sampleErrors,
+            }))
+          ),
+        }
+      );
+    }
+
+    return NextResponse.json({
+      ok: problems.length === 0,
+      lookbackMinutes,
+      minExecutions,
+      maxErrorRate,
+      checkedOrgs: orgs.length,
+      problemCount: problems.length,
+      orgs,
+      problems,
+      generatedAt: new Date().toISOString(),
+    });
+  } catch (error) {
+    logSystemError(
+      ErrorCategory.DATABASE,
+      "Failed to run post-deploy verification",
+      error,
+      { endpoint: "/api/internal/post-deploy-verification", operation: "get" }
+    );
+    return NextResponse.json(
+      {
+        ok: false,
+        error:
+          error instanceof Error
+            ? error.message
+            : "Failed to run post-deploy verification",
+      },
+      { status: HttpStatus.INTERNAL_SERVER_ERROR }
+    );
+  }
+}
diff --git a/lib/metrics/db-metrics.ts b/lib/metrics/db-metrics.ts
index cb62659d4..1b81ae9ae 100644
--- a/lib/metrics/db-metrics.ts
+++ b/lib/metrics/db-metrics.ts
@@ -50,6 +50,7 @@ import {
 } from "@/lib/db/schema";
 import { ERROR_STATUSES } from "@/lib/errors/execution-status";
 import { ErrorCategory, logSystemWarn } from "@/lib/logging";
+import { MANAGED_ORG_SLUGS as MANAGED_ORG_SLUGS_SOURCE } from "@/lib/orgs/managed-clients";
 import type { BillingStatus } from "./types";
 
 // Label value used for workflow executions whose workflow has no organization
@@ -59,13 +60,12 @@ import type { BillingStatus } from "./types";
 // workflows still produce a series rather than silently dropping increments.
 export const ANONYMOUS_ORG_SLUG = "_anonymous";
 
-// Org slugs for the managed clients (Sky, Ajna) whose per-workflow error series
-// power the managed-client user-error alerts. The per-workflow gauge is scoped
-// to these slugs so `workflow_id` never becomes an unbounded label across the
-// whole user base — only managed workflows that have errored emit a series.
-// Mirrors `local.managed_org_slugs_regex` in the infra Grafana alert config;
-// adding a managed org requires updating both lists.
-export const MANAGED_ORG_SLUGS = ["techops-services", "ajna"] as const;
+// Re-exported from the dependency-free source of truth so the metrics scraper,
+// the post-deploy verification route, and scripts all read the same list. The
+// per-workflow error gauge is scoped to these slugs so `workflow_id` never
+// becomes an unbounded label across the whole user base — only managed
+// workflows that have errored emit a series.
+export const MANAGED_ORG_SLUGS = MANAGED_ORG_SLUGS_SOURCE;
 
 // Histogram bucket boundaries in milliseconds (must match prometheus.ts)
 const WORKFLOW_DURATION_BUCKETS = [
diff --git a/lib/orgs/managed-clients.ts b/lib/orgs/managed-clients.ts
new file mode 100644
index 000000000..403d4a30e
--- /dev/null
+++ b/lib/orgs/managed-clients.ts
@@ -0,0 +1,15 @@
+/**
+ * Org slugs for the managed clients that get white-glove operational
+ * treatment: their per-workflow error series power the managed-client
+ * user-error Grafana alerts, and they are part of the post-deploy
+ * verification cohort.
+ *
+ * Kept dependency-free so it can be imported from API routes, the metrics
+ * scraper, and scripts without pulling in `server-only` or the metrics pool.
+ *
+ * Mirrors `local.managed_org_slugs_regex` in the infra Grafana alert config;
+ * adding a managed org requires updating both lists.
+ */
+export const MANAGED_ORG_SLUGS = ["techops-services", "ajna"] as const;
+
+export type ManagedOrgSlug = (typeof MANAGED_ORG_SLUGS)[number];

From 32d78eb470d2600fc6a3c373f9c7a4a635b166f4 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Fri, 26 Jun 2026 12:51:13 -0300
Subject: [PATCH 02/10] fix(ci): guard discord alert against the embed
 description limit

---
 .github/workflows/post-deploy-verification.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
index 0c12dfefa..2ff5d7c75 100644
--- a/.github/workflows/post-deploy-verification.yml
+++ b/.github/workflows/post-deploy-verification.yml
@@ -148,6 +148,14 @@ jobs:
 
           echo "$SUMMARY"
 
+          # Defensive guard against Discord's embed description limit (4096).
+          # The aggregate message is tiny today, but keep this so a future edit
+          # that adds detail cannot produce a 400 from an over-length payload.
+          MAX_DESC_LENGTH=4000
+          if [ ${#SUMMARY} -gt $MAX_DESC_LENGTH ]; then
+            SUMMARY="${SUMMARY:0:$MAX_DESC_LENGTH}"
+          fi
+
           if [ -n "${DISCORD_WEBHOOK:-}" ]; then
             jq -n \
               --arg title "Post-deploy verification failed" \

From 37e54b4d05b7cbef9959b0962009d0020adf9a60 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Fri, 26 Jun 2026 14:07:55 -0300
Subject: [PATCH 03/10] feat(ci): poll the verification check once a minute for
 up to 5 minutes

---
 .../workflows/post-deploy-verification.yml    | 144 ++++++++++++------
 .../post-deploy-verification/route.ts         |  29 +++-
 2 files changed, 117 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
index 2ff5d7c75..733583825 100644
--- a/.github/workflows/post-deploy-verification.yml
+++ b/.github/workflows/post-deploy-verification.yml
@@ -43,10 +43,14 @@ jobs:
       NAMESPACE: keeperhub
       SERVICE_NAME: keeperhub
       SSM_ENV_PREFIX: techops-prod
-      # Window and thresholds the verification reads recent executions against.
-      # An org is flagged when it has any system_error, or when it ran at least
-      # MIN_EXECUTIONS and its error rate exceeds MAX_ERROR_RATE over the window.
-      LOOKBACK_MINUTES: "60"
+      # The job polls the stateless check endpoint once a minute for at most
+      # MONITOR_MINUTES, watching only executions on the new build (anchored at
+      # the loop start). It stops early the moment an org is flagged, so the
+      # common green path costs ~1 poll, and the worst case is capped at 5 min
+      # of runner time. An org is flagged when it has any system_error, or when
+      # it ran at least MIN_EXECUTIONS and its error rate exceeds MAX_ERROR_RATE.
+      MONITOR_MINUTES: "5"
+      POLL_INTERVAL_SECONDS: "60"
       MIN_EXECUTIONS: "1"
       MAX_ERROR_RATE: "0.5"
     steps:
@@ -69,60 +73,100 @@ jobs:
           echo "::add-mask::$SECRET"
           echo "INTERNAL_SERVICE_HMAC_SECRET=$SECRET" >> "$GITHUB_ENV"
 
-      - name: Run post-deploy verification (in-cluster)
+      - name: Monitor the new build (in-cluster, poll once a minute)
         id: verify
         run: |
-          set -euo pipefail
+          set -uo pipefail
 
           CALLER="scheduler"
           METHOD="GET"
           PATHNAME="/api/internal/post-deploy-verification"
-          QUERY="lookbackMinutes=${LOOKBACK_MINUTES}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
-          TIMESTAMP=$(date +%s)
-
-          # The verifier signs over url.pathname only (no query string), so the
-          # thresholds ride in the query without affecting the signature.
-          BODY_DIGEST=$(printf '' | openssl dgst -sha256 | awk '{print $2}')
-          SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP")
-          SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}')
-
-          URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?${QUERY}"
-          echo "Verifying ${PATHNAME} over a ${LOOKBACK_MINUTES}m window (in-cluster)..."
-
-          # The secret never enters the pod -- only the precomputed signature does.
-          RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
-            --namespace "${NAMESPACE}" \
-            --restart=Never \
-            --attach \
-            --rm \
-            --quiet \
-            --image=curlimages/curl:8.11.0 \
-            --env="URL=${URL}" \
-            --env="CALLER=${CALLER}" \
-            --env="TIMESTAMP=${TIMESTAMP}" \
-            --env="SIGNATURE=${SIGNATURE}" \
-            --command -- sh -c '
-              curl -sS --max-time 30 \
-                -H "X-KH-Caller: ${CALLER}" \
-                -H "X-KH-Timestamp: ${TIMESTAMP}" \
-                -H "X-KH-Signature: ${SIGNATURE}" \
-                "$URL"
-            ')
-
-          echo "$RESPONSE" > /tmp/verification.json
-
-          # This is a PUBLIC repo: Actions logs are world-readable. Surface only
-          # non-identifying aggregates here. Per-org slugs, names, and sample
-          # error text stay in the app's internal logs (Loki) -- never printed to
-          # CI or sent to Discord. The raw endpoint error (which could carry
-          # sensitive detail) is also never echoed.
-          OK=$(jq -r '.ok // false' /tmp/verification.json 2>/dev/null || echo false)
-          CHECKED=$(jq -r '.checkedOrgs // "?"' /tmp/verification.json 2>/dev/null || echo "?")
-          PROBLEMS=$(jq -r '.problemCount // "?"' /tmp/verification.json 2>/dev/null || echo "?")
-          echo "Verification: ok=${OK}, checkedOrgs=${CHECKED}, problemCount=${PROBLEMS}"
+
+          # Anchor the window at the loop start so every poll counts only
+          # executions that began after the deploy -- i.e. runs on the new build.
+          SINCE=$(date +%s)
+          DEADLINE=$(( SINCE + MONITOR_MINUTES * 60 ))
+
+          ATTEMPT=0
+          PROBLEM_FOUND=0
+          GOT_SIGNAL=0
+          CHECKED="?"
+          PROBLEMS="?"
+
+          while true; do
+            ATTEMPT=$(( ATTEMPT + 1 ))
+
+            # Re-sign per poll: X-KH-Timestamp must be within the verifier's
+            # replay window. The verifier signs over url.pathname only, so the
+            # query (since/thresholds) does not affect the signature.
+            TIMESTAMP=$(date +%s)
+            BODY_DIGEST=$(printf '' | openssl dgst -sha256 | awk '{print $2}')
+            SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP")
+            SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}')
+
+            URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
+
+            # The secret never enters the pod -- only the precomputed signature does.
+            RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${ATTEMPT}" \
+              --namespace "${NAMESPACE}" \
+              --restart=Never \
+              --attach \
+              --rm \
+              --quiet \
+              --image=curlimages/curl:8.11.0 \
+              --env="URL=${URL}" \
+              --env="CALLER=${CALLER}" \
+              --env="TIMESTAMP=${TIMESTAMP}" \
+              --env="SIGNATURE=${SIGNATURE}" \
+              --command -- sh -c '
+                curl -sS --max-time 30 \
+                  -H "X-KH-Caller: ${CALLER}" \
+                  -H "X-KH-Timestamp: ${TIMESTAMP}" \
+                  -H "X-KH-Signature: ${SIGNATURE}" \
+                  "$URL"
+              ' 2>/dev/null) || true
+
+            # PUBLIC repo: Actions logs are world-readable. Surface only
+            # non-identifying aggregates. Per-org slugs/names/error text stay in
+            # internal logs (Loki); the raw endpoint error is never echoed.
+            OK=$(printf '%s' "$RESPONSE" | jq -r '.ok // empty' 2>/dev/null)
+            PC=$(printf '%s' "$RESPONSE" | jq -r '.problemCount // empty' 2>/dev/null)
+            CO=$(printf '%s' "$RESPONSE" | jq -r '.checkedOrgs // empty' 2>/dev/null)
+            TE=$(printf '%s' "$RESPONSE" | jq -r '.totalExecutions // empty' 2>/dev/null)
+
+            if [ -n "$OK" ]; then
+              GOT_SIGNAL=1
+              CHECKED="$CO"
+              PROBLEMS="$PC"
+              echo "Attempt ${ATTEMPT}: ok=${OK}, checkedOrgs=${CO}, problemCount=${PC}, totalExecutions=${TE}"
+              if [ "$OK" != "true" ]; then
+                PROBLEM_FOUND=1
+                echo "Problem detected on the new build; stopping monitoring early."
+                break
+              fi
+            else
+              echo "Attempt ${ATTEMPT}: no valid response (transient); will retry."
+            fi
+
+            if [ "$(date +%s)" -ge "$DEADLINE" ]; then
+              echo "Monitored ${MONITOR_MINUTES}m with no problems."
+              break
+            fi
+            sleep "${POLL_INTERVAL_SECONDS}"
+          done
+
+          if [ "$PROBLEM_FOUND" -eq 1 ]; then
+            RESULT="false"
+          elif [ "$GOT_SIGNAL" -eq 1 ]; then
+            RESULT="true"
+          else
+            # Never got a parseable response in the whole window -- cannot
+            # confirm health, so do not greenlight.
+            RESULT="error"
+          fi
 
           {
-            echo "ok=$OK"
+            echo "ok=$RESULT"
             echo "checked=$CHECKED"
             echo "problems=$PROBLEMS"
           } >> "$GITHUB_OUTPUT"
diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
index 284c0eaed..9849e355c 100644
--- a/app/api/internal/post-deploy-verification/route.ts
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -13,8 +13,11 @@
  *     faults are the strongest signal of a deploy regression), OR
  *   - it ran at least `minExecutions` and its error rate exceeds `maxErrorRate`.
  *
- * Window and thresholds are query-param overridable so the CI job can tune them
- * without a redeploy: `lookbackMinutes`, `minExecutions`, `maxErrorRate`.
+ * This is a stateless single check (one query per request); the CI job owns the
+ * timing and polls it. Window and thresholds are query-param overridable so the
+ * job can tune them without a redeploy: `since` (unix seconds, the window
+ * anchor), `lookbackMinutes` (fallback when `since` is absent), `minExecutions`,
+ * and `maxErrorRate`.
  */
 import { and, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
 import { NextResponse } from "next/server";
@@ -168,18 +171,30 @@ export async function GET(request: Request): Promise<NextResponse> {
     DEFAULT_MAX_ERROR_RATE
   );
 
+  // Stateless single check: one query per request. The CI job owns the timing
+  // (it polls this endpoint). `since` (unix seconds) is the window anchor the
+  // caller passes so a poll counts only executions that started after the
+  // deploy -- i.e. runs on the new build. Falls back to the backward lookback
+  // window when omitted.
+  const sinceParam = url.searchParams.get("since");
+  const sinceEpoch = sinceParam ? Number.parseInt(sinceParam, 10) : Number.NaN;
+  const since = Number.isFinite(sinceEpoch)
+    ? new Date(sinceEpoch * 1000)
+    : new Date(Date.now() - lookbackMinutes * 60 * 1000);
+
   try {
-    const since = new Date(Date.now() - lookbackMinutes * 60 * 1000);
+    const windowStart = since.toISOString();
     const targetOrgs = await resolveTargetOrgs();
 
     if (targetOrgs.length === 0) {
       return NextResponse.json({
         ok: true,
-        lookbackMinutes,
+        windowStart,
         minExecutions,
         maxErrorRate,
         checkedOrgs: 0,
         problemCount: 0,
+        totalExecutions: 0,
         orgs: [],
         problems: [],
         generatedAt: new Date().toISOString(),
@@ -266,6 +281,7 @@ export async function GET(request: Request): Promise<NextResponse> {
     }
 
     const problems = orgs.filter((org) => org.isProblem);
+    const totalExecutions = orgs.reduce((sum, org) => sum + org.total, 0);
 
     // The per-org detail (slugs, names, sample error text) is customer data and
     // must not surface in the public-repo CI logs or an external Discord channel.
@@ -279,7 +295,7 @@ export async function GET(request: Request): Promise<NextResponse> {
         {
           problem_count: String(problems.length),
           checked_orgs: String(orgs.length),
-          lookback_minutes: String(lookbackMinutes),
+          window_start: windowStart,
           problems: JSON.stringify(
             problems.map((org) => ({
               slug: org.slug,
@@ -297,11 +313,12 @@ export async function GET(request: Request): Promise<NextResponse> {
 
     return NextResponse.json({
       ok: problems.length === 0,
-      lookbackMinutes,
+      windowStart,
       minExecutions,
       maxErrorRate,
       checkedOrgs: orgs.length,
       problemCount: problems.length,
+      totalExecutions,
       orgs,
       problems,
       generatedAt: new Date().toISOString(),

From 746e943877268d85a4ff661da3b67cf4d15634ad Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Fri, 26 Jun 2026 15:22:35 -0300
Subject: [PATCH 04/10] feat(ci): gate verification cohort to orgs with a
 schedule firing in the window

---
 .../workflows/post-deploy-verification.yml    |  2 +-
 .../post-deploy-verification/route.ts         | 61 +++++++++++++++----
 2 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
index 733583825..6a93104cc 100644
--- a/.github/workflows/post-deploy-verification.yml
+++ b/.github/workflows/post-deploy-verification.yml
@@ -104,7 +104,7 @@ jobs:
             SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP")
             SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}')
 
-            URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
+            URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&until=${DEADLINE}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
 
             # The secret never enters the pod -- only the precomputed signature does.
             RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${ATTEMPT}" \
diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
index 9849e355c..86b5037ff 100644
--- a/app/api/internal/post-deploy-verification/route.ts
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -8,6 +8,13 @@
  * runs inside the cluster against the in-pod service URL (the public hostname
  * is WAF-blocked for /api/internal/*).
  *
+ * The cohort is gated to orgs that actually have work due in the window: an
+ * enabled, non-deleted, non-deactivated workflow with an enabled schedule whose
+ * next run falls inside [`since`, `until`]. Idle orgs (no live workflow, or
+ * schedules that won't fire in the window) are excluded so a deploy is not
+ * judged against orgs that were never going to run. Execution counting covers
+ * every trigger type (scheduled, block, webhook, event, manual).
+ *
  * An org is flagged as a problem when:
  *   - it produced any `system_error` executions in the window (platform/infra
  *     faults are the strongest signal of a deploy regression), OR
@@ -15,17 +22,18 @@
  *
  * This is a stateless single check (one query per request); the CI job owns the
  * timing and polls it. Window and thresholds are query-param overridable so the
- * job can tune them without a redeploy: `since` (unix seconds, the window
- * anchor), `lookbackMinutes` (fallback when `since` is absent), `minExecutions`,
- * and `maxErrorRate`.
+ * job can tune them without a redeploy: `since`/`until` (unix seconds, the
+ * window bounds), `lookbackMinutes` (fallback when `since`/`until` are absent),
+ * `minExecutions`, and `maxErrorRate`.
  */
-import { and, desc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
+import { and, desc, eq, gte, inArray, isNull, lte, or, sql } from "drizzle-orm";
 import { NextResponse } from "next/server";
 import { db } from "@/lib/db";
 import {
   organization,
   organizationSubscriptions,
   workflowExecutions,
+  workflowSchedules,
   workflows,
 } from "@/lib/db/schema";
 import { ERROR_STATUSES } from "@/lib/errors/execution-status";
@@ -70,9 +78,17 @@ function parseNumberParam(value: string | null, fallback: number): number {
   return Number.isFinite(parsed) ? parsed : fallback;
 }
 
-async function resolveTargetOrgs(): Promise<TargetOrg[]> {
+async function resolveTargetOrgs(
+  windowStart: Date,
+  windowEnd: Date
+): Promise<TargetOrg[]> {
+  // Only verify orgs that actually have something due to run during the
+  // monitoring window: an enabled, non-deleted, non-deactivated workflow with
+  // an enabled schedule whose next run lands inside [windowStart, windowEnd].
+  // Without this an idle org (no live workflow, or schedules that won't fire in
+  // our window) is either dead weight or a false-positive source.
   const rows = await db
-    .select({
+    .selectDistinct({
       id: organization.id,
       slug: organization.slug,
       name: organization.name,
@@ -83,13 +99,24 @@ async function resolveTargetOrgs(): Promise<TargetOrg[]> {
       organizationSubscriptions,
       eq(organizationSubscriptions.organizationId, organization.id)
     )
+    .innerJoin(workflows, eq(workflows.organizationId, organization.id))
+    .innerJoin(
+      workflowSchedules,
+      eq(workflowSchedules.workflowId, workflows.id)
+    )
     .where(
       and(
         isNull(organization.deactivatedAt),
         or(
           inArray(organization.slug, [...MANAGED_ORG_SLUGS]),
           eq(organizationSubscriptions.plan, "enterprise")
-        )
+        ),
+        eq(workflows.enabled, true),
+        isNull(workflows.deletedAt),
+        isNull(workflows.deactivatedAt),
+        eq(workflowSchedules.enabled, true),
+        gte(workflowSchedules.nextRunAt, windowStart),
+        lte(workflowSchedules.nextRunAt, windowEnd)
       )
     );
 
@@ -172,24 +199,33 @@ export async function GET(request: Request): Promise<NextResponse> {
   );
 
   // Stateless single check: one query per request. The CI job owns the timing
-  // (it polls this endpoint). `since` (unix seconds) is the window anchor the
-  // caller passes so a poll counts only executions that started after the
-  // deploy -- i.e. runs on the new build. Falls back to the backward lookback
-  // window when omitted.
+  // (it polls this endpoint). `since`/`until` (unix seconds) bound the
+  // monitoring window the caller is watching: executions are counted from
+  // `since` (so a poll sees only runs on the new build), and the cohort is
+  // gated to orgs with a schedule firing within [`since`, `until`]. Both fall
+  // back to the backward lookback window when omitted.
   const sinceParam = url.searchParams.get("since");
   const sinceEpoch = sinceParam ? Number.parseInt(sinceParam, 10) : Number.NaN;
   const since = Number.isFinite(sinceEpoch)
     ? new Date(sinceEpoch * 1000)
     : new Date(Date.now() - lookbackMinutes * 60 * 1000);
 
+  const untilParam = url.searchParams.get("until");
+  const untilEpoch = untilParam ? Number.parseInt(untilParam, 10) : Number.NaN;
+  const until = Number.isFinite(untilEpoch)
+    ? new Date(untilEpoch * 1000)
+    : new Date(since.getTime() + lookbackMinutes * 60 * 1000);
+
   try {
     const windowStart = since.toISOString();
-    const targetOrgs = await resolveTargetOrgs();
+    const windowEnd = until.toISOString();
+    const targetOrgs = await resolveTargetOrgs(since, until);
 
     if (targetOrgs.length === 0) {
       return NextResponse.json({
         ok: true,
         windowStart,
+        windowEnd,
         minExecutions,
         maxErrorRate,
         checkedOrgs: 0,
@@ -314,6 +350,7 @@ export async function GET(request: Request): Promise<NextResponse> {
     return NextResponse.json({
       ok: problems.length === 0,
       windowStart,
+      windowEnd,
       minExecutions,
       maxErrorRate,
       checkedOrgs: orgs.length,

From 440bb064ac1d6c8805e4d8191bc7a74452ccbbae Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Tue, 30 Jun 2026 21:34:40 -0300
Subject: [PATCH 05/10] refactor(observability): source managed-client cohort
 from env, not hardcoded slugs

Replace the hardcoded managed-client slug list with getManagedOrgSlugs(),
parsed from MANAGED_ORGS_CONFIG (a JSON dict injected at deploy from Parameter
Store). This keeps client identities out of the public source; the metrics
scraper and the post-deploy verification route now read the same env-sourced
list. Empty/unset config yields no managed cohort (bounded gauge, safe default).

Also scrub the remaining client names from the backfill script, unit-test
fixtures, and code comments.
---
 .env.example                             |  9 ++++
 lib/errors/classify.ts                   |  5 +-
 lib/errors/finalize-error.ts             |  2 +-
 lib/logging.ts                           |  2 +-
 lib/metrics/METRICS_REFERENCE.md         |  8 ++--
 lib/metrics/db-metrics.ts                | 31 ++++++------
 lib/orgs/managed-clients.ts              | 61 ++++++++++++++++++++----
 scripts/backfill-error-classification.ts | 27 ++++++-----
 tests/unit/db-metrics-cache.test.ts      | 27 ++++++-----
 9 files changed, 116 insertions(+), 56 deletions(-)

diff --git a/.env.example b/.env.example
index 046938fc2..75d406d6e 100644
--- a/.env.example
+++ b/.env.example
@@ -48,6 +48,15 @@ SENTRY_ORG=
 SENTRY_PROJECT=
 SENTRY_AUTH_TOKEN=
 
+# Managed-client cohort + per-client PagerDuty routing. In deployed envs this is
+# injected from AWS Parameter Store so client slugs and routing keys never live
+# in source. JSON dict keyed by org slug; empty/unset => no managed cohort.
+# Example: {"my-client":{"pagerdutyRoutingKey":"R0..."}}
+MANAGED_ORGS_CONFIG=
+# Fallback PagerDuty Events v2 routing key for enterprise-plan orgs without a
+# dedicated entry in MANAGED_ORGS_CONFIG. Unset => those orgs are not paged.
+DEFAULT_PAGERDUTY_ROUTING_KEY=
+
 # Internal service-to-service auth (HMAC). AGENTIC_WALLET_HMAC_KMS_KEY is the
 # 32-byte base64 AES key the app uses to encrypt/decrypt the shared signing
 # secret in the internal_service_hmac_secrets store; it must be set for internal
diff --git a/lib/errors/classify.ts b/lib/errors/classify.ts
index 3d62bf33b..fcca5afe1 100644
--- a/lib/errors/classify.ts
+++ b/lib/errors/classify.ts
@@ -16,9 +16,8 @@ import { ErrorCategory } from "@/lib/logging";
  *                    null for user failures (which surface their raw message).
  *
  * The classifier is intentionally pattern-driven against real production
- * messages observed for managed clients (Sky/Ajna) so the resulting
- * `error_type` label on `workflow_executions` lets the SLI alert filter
- * out user-config noise.
+ * messages observed for managed clients so the resulting `error_type` label on
+ * `workflow_executions` lets the SLI alert filter out user-config noise.
  *
  * Default for unmatched messages is WORKFLOW_ENGINE / errorType="system" /
  * code=DEFAULT_SYSTEM_ERROR_CODE. That defaults to "treat unknown as system"
diff --git a/lib/errors/finalize-error.ts b/lib/errors/finalize-error.ts
index 14585be34..e4f1d279e 100644
--- a/lib/errors/finalize-error.ts
+++ b/lib/errors/finalize-error.ts
@@ -15,7 +15,7 @@ import { ANONYMOUS_ORG_SLUG } from "@/lib/metrics/db-metrics";
  * `workflow_executions` row has been written with status='error'.
  *
  * Resolves the org slug for the workflow so the counter series is scoped
- * for the SLA alert (Sky/Ajna). Falls back to ANONYMOUS_ORG_SLUG for
+ * for the managed-client SLA alert. Falls back to ANONYMOUS_ORG_SLUG for
  * personal workflows so personal failures still emit a series.
  *
  * Safe to call after the DB write succeeded. Errors are caught and dropped
diff --git a/lib/logging.ts b/lib/logging.ts
index b82369e70..af1bd699f 100644
--- a/lib/logging.ts
+++ b/lib/logging.ts
@@ -134,7 +134,7 @@ function buildErrPayload(
  * Loki labels, not just as text inside the message string.
  *
  * The human-readable `msg` field retains the original message and the
- * `[org:Sky][exec:xyz]` tag so `kubectl logs` greps still work.
+ * `[org:<slug>][exec:xyz]` tag so `kubectl logs` greps still work.
  */
 function buildStructuredPayload(args: {
   message: string;
diff --git a/lib/metrics/METRICS_REFERENCE.md b/lib/metrics/METRICS_REFERENCE.md
index eba5ca4ca..7a1fa90d3 100644
--- a/lib/metrics/METRICS_REFERENCE.md
+++ b/lib/metrics/METRICS_REFERENCE.md
@@ -84,7 +84,7 @@ Error metrics tracking failures and exceptions.
 | Metric Name | Description | Labels | Target | Source |
 |-------------|-------------|--------|--------|--------|
 | `workflow.execution.errors` | Failed workflow executions | - | < 5% | DB |
-| `workflow.errors.by_workflow` | Errored executions in the **last hour** (rolling 1h window), managed orgs only. Powers the Sky/Ajna managed-client error alerts; the alert reads it directly (no offset). Not cumulative. | `workflow_id`, `org_slug`, `error_type` | - | DB |
+| `workflow.errors.by_workflow` | Errored executions in the **last hour** (rolling 1h window), managed orgs only. Powers the managed-client error alerts; the alert reads it directly (no offset). Not cumulative. | `workflow_id`, `org_slug`, `error_type` | - | DB |
 | `workflow.step.errors` | Failed step executions | `step_type` | < 10% | DB |
 | `plugin.action.errors` | Failed plugin actions | `plugin_name`, `action_name`, `error_type` | < 20% | API |
 | `api.errors.total` | API errors (webhook failures) | `endpoint`, `status_code`, `error_type` | count | API |
@@ -374,7 +374,7 @@ max by (status) (keeperhub_workflow_executions_total{...})
 sum by (status) (keeperhub_workflow_executions_total{...})
 ```
 
-`keeperhub_workflow_executions_total` and `keeperhub_workflow_execution_errors_total` are labeled by `org_slug` so dashboards/alerts can scope to managed clients. Personal/anonymous workflows are emitted under `org_slug="_anonymous"` so the sum across `org_slug` for a given status equals the unfiltered per-status total. To filter to managed clients, add `org_slug=~"techops-services|ajna"` (or the inverse `!~` for user workflows).
+`keeperhub_workflow_executions_total` and `keeperhub_workflow_execution_errors_total` are labeled by `org_slug` so dashboards/alerts can scope to managed clients. Personal/anonymous workflows are emitted under `org_slug="_anonymous"` so the sum across `org_slug` for a given status equals the unfiltered per-status total. To filter to managed clients, add `org_slug=~"$managed_slugs"` (or the inverse `!~` for user workflows).
 
 **Metrics requiring `max()` aggregation:**
 
@@ -422,11 +422,11 @@ sum by (status) (
 
 # Error rate over last hour, scoped to managed orgs
 100 * sum(max by (org_slug) (
-        delta(keeperhub_workflow_execution_errors_total{org_slug=~"techops-services|ajna"}[1h])
+        delta(keeperhub_workflow_execution_errors_total{org_slug=~"$managed_slugs"}[1h])
       ))
     / clamp_min(
         sum(max by (status, org_slug) (
-          delta(keeperhub_workflow_executions_total{org_slug=~"techops-services|ajna"}[1h])
+          delta(keeperhub_workflow_executions_total{org_slug=~"$managed_slugs"}[1h])
         )),
         1
       )
diff --git a/lib/metrics/db-metrics.ts b/lib/metrics/db-metrics.ts
index 1b81ae9ae..df625cd51 100644
--- a/lib/metrics/db-metrics.ts
+++ b/lib/metrics/db-metrics.ts
@@ -50,7 +50,7 @@ import {
 } from "@/lib/db/schema";
 import { ERROR_STATUSES } from "@/lib/errors/execution-status";
 import { ErrorCategory, logSystemWarn } from "@/lib/logging";
-import { MANAGED_ORG_SLUGS as MANAGED_ORG_SLUGS_SOURCE } from "@/lib/orgs/managed-clients";
+import { getManagedOrgSlugs } from "@/lib/orgs/managed-clients";
 import type { BillingStatus } from "./types";
 
 // Label value used for workflow executions whose workflow has no organization
@@ -60,12 +60,10 @@ import type { BillingStatus } from "./types";
 // workflows still produce a series rather than silently dropping increments.
 export const ANONYMOUS_ORG_SLUG = "_anonymous";
 
-// Re-exported from the dependency-free source of truth so the metrics scraper,
-// the post-deploy verification route, and scripts all read the same list. The
-// per-workflow error gauge is scoped to these slugs so `workflow_id` never
-// becomes an unbounded label across the whole user base — only managed
-// workflows that have errored emit a series.
-export const MANAGED_ORG_SLUGS = MANAGED_ORG_SLUGS_SOURCE;
+// The per-workflow error gauge is scoped to the managed-client slugs (from
+// `getManagedOrgSlugs()`, env-sourced) so `workflow_id` never becomes an
+// unbounded label across the whole user base — only managed workflows that have
+// errored emit a series.
 
 // Histogram bucket boundaries in milliseconds (must match prometheus.ts)
 const WORKFLOW_DURATION_BUCKETS = [
@@ -160,9 +158,7 @@ export async function getWorkflowStatsFromDb(): Promise<WorkflowStats> {
       .from(workflowExecutions)
       .innerJoin(workflows, eq(workflowExecutions.workflowId, workflows.id))
       .leftJoin(organization, eq(workflows.organizationId, organization.id))
-      .where(
-        gte(workflowExecutions.startedAt, sql`now() - interval '30 days'`)
-      )
+      .where(gte(workflowExecutions.startedAt, sql`now() - interval '30 days'`))
       .groupBy(
         workflowExecutions.status,
         organization.slug,
@@ -292,11 +288,18 @@ export type WorkflowErrorsByWorkflow = Array<{
  * last hour keeps the row set tiny so the query is an index range scan on
  * idx_workflow_executions_error_completed_at and finishes in milliseconds.
  *
- * Scoped to MANAGED_ORG_SLUGS to bound `workflow_id` cardinality. errorType is
- * the `workflow_executions.error_type` column, projected to "unknown" for
- * rows that predate classification so every series carries a populated label.
+ * Scoped to the managed-client slugs to bound `workflow_id` cardinality.
+ * errorType is the `workflow_executions.error_type` column, projected to
+ * "unknown" for rows that predate classification so every series carries a
+ * populated label.
  */
 export async function getWorkflowErrorsByWorkflowFromDb(): Promise<WorkflowErrorsByWorkflow> {
+  const managedSlugs = getManagedOrgSlugs();
+  if (managedSlugs.length === 0) {
+    // No managed cohort configured -> emit no managed series (keeps the gauge
+    // bounded and avoids a pointless query).
+    return [];
+  }
   try {
     const rows = await db
       .select({
@@ -312,7 +315,7 @@ export async function getWorkflowErrorsByWorkflowFromDb(): Promise<WorkflowError
         and(
           inArray(workflowExecutions.status, [...ERROR_STATUSES]),
           sql`${workflowExecutions.completedAt} >= now() - interval '1 hour'`,
-          inArray(organization.slug, [...MANAGED_ORG_SLUGS])
+          inArray(organization.slug, managedSlugs)
         )
       )
       .groupBy(
diff --git a/lib/orgs/managed-clients.ts b/lib/orgs/managed-clients.ts
index 403d4a30e..ca744b106 100644
--- a/lib/orgs/managed-clients.ts
+++ b/lib/orgs/managed-clients.ts
@@ -1,15 +1,56 @@
 /**
- * Org slugs for the managed clients that get white-glove operational
- * treatment: their per-workflow error series power the managed-client
- * user-error Grafana alerts, and they are part of the post-deploy
- * verification cohort.
+ * Managed-client cohort + alert routing, sourced from env -- never hardcoded.
  *
- * Kept dependency-free so it can be imported from API routes, the metrics
- * scraper, and scripts without pulling in `server-only` or the metrics pool.
+ * `MANAGED_ORGS_CONFIG` is a JSON object keyed by org slug, injected at deploy
+ * from AWS Parameter Store (SecureString) so client identities and their
+ * PagerDuty routing keys never live in source (this repo is public). Shape:
+ *   {"<slug>":{"pagerdutyRoutingKey":"<events-v2-key>"}}
  *
- * Mirrors `local.managed_org_slugs_regex` in the infra Grafana alert config;
- * adding a managed org requires updating both lists.
+ * Consumers: the metrics scraper (managed-client error gauge, scoped to these
+ * slugs to bound `workflow_id` cardinality) and the post-deploy verification
+ * route (cohort selection + per-client PagerDuty routing). Kept dependency-free
+ * so it can be imported from API routes, the metrics scraper, and scripts.
+ *
+ * Empty / unset config => no managed slugs: the gauge emits no managed series
+ * and the verification cohort falls back to enterprise-plan orgs only. Parsed
+ * per call (the JSON is tiny) so a value change takes effect without caching.
  */
-export const MANAGED_ORG_SLUGS = ["techops-services", "ajna"] as const;
 
-export type ManagedOrgSlug = (typeof MANAGED_ORG_SLUGS)[number];
+export type ManagedOrgConfig = {
+  pagerdutyRoutingKey?: string;
+};
+
+function parseManagedOrgs(): Record<string, ManagedOrgConfig> {
+  const raw = process.env.MANAGED_ORGS_CONFIG;
+  if (!raw) {
+    return {};
+  }
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      return parsed as Record<string, ManagedOrgConfig>;
+    }
+  } catch {
+    // Malformed config behaves like "no managed orgs" rather than crashing the
+    // caller (a metrics scrape or the verification route).
+  }
+  return {};
+}
+
+/** Slugs of the managed-client cohort. Empty when unconfigured. */
+export function getManagedOrgSlugs(): string[] {
+  return Object.keys(parseManagedOrgs());
+}
+
+/**
+ * PagerDuty Events v2 routing key for a slug. Falls back to
+ * `DEFAULT_PAGERDUTY_ROUTING_KEY` (for enterprise-plan orgs without a dedicated
+ * service). Undefined when neither is set -- callers then skip paging.
+ */
+export function getPagerDutyRoutingKey(slug: string): string | undefined {
+  return (
+    parseManagedOrgs()[slug]?.pagerdutyRoutingKey ??
+    process.env.DEFAULT_PAGERDUTY_ROUTING_KEY ??
+    undefined
+  );
+}
diff --git a/scripts/backfill-error-classification.ts b/scripts/backfill-error-classification.ts
index 67baabc16..cba00028d 100644
--- a/scripts/backfill-error-classification.ts
+++ b/scripts/backfill-error-classification.ts
@@ -1,22 +1,24 @@
 /**
- * KEEP-545: one-time backfill of `error_category` and `error_type`
- * columns on `workflow_executions` rows for managed-client orgs (Sky and
- * Ajna) over the last 90 days.
+ * One-time backfill of `error_category` and `error_type` columns on
+ * `workflow_executions` rows for the managed-client orgs over the last 90 days.
  *
  * The DB schema migration that added these columns leaves them null on
- * historical rows. The SLA alert post-PR-2 watches the new counter, not
- * the historical column, so backfill is not required for the alert. The
- * reason to run this is to produce an immediate inventory of "real
- * engineering failures vs user-config noise" inside the managed-client
- * scope without writing one-off SQL.
+ * historical rows. The SLA alert watches the new counter, not the historical
+ * column, so backfill is not required for the alert. The reason to run this is
+ * to produce an immediate inventory of "real engineering failures vs
+ * user-config noise" inside the managed-client scope without writing one-off
+ * SQL.
  *
- * Idempotent: skips rows that already have both classification columns
- * set. Safe to re-run.
+ * Idempotent: skips rows that already have both classification columns set.
+ * Safe to re-run.
+ *
+ * Org scope defaults to the managed cohort (`getManagedOrgSlugs()`, sourced
+ * from MANAGED_ORGS_CONFIG); pass `--orgs` to override.
  *
  * Usage:
  *   pnpm tsx scripts/backfill-error-classification.ts --dry-run
  *   pnpm tsx scripts/backfill-error-classification.ts
- *   pnpm tsx scripts/backfill-error-classification.ts --orgs sky,ajna --days 30
+ *   pnpm tsx scripts/backfill-error-classification.ts --orgs slug-a,slug-b --days 30
  *
  * Output:
  *   - Progress lines per batch (`processed=N updated=M skipped=K`)
@@ -27,8 +29,9 @@ import { and, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
 import { db } from "@/lib/db";
 import { organization, workflowExecutions, workflows } from "@/lib/db/schema";
 import { classifyExecutionError } from "@/lib/errors/classify";
+import { getManagedOrgSlugs } from "@/lib/orgs/managed-clients";
 
-const DEFAULT_ORGS = ["techops-services", "ajna"];
+const DEFAULT_ORGS = getManagedOrgSlugs();
 const DEFAULT_DAYS = 90;
 const BATCH_SIZE = 500;
 
diff --git a/tests/unit/db-metrics-cache.test.ts b/tests/unit/db-metrics-cache.test.ts
index 0241d8738..65bafb2fc 100644
--- a/tests/unit/db-metrics-cache.test.ts
+++ b/tests/unit/db-metrics-cache.test.ts
@@ -140,10 +140,10 @@ const CACHE_LOOKUP_HIT_RE =
   /keeperhub_db_metrics_cache_lookups_total\{result="hit"\}\s+\d+/;
 const REFRESH_SUCCESS_RE =
   /keeperhub_db_metrics_refresh_total\{outcome="success"\}\s+\d+/;
-const ERRORS_BY_WORKFLOW_SKY_RE =
-  /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_sky_1"[^}]*org_slug="techops-services"[^}]*error_type="user"[^}]*\}\s+7/;
-const ERRORS_BY_WORKFLOW_AJNA_RE =
-  /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_ajna_1"[^}]*org_slug="ajna"[^}]*error_type="system"[^}]*\}\s+2/;
+const ERRORS_BY_WORKFLOW_A_RE =
+  /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_a_1"[^}]*org_slug="managed-a"[^}]*error_type="user"[^}]*\}\s+7/;
+const ERRORS_BY_WORKFLOW_B_RE =
+  /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_b_1"[^}]*org_slug="managed-b"[^}]*error_type="system"[^}]*\}\s+2/;
 const ERRORS_BY_CATEGORY_SYSTEM_RE =
   /keeperhub_system_errors_by_category\{[^}]*error_category="network_rpc"[^}]*error_type="system"[^}]*\}\s+5/;
 const ERRORS_BY_CATEGORY_UNKNOWN_RE =
@@ -392,14 +392,14 @@ describe("keeperhub_workflow_errors_by_workflow gauge", () => {
   it("emits one series per (workflow_id, org_slug, error_type) from the DB query", async () => {
     dbMocks.getWorkflowErrorsByWorkflowFromDb.mockResolvedValue([
       {
-        workflowId: "wf_sky_1",
-        orgSlug: "techops-services",
+        workflowId: "wf_a_1",
+        orgSlug: "managed-a",
         errorType: "user",
         count: 7,
       },
       {
-        workflowId: "wf_ajna_1",
-        orgSlug: "ajna",
+        workflowId: "wf_b_1",
+        orgSlug: "managed-b",
         errorType: "system",
         count: 2,
       },
@@ -408,13 +408,18 @@ describe("keeperhub_workflow_errors_by_workflow gauge", () => {
     await updateDbMetrics();
     const out = await getDbMetrics();
 
-    expect(out).toMatch(ERRORS_BY_WORKFLOW_SKY_RE);
-    expect(out).toMatch(ERRORS_BY_WORKFLOW_AJNA_RE);
+    expect(out).toMatch(ERRORS_BY_WORKFLOW_A_RE);
+    expect(out).toMatch(ERRORS_BY_WORKFLOW_B_RE);
   });
 
   it("clears stale series when a workflow stops appearing in the query", async () => {
     dbMocks.getWorkflowErrorsByWorkflowFromDb.mockResolvedValueOnce([
-      { workflowId: "wf_gone", orgSlug: "ajna", errorType: "user", count: 3 },
+      {
+        workflowId: "wf_gone",
+        orgSlug: "managed-b",
+        errorType: "user",
+        count: 3,
+      },
     ]);
     await updateDbMetrics();
     expect(await getDbMetrics()).toContain('workflow_id="wf_gone"');

From c983d04322b1c7466cd2e884772ad65ea8a715c6 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Tue, 30 Jun 2026 21:34:59 -0300
Subject: [PATCH 06/10] feat(ci): route post-deploy alerts to PagerDuty per
 client

Replace the Discord notification with server-side PagerDuty paging: each flagged
managed/enterprise org pages its own service via a per-client routing key (env
sourced), deduped to one incident per org per deploy. Slugs and routing keys
never enter the CI job.

Add "no executions in the window" detection (P2) concluded on the loop's final
poll, alongside system_error (P2) and user-error-rate (P3). The workflow passes
final/deployId and now only fails the job red on problems; PagerDuty does the
paging.
---
 .../workflows/post-deploy-verification.yml    |  68 ++++------
 .../post-deploy-verification/route.ts         | 117 ++++++++++++++++--
 lib/alerting/pagerduty.ts                     |  77 ++++++++++++
 3 files changed, 205 insertions(+), 57 deletions(-)
 create mode 100644 lib/alerting/pagerduty.ts

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
index 6a93104cc..85e208399 100644
--- a/.github/workflows/post-deploy-verification.yml
+++ b/.github/workflows/post-deploy-verification.yml
@@ -75,6 +75,10 @@ jobs:
 
       - name: Monitor the new build (in-cluster, poll once a minute)
         id: verify
+        env:
+          # Scopes the endpoint's PagerDuty dedup key so every poll of this
+          # deploy collapses into one incident per org.
+          DEPLOY_ID: ${{ github.event.workflow_run.head_sha }}
         run: |
           set -uo pipefail
 
@@ -104,7 +108,15 @@ jobs:
             SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP")
             SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}')
 
-            URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&until=${DEADLINE}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
+            # Mark the last poll: only then can the endpoint conclude "no
+            # executions in the window" (a due schedule may fire at the end).
+            if [ $(( $(date +%s) + POLL_INTERVAL_SECONDS )) -ge "$DEADLINE" ]; then
+              FINAL=1
+            else
+              FINAL=0
+            fi
+
+            URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&until=${DEADLINE}&final=${FINAL}&deployId=${DEPLOY_ID}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}"
 
             # The secret never enters the pod -- only the precomputed signature does.
             RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${ATTEMPT}" \
@@ -171,58 +183,22 @@ jobs:
             echo "problems=$PROBLEMS"
           } >> "$GITHUB_OUTPUT"
 
-      - name: Alert and fail on verification problems
+      - name: Fail on verification problems
         if: always() && steps.verify.outputs.ok != 'true'
         env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK_DEPLOY_ALERTS }}
           CHECKED: ${{ steps.verify.outputs.checked }}
           PROBLEMS: ${{ steps.verify.outputs.problems }}
         run: |
-          REPO_URL="https://github.com/${{ github.repository }}"
-          RUN_URL="${REPO_URL}/actions/runs/${{ github.run_id }}"
-
-          # Aggregate-only message. No org slugs/names/error text -- those live in
-          # internal logs (Loki: "Post-deploy verification flagged ..."). Safe to
-          # post to a chat channel without leaking customer data.
+          # Paging is done server-side by the endpoint (per-client PagerDuty
+          # routing), so the CI job carries no slugs or routing keys. This step
+          # just surfaces a red job + aggregate counts; the per-org detail lives
+          # in PagerDuty and internal logs (Loki: "post-deploy verification
+          # flagged ...").
           if [ -n "${PROBLEMS:-}" ] && [ "${PROBLEMS}" != "?" ]; then
-            SUMMARY="Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) erroring on the new build. Details are in internal logs (search: post-deploy verification flagged)."
-          else
-            SUMMARY="Post-deploy verification could not complete. See the Actions run and internal logs."
-          fi
-
-          echo "$SUMMARY"
-
-          # Defensive guard against Discord's embed description limit (4096).
-          # The aggregate message is tiny today, but keep this so a future edit
-          # that adds detail cannot produce a 400 from an over-length payload.
-          MAX_DESC_LENGTH=4000
-          if [ ${#SUMMARY} -gt $MAX_DESC_LENGTH ]; then
-            SUMMARY="${SUMMARY:0:$MAX_DESC_LENGTH}"
-          fi
-
-          if [ -n "${DISCORD_WEBHOOK:-}" ]; then
-            jq -n \
-              --arg title "Post-deploy verification failed" \
-              --arg url "$RUN_URL" \
-              --arg desc "$SUMMARY" \
-              '{
-                username: "KeeperHub Deploy Verifier",
-                embeds: [{
-                  title: $title,
-                  url: $url,
-                  description: $desc,
-                  color: 15548997
-                }]
-              }' > /tmp/discord-payload.json
-
-            HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-              -H "Content-Type: application/json" \
-              -d @/tmp/discord-payload.json \
-              "$DISCORD_WEBHOOK")
-            echo "Discord notification HTTP ${HTTP_STATUS}"
+            echo "Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) on the new build. Paged via PagerDuty; details in internal logs."
           else
-            echo "DISCORD_WEBHOOK_DEPLOY_ALERTS not set; skipping Discord notification (failure still surfaces via the red job and internal logs)."
+            echo "Post-deploy verification could not complete. See the Actions run and internal logs."
           fi
 
-          echo "::error::Post-deploy verification flagged managed/enterprise org errors (count only; see internal logs for specifics)."
+          echo "::error::Post-deploy verification flagged managed/enterprise org problems (count only; PagerDuty + internal logs have specifics)."
           exit 1
diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
index 86b5037ff..42e2d78c6 100644
--- a/app/api/internal/post-deploy-verification/route.ts
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -1,12 +1,15 @@
 /**
  * Post-deployment verification for managed and enterprise organizations.
  *
- * Reads recent workflow execution outcomes for the white-glove cohort
- * (MANAGED_ORG_SLUGS plus any org on an `enterprise` subscription) and reports
- * whether any of them are erroring after a deploy. Authenticated as an internal
- * service (HMAC) and invoked from the post-deploy-verification CI job, which
- * runs inside the cluster against the in-pod service URL (the public hostname
- * is WAF-blocked for /api/internal/*).
+ * Reads recent workflow execution outcomes for the white-glove cohort (the
+ * env-sourced managed slugs from getManagedOrgSlugs() plus any org on an
+ * `enterprise` subscription) and reports whether any of them regressed after a
+ * deploy. Authenticated as an internal service (HMAC) and invoked from the
+ * post-deploy-verification CI job, which runs inside the cluster against the
+ * in-pod service URL (the public hostname is WAF-blocked for /api/internal/*).
+ *
+ * On a flagged org this pages that client's own PagerDuty service (per-client
+ * routing key, env sourced), deduped to one incident per org per deploy.
  *
  * The cohort is gated to orgs that actually have work due in the window: an
  * enabled, non-deleted, non-deactivated workflow with an enabled schedule whose
@@ -28,6 +31,10 @@
  */
 import { and, desc, eq, gte, inArray, isNull, lte, or, sql } from "drizzle-orm";
 import { NextResponse } from "next/server";
+import {
+  type PagerDutySeverity,
+  triggerPagerDutyAlert,
+} from "@/lib/alerting/pagerduty";
 import { db } from "@/lib/db";
 import {
   organization,
@@ -40,13 +47,27 @@ import { ERROR_STATUSES } from "@/lib/errors/execution-status";
 import { HttpStatus } from "@/lib/http-status";
 import { authenticateInternalService } from "@/lib/internal-service-auth";
 import { ErrorCategory, logSystemError, logSystemWarn } from "@/lib/logging";
-import { MANAGED_ORG_SLUGS } from "@/lib/orgs/managed-clients";
+import {
+  getManagedOrgSlugs,
+  getPagerDutyRoutingKey,
+} from "@/lib/orgs/managed-clients";
 
 const DEFAULT_LOOKBACK_MINUTES = 60;
 const DEFAULT_MIN_EXECUTIONS = 1;
 const DEFAULT_MAX_ERROR_RATE = 0.5;
 const SAMPLE_ERRORS_PER_ORG = 3;
 const MAX_SAMPLE_ERROR_LENGTH = 300;
+const PAGERDUTY_SOURCE = "post-deploy-verification";
+
+// Internal P-levels. A missed execution or a platform (system_error) fault is a
+// P2 (a managed/enterprise client's automation silently broke); a user-error
+// rate spike is a P3 (more likely the client's own input/config than the
+// deploy). PagerDuty severity: P2 -> critical, P3 -> warning.
+type AlertSeverity = "P2" | "P3";
+
+function pagerDutySeverity(severity: AlertSeverity): PagerDutySeverity {
+  return severity === "P2" ? "critical" : "warning";
+}
 
 type TargetOrg = {
   id: string;
@@ -66,6 +87,7 @@ type OrgVerification = {
   systemErrors: number;
   errorRate: number;
   isProblem: boolean;
+  severity: AlertSeverity | null;
   reasons: string[];
   sampleErrors: string[];
 };
@@ -87,6 +109,10 @@ async function resolveTargetOrgs(
   // an enabled schedule whose next run lands inside [windowStart, windowEnd].
   // Without this an idle org (no live workflow, or schedules that won't fire in
   // our window) is either dead weight or a false-positive source.
+  //
+  // Cohort = managed slugs (env-sourced) plus any enterprise-plan org. When no
+  // managed slugs are configured we fall back to enterprise-plan orgs only.
+  const managedSlugs = getManagedOrgSlugs();
   const rows = await db
     .selectDistinct({
       id: organization.id,
@@ -107,10 +133,12 @@ async function resolveTargetOrgs(
     .where(
       and(
         isNull(organization.deactivatedAt),
-        or(
-          inArray(organization.slug, [...MANAGED_ORG_SLUGS]),
-          eq(organizationSubscriptions.plan, "enterprise")
-        ),
+        managedSlugs.length > 0
+          ? or(
+              inArray(organization.slug, managedSlugs),
+              eq(organizationSubscriptions.plan, "enterprise")
+            )
+          : eq(organizationSubscriptions.plan, "enterprise"),
         eq(workflows.enabled, true),
         isNull(workflows.deletedAt),
         isNull(workflows.deactivatedAt),
@@ -175,6 +203,45 @@ async function fetchSampleErrors(
   return byOrg;
 }
 
+/**
+ * Page each flagged org's own PagerDuty service (per-client routing key, env
+ * sourced). The dedup key is per (deploy, org) so the verification poll loop
+ * collapses into one incident per org per deploy. Orgs without a routing key
+ * (and no default) are skipped -- they still surface in the response + Loki.
+ */
+async function pageProblemOrgs(
+  problems: OrgVerification[],
+  deployId: string,
+  windowStart: string
+): Promise<void> {
+  await Promise.all(
+    problems.map((org) => {
+      const routingKey = getPagerDutyRoutingKey(org.slug);
+      if (!(routingKey && org.severity)) {
+        return Promise.resolve(false);
+      }
+      return triggerPagerDutyAlert({
+        routingKey,
+        dedupKey: `post-deploy/${deployId || windowStart}/${org.organizationId}`,
+        severity: pagerDutySeverity(org.severity),
+        source: PAGERDUTY_SOURCE,
+        summary: `[${org.severity}] Post-deploy regression for ${org.slug}: ${org.reasons.join("; ")}`,
+        customDetails: {
+          org_slug: org.slug,
+          org_name: org.name,
+          plan: org.plan,
+          total: org.total,
+          user_errors: org.userErrors,
+          system_errors: org.systemErrors,
+          reasons: org.reasons,
+          sample_errors: org.sampleErrors,
+          window_start: windowStart,
+        },
+      });
+    })
+  );
+}
+
 export async function GET(request: Request): Promise<NextResponse> {
   const auth = await authenticateInternalService(request);
   if (!auth.authenticated) {
@@ -198,6 +265,15 @@ export async function GET(request: Request): Promise<NextResponse> {
     DEFAULT_MAX_ERROR_RATE
   );
 
+  // `final=1` on the CI loop's last poll: only then can "no executions in the
+  // window" be concluded (a due schedule may fire at the very end of the
+  // window). `deployId` (commit SHA) scopes the PagerDuty dedup key so all
+  // polls of one deploy collapse into a single incident per org.
+  const isFinalPoll =
+    url.searchParams.get("final") === "1" ||
+    url.searchParams.get("final") === "true";
+  const deployId = url.searchParams.get("deployId") ?? "";
+
   // Stateless single check: one query per request. The CI job owns the timing
   // (it polls this endpoint). `since`/`until` (unix seconds) bound the
   // monitoring window the caller is watching: executions are counted from
@@ -279,15 +355,29 @@ export async function GET(request: Request): Promise<NextResponse> {
       const errorRate = stats.total > 0 ? errorCount / stats.total : 0;
 
       const reasons: string[] = [];
+      let severity: AlertSeverity | null = null;
+
+      // No executions: the org had a schedule due in the window but produced
+      // nothing -- its automation silently did not run. Only conclusive on the
+      // final poll. P2.
+      if (isFinalPoll && stats.total === 0) {
+        reasons.push("no executions in the monitoring window");
+        severity = "P2";
+      }
+      // Platform/infra faults are the strongest signal of a deploy regression. P2.
       if (stats.systemErrors > 0) {
         reasons.push(`${stats.systemErrors} system_error execution(s)`);
+        severity = "P2";
       }
+      // User-error rate spike: more likely the client's own input/config than
+      // the deploy. P3, and never downgrades an already-P2 org.
       if (stats.total >= minExecutions && errorRate > maxErrorRate) {
         reasons.push(
           `error rate ${(errorRate * 100).toFixed(1)}% exceeds ${(
             maxErrorRate * 100
           ).toFixed(1)}% over ${stats.total} execution(s)`
         );
+        severity = severity ?? "P3";
       }
 
       return {
@@ -301,6 +391,7 @@ export async function GET(request: Request): Promise<NextResponse> {
         systemErrors: stats.systemErrors,
         errorRate,
         isProblem: reasons.length > 0,
+        severity,
         reasons,
         sampleErrors: [],
       };
@@ -345,6 +436,10 @@ export async function GET(request: Request): Promise<NextResponse> {
           ),
         }
       );
+
+      // Page each flagged org's PagerDuty service. Never throws; a paging
+      // failure is logged but does not fail the verification response.
+      await pageProblemOrgs(problems, deployId, windowStart);
     }
 
     return NextResponse.json({
diff --git a/lib/alerting/pagerduty.ts b/lib/alerting/pagerduty.ts
new file mode 100644
index 000000000..fba97ea1f
--- /dev/null
+++ b/lib/alerting/pagerduty.ts
@@ -0,0 +1,77 @@
+/**
+ * Minimal PagerDuty Events API v2 client (trigger-only).
+ *
+ * Used by post-deploy verification to page the owning client's PagerDuty
+ * service when a managed/enterprise org regresses after a deploy. Routing keys
+ * are per-client (see lib/orgs/managed-clients.ts) so each org's incident lands
+ * on its own service.
+ *
+ * `dedupKey` collapses repeated triggers for the same (deploy, org, reason)
+ * into a single incident across the verification poll loop, so a 5-minute
+ * window does not open five incidents.
+ */
+import { ErrorCategory, logInfo, logSystemError } from "@/lib/logging";
+
+const PAGERDUTY_ENQUEUE_URL = "https://events.pagerduty.com/v2/enqueue";
+
+/** Our internal P-levels mapped onto PagerDuty Events v2 severities. */
+export type PagerDutySeverity = "critical" | "error" | "warning" | "info";
+
+export type PagerDutyTrigger = {
+  routingKey: string;
+  dedupKey: string;
+  summary: string;
+  severity: PagerDutySeverity;
+  source: string;
+  customDetails?: Record<string, unknown>;
+};
+
+/**
+ * Fire a trigger event. Returns true on a 2xx from PagerDuty. Never throws:
+ * a paging failure must not break the verification request that called it.
+ */
+export async function triggerPagerDutyAlert(
+  event: PagerDutyTrigger
+): Promise<boolean> {
+  try {
+    const response = await fetch(PAGERDUTY_ENQUEUE_URL, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        routing_key: event.routingKey,
+        event_action: "trigger",
+        dedup_key: event.dedupKey,
+        payload: {
+          summary: event.summary,
+          severity: event.severity,
+          source: event.source,
+          custom_details: event.customDetails,
+        },
+      }),
+    });
+
+    if (!response.ok) {
+      logSystemError(
+        ErrorCategory.EXTERNAL_SERVICE,
+        "[PagerDuty] Trigger rejected",
+        new Error(`PagerDuty returned ${response.status}`),
+        { dedup_key: event.dedupKey, status: String(response.status) }
+      );
+      return false;
+    }
+
+    logInfo("[PagerDuty] Triggered alert", {
+      dedup_key: event.dedupKey,
+      severity: event.severity,
+    });
+    return true;
+  } catch (error) {
+    logSystemError(
+      ErrorCategory.EXTERNAL_SERVICE,
+      "[PagerDuty] Failed to send trigger",
+      error,
+      { dedup_key: event.dedupKey }
+    );
+    return false;
+  }
+}

From 12575bf08346d6226309c3d4e71e17e02162d843 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Tue, 30 Jun 2026 21:35:17 -0300
Subject: [PATCH 07/10] build(deploy): inject managed-orgs config and pagerduty
 key from Parameter Store

Add MANAGED_ORGS_CONFIG + DEFAULT_PAGERDUTY_ROUTING_KEY to the app env and
MANAGED_ORGS_CONFIG to the metrics-collector env in the keeperhub-stack values
(prod + staging). Runtime env only -- no Docker build arg needed since these are
not NEXT_PUBLIC and are read at request time.
---
 deploy/keeperhub-stack/prod/values.yaml    | 20 ++++++++++++++++++++
 deploy/keeperhub-stack/staging/values.yaml | 20 ++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/deploy/keeperhub-stack/prod/values.yaml b/deploy/keeperhub-stack/prod/values.yaml
index 9e0d9d04b..da9deda7e 100644
--- a/deploy/keeperhub-stack/prod/values.yaml
+++ b/deploy/keeperhub-stack/prod/values.yaml
@@ -481,6 +481,19 @@ app:
       type: parameterStore
       name: kh-admin-secret
       parameter_name: /eks/techops-prod/keeperhub/kh-admin-secret
+    # Managed-client cohort + per-client PagerDuty routing. Single source of
+    # truth (JSON dict {"<slug>":{"pagerdutyRoutingKey":"..."}}) read by the
+    # post-deploy verification route; keeps client slugs out of the source.
+    MANAGED_ORGS_CONFIG:
+      type: parameterStore
+      name: managed-orgs
+      parameter_name: /eks/techops-prod/keeperhub/managed-orgs
+    # Fallback PagerDuty routing key for enterprise-plan orgs without a
+    # dedicated service entry in MANAGED_ORGS_CONFIG.
+    DEFAULT_PAGERDUTY_ROUTING_KEY:
+      type: parameterStore
+      name: default-pagerduty-routing-key
+      parameter_name: /eks/techops-prod/keeperhub/default-pagerduty-routing-key
 
   externalSecrets:
     clusterSecretStoreName: techops-prod
@@ -1074,6 +1087,13 @@ metricsCollector:
     NODE_ENV:
       type: kv
       value: "production"
+    # Managed-client slugs (same dict the app reads) so the per-workflow
+    # managed-client error gauge stays scoped. No PagerDuty key here -- the
+    # collector never pages.
+    MANAGED_ORGS_CONFIG:
+      type: parameterStore
+      name: managed-orgs
+      parameter_name: /eks/techops-prod/keeperhub/managed-orgs
 
   externalSecrets:
     clusterSecretStoreName: techops-prod
diff --git a/deploy/keeperhub-stack/staging/values.yaml b/deploy/keeperhub-stack/staging/values.yaml
index 1d1c7df6f..854493039 100644
--- a/deploy/keeperhub-stack/staging/values.yaml
+++ b/deploy/keeperhub-stack/staging/values.yaml
@@ -483,6 +483,19 @@ app:
       type: parameterStore
       name: kh-admin-secret
       parameter_name: /eks/techops-staging/keeperhub/kh-admin-secret
+    # Managed-client cohort + per-client PagerDuty routing. Single source of
+    # truth (JSON dict {"<slug>":{"pagerdutyRoutingKey":"..."}}) read by the
+    # post-deploy verification route; keeps client slugs out of the source.
+    MANAGED_ORGS_CONFIG:
+      type: parameterStore
+      name: managed-orgs
+      parameter_name: /eks/techops-staging/keeperhub/managed-orgs
+    # Fallback PagerDuty routing key for enterprise-plan orgs without a
+    # dedicated service entry in MANAGED_ORGS_CONFIG.
+    DEFAULT_PAGERDUTY_ROUTING_KEY:
+      type: parameterStore
+      name: default-pagerduty-routing-key
+      parameter_name: /eks/techops-staging/keeperhub/default-pagerduty-routing-key
 
   externalSecrets:
     clusterSecretStoreName: techops-staging
@@ -1076,6 +1089,13 @@ metricsCollector:
     NODE_ENV:
       type: kv
       value: "staging"
+    # Managed-client slugs (same dict the app reads) so the per-workflow
+    # managed-client error gauge stays scoped. No PagerDuty key here -- the
+    # collector never pages.
+    MANAGED_ORGS_CONFIG:
+      type: parameterStore
+      name: managed-orgs
+      parameter_name: /eks/techops-staging/keeperhub/managed-orgs
 
   externalSecrets:
     clusterSecretStoreName: techops-staging

From 7598c295baf8e0dfa19b981c74c669ced39aed5b Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Mon, 13 Jul 2026 19:12:50 -0300
Subject: [PATCH 08/10] fix(ci): clamp post-deploy verification query params to
 safe ranges

---
 .../post-deploy-verification/route.ts         | 30 ++++++++++++++-----
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
index 42e2d78c6..764d2a722 100644
--- a/app/api/internal/post-deploy-verification/route.ts
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -92,12 +92,20 @@ type OrgVerification = {
   sampleErrors: string[];
 };
 
-function parseNumberParam(value: string | null, fallback: number): number {
-  if (!value) {
-    return fallback;
+function parseNumberParam(
+  value: string | null,
+  fallback: number,
+  bounds?: { min?: number; max?: number }
+): number {
+  const parsed = value ? Number.parseFloat(value) : Number.NaN;
+  let result = Number.isFinite(parsed) ? parsed : fallback;
+  if (bounds?.min !== undefined) {
+    result = Math.max(bounds.min, result);
   }
-  const parsed = Number.parseFloat(value);
-  return Number.isFinite(parsed) ? parsed : fallback;
+  if (bounds?.max !== undefined) {
+    result = Math.min(bounds.max, result);
+  }
+  return result;
 }
 
 async function resolveTargetOrgs(
@@ -254,15 +262,21 @@ export async function GET(request: Request): Promise<NextResponse> {
   const url = new URL(request.url);
   const lookbackMinutes = parseNumberParam(
     url.searchParams.get("lookbackMinutes"),
-    DEFAULT_LOOKBACK_MINUTES
+    DEFAULT_LOOKBACK_MINUTES,
+    { min: 1 }
   );
+  // Clamp so an out-of-range value can't widen the check: minExecutions < 1
+  // would make the error-rate branch evaluate on every org, and a maxErrorRate
+  // outside [0,1] is meaningless for a ratio.
   const minExecutions = parseNumberParam(
     url.searchParams.get("minExecutions"),
-    DEFAULT_MIN_EXECUTIONS
+    DEFAULT_MIN_EXECUTIONS,
+    { min: 1 }
   );
   const maxErrorRate = parseNumberParam(
     url.searchParams.get("maxErrorRate"),
-    DEFAULT_MAX_ERROR_RATE
+    DEFAULT_MAX_ERROR_RATE,
+    { min: 0, max: 1 }
   );
 
   // `final=1` on the CI loop's last poll: only then can "no executions in the

From 3d9087555ac0fe3aa2669a04f8e718170ff360e5 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Mon, 13 Jul 2026 19:15:57 -0300
Subject: [PATCH 09/10] fix(ci): fail post-deploy verification only on
 confirmed problems, warn on no signal

---
 .../workflows/post-deploy-verification.yml    | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml
index 85e208399..55b88b05e 100644
--- a/.github/workflows/post-deploy-verification.yml
+++ b/.github/workflows/post-deploy-verification.yml
@@ -183,22 +183,25 @@ jobs:
             echo "problems=$PROBLEMS"
           } >> "$GITHUB_OUTPUT"
 
-      - name: Fail on verification problems
-        if: always() && steps.verify.outputs.ok != 'true'
+      # A confirmed problem (ok=false) fails the job red; the endpoint has
+      # already paged the affected clients via PagerDuty. Paging is server-side,
+      # so this step carries no slugs or routing keys -- just aggregate counts;
+      # per-org detail lives in PagerDuty and internal logs.
+      - name: Fail on confirmed problems
+        if: always() && steps.verify.outputs.ok == 'false'
         env:
           CHECKED: ${{ steps.verify.outputs.checked }}
           PROBLEMS: ${{ steps.verify.outputs.problems }}
         run: |
-          # Paging is done server-side by the endpoint (per-client PagerDuty
-          # routing), so the CI job carries no slugs or routing keys. This step
-          # just surfaces a red job + aggregate counts; the per-org detail lives
-          # in PagerDuty and internal logs (Loki: "post-deploy verification
-          # flagged ...").
-          if [ -n "${PROBLEMS:-}" ] && [ "${PROBLEMS}" != "?" ]; then
-            echo "Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) on the new build. Paged via PagerDuty; details in internal logs."
-          else
-            echo "Post-deploy verification could not complete. See the Actions run and internal logs."
-          fi
-
+          echo "Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) on the new build. Paged via PagerDuty; details in internal logs."
           echo "::error::Post-deploy verification flagged managed/enterprise org problems (count only; PagerDuty + internal logs have specifics)."
           exit 1
+
+      # No signal (ok=error): the check never got a parseable response -- a
+      # transient cluster/image-pull blip, or (pre-KEEP-834) OIDC not yet
+      # authorized to run in-cluster. That is not a confirmed regression and
+      # pages nobody, so warn instead of failing the deploy verification red.
+      - name: Warn when verification could not complete
+        if: always() && steps.verify.outputs.ok == 'error'
+        run: |
+          echo "::warning::Post-deploy verification could not reach the cluster or got no signal; nothing was confirmed and no client was paged. See the Actions run and internal logs."

From 2a41f3362d1c24580a8d34b72d24ea66cb26b446 Mon Sep 17 00:00:00 2001
From: joelorzet <joelorzt@gmail.com>
Date: Mon, 13 Jul 2026 19:23:00 -0300
Subject: [PATCH 10/10] refactor(ci): select cohort directly and judge on
 executions; no-run P2 only when a schedule was due

---
 .../post-deploy-verification/route.ts         | 102 ++++++++++++------
 1 file changed, 67 insertions(+), 35 deletions(-)

diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts
index 764d2a722..24ef3868f 100644
--- a/app/api/internal/post-deploy-verification/route.ts
+++ b/app/api/internal/post-deploy-verification/route.ts
@@ -11,17 +11,23 @@
  * On a flagged org this pages that client's own PagerDuty service (per-client
  * routing key, env sourced), deduped to one incident per org per deploy.
  *
- * The cohort is gated to orgs that actually have work due in the window: an
- * enabled, non-deleted, non-deactivated workflow with an enabled schedule whose
- * next run falls inside [`since`, `until`]. Idle orgs (no live workflow, or
- * schedules that won't fire in the window) are excluded so a deploy is not
- * judged against orgs that were never going to run. Execution counting covers
- * every trigger type (scheduled, block, webhook, event, manual).
+ * The cohort is the managed + enterprise set, selected directly. Judgement is
+ * on what actually executed in the window (every trigger type: scheduled,
+ * block, webhook, event, manual), NOT on a future schedule -- so a run that
+ * errors is caught even after its schedule's nextRunAt has advanced out of the
+ * window. The schedule is consulted only to decide whether a run was EXPECTED
+ * (for the no-executions case).
  *
  * An org is flagged as a problem when:
- *   - it produced any `system_error` executions in the window (platform/infra
- *     faults are the strongest signal of a deploy regression), OR
- *   - it ran at least `minExecutions` and its error rate exceeds `maxErrorRate`.
+ *   - it had a schedule due in the window (`nextRunAt` in [`since`, `until`])
+ *     and produced no executions -- a silent no-run (P2), OR
+ *   - it produced any `system_error` executions (P2), OR
+ *   - it ran at least `minExecutions` and its error rate exceeds `maxErrorRate`
+ *     (P3).
+ *
+ * An org whose next scheduled run is after the window (or that has no due
+ * schedule) and produced nothing is out of context: it passes green rather than
+ * failing the pipeline.
  *
  * This is a stateless single check (one query per request); the CI job owns the
  * timing and polls it. Window and thresholds are query-param overridable so the
@@ -74,6 +80,8 @@ type TargetOrg = {
   slug: string;
   name: string;
   plan: string;
+  /** True when the org had an enabled schedule due to fire in the window. */
+  dueInWindow: boolean;
 };
 
 type OrgVerification = {
@@ -86,6 +94,8 @@ type OrgVerification = {
   userErrors: number;
   systemErrors: number;
   errorRate: number;
+  /** Was a scheduled run expected in the window (drives the no-executions P2). */
+  dueInWindow: boolean;
   isProblem: boolean;
   severity: AlertSeverity | null;
   reasons: string[];
@@ -112,16 +122,14 @@ async function resolveTargetOrgs(
   windowStart: Date,
   windowEnd: Date
 ): Promise<TargetOrg[]> {
-  // Only verify orgs that actually have something due to run during the
-  // monitoring window: an enabled, non-deleted, non-deactivated workflow with
-  // an enabled schedule whose next run lands inside [windowStart, windowEnd].
-  // Without this an idle org (no live workflow, or schedules that won't fire in
-  // our window) is either dead weight or a false-positive source.
-  //
-  // Cohort = managed slugs (env-sourced) plus any enterprise-plan org. When no
-  // managed slugs are configured we fall back to enterprise-plan orgs only.
+  // Cohort = managed slugs (env-sourced) plus any enterprise-plan org, selected
+  // DIRECTLY -- not gated on a schedule. Errors are then judged on what actually
+  // executed (any trigger type), so a run that errors is caught even after its
+  // schedule's nextRunAt has advanced out of the window, and webhook/event/
+  // manual-only orgs are covered too. When no managed slugs are configured the
+  // cohort is enterprise-plan orgs only.
   const managedSlugs = getManagedOrgSlugs();
-  const rows = await db
+  const cohort = await db
     .selectDistinct({
       id: organization.id,
       slug: organization.slug,
@@ -133,11 +141,6 @@ async function resolveTargetOrgs(
       organizationSubscriptions,
       eq(organizationSubscriptions.organizationId, organization.id)
     )
-    .innerJoin(workflows, eq(workflows.organizationId, organization.id))
-    .innerJoin(
-      workflowSchedules,
-      eq(workflowSchedules.workflowId, workflows.id)
-    )
     .where(
       and(
         isNull(organization.deactivatedAt),
@@ -146,7 +149,29 @@ async function resolveTargetOrgs(
               inArray(organization.slug, managedSlugs),
               eq(organizationSubscriptions.plan, "enterprise")
             )
-          : eq(organizationSubscriptions.plan, "enterprise"),
+          : eq(organizationSubscriptions.plan, "enterprise")
+      )
+    );
+
+  if (cohort.length === 0) {
+    return [];
+  }
+
+  // Which cohort orgs had an enabled schedule due to fire in the window (an
+  // enabled, non-deleted, non-deactivated workflow whose nextRunAt is in
+  // [windowStart, windowEnd])? Only these are eligible for the "no executions"
+  // P2 -- an org whose next run is after the window is out of context and, if
+  // it produced nothing, passes green rather than failing the pipeline. A miss
+  // is caught precisely: if the run never happened nextRunAt has not advanced,
+  // so it stays in-window here.
+  const cohortIds = cohort.map((org) => org.id);
+  const dueRows = await db
+    .selectDistinct({ organizationId: workflows.organizationId })
+    .from(workflowSchedules)
+    .innerJoin(workflows, eq(workflows.id, workflowSchedules.workflowId))
+    .where(
+      and(
+        inArray(workflows.organizationId, cohortIds),
         eq(workflows.enabled, true),
         isNull(workflows.deletedAt),
         isNull(workflows.deactivatedAt),
@@ -155,12 +180,14 @@ async function resolveTargetOrgs(
         lte(workflowSchedules.nextRunAt, windowEnd)
       )
     );
+  const dueOrgIds = new Set(dueRows.map((row) => row.organizationId));
 
-  return rows.map((row) => ({
+  return cohort.map((row) => ({
     id: row.id,
     slug: row.slug,
     name: row.name,
     plan: row.plan ?? "free",
+    dueInWindow: dueOrgIds.has(row.id),
   }));
 }
 
@@ -290,10 +317,10 @@ export async function GET(request: Request): Promise<NextResponse> {
 
   // Stateless single check: one query per request. The CI job owns the timing
   // (it polls this endpoint). `since`/`until` (unix seconds) bound the
-  // monitoring window the caller is watching: executions are counted from
-  // `since` (so a poll sees only runs on the new build), and the cohort is
-  // gated to orgs with a schedule firing within [`since`, `until`]. Both fall
-  // back to the backward lookback window when omitted.
+  // monitoring window: executions are counted from `since` (so a poll sees only
+  // runs on the new build), and a schedule due within [`since`, `until`] marks
+  // an org as expected-to-run (the no-executions P2). Both fall back to the
+  // backward lookback window when omitted.
   const sinceParam = url.searchParams.get("since");
   const sinceEpoch = sinceParam ? Number.parseInt(sinceParam, 10) : Number.NaN;
   const since = Number.isFinite(sinceEpoch)
@@ -371,14 +398,18 @@ export async function GET(request: Request): Promise<NextResponse> {
       const reasons: string[] = [];
       let severity: AlertSeverity | null = null;
 
-      // No executions: the org had a schedule due in the window but produced
-      // nothing -- its automation silently did not run. Only conclusive on the
-      // final poll. P2.
-      if (isFinalPoll && stats.total === 0) {
-        reasons.push("no executions in the monitoring window");
+      // No executions: only when a run was actually EXPECTED (a schedule was due
+      // in the window) and the org produced nothing -- its automation silently
+      // did not run. Only conclusive on the final poll. P2. An org whose next
+      // run is after the window is out of context: it stays green here.
+      if (isFinalPoll && org.dueInWindow && stats.total === 0) {
+        reasons.push(
+          "no executions in the monitoring window (a schedule was due)"
+        );
         severity = "P2";
       }
-      // Platform/infra faults are the strongest signal of a deploy regression. P2.
+      // Platform/infra faults are the strongest signal of a deploy regression,
+      // judged on what actually ran (any trigger type). P2.
       if (stats.systemErrors > 0) {
         reasons.push(`${stats.systemErrors} system_error execution(s)`);
         severity = "P2";
@@ -404,6 +435,7 @@ export async function GET(request: Request): Promise<NextResponse> {
         userErrors: stats.userErrors,
         systemErrors: stats.systemErrors,
         errorRate,
+        dueInWindow: org.dueInWindow,
         isProblem: reasons.length > 0,
         severity,
         reasons,