diff --git a/.env.example b/.env.example index 046938fc2..75d406d6e 100644 --- a/.env.example +++ b/.env.example @@ -48,6 +48,15 @@ SENTRY_ORG= SENTRY_PROJECT= SENTRY_AUTH_TOKEN= +# Managed-client cohort + per-client PagerDuty routing. In deployed envs this is +# injected from AWS Parameter Store so client slugs and routing keys never live +# in source. JSON dict keyed by org slug; empty/unset => no managed cohort. +# Example: {"my-client":{"pagerdutyRoutingKey":"R0..."}} +MANAGED_ORGS_CONFIG= +# Fallback PagerDuty Events v2 routing key for enterprise-plan orgs without a +# dedicated entry in MANAGED_ORGS_CONFIG. Unset => those orgs are not paged. +DEFAULT_PAGERDUTY_ROUTING_KEY= + # Internal service-to-service auth (HMAC). AGENTIC_WALLET_HMAC_KMS_KEY is the # 32-byte base64 AES key the app uses to encrypt/decrypt the shared signing # secret in the internal_service_hmac_secrets store; it must be set for internal diff --git a/.github/workflows/post-deploy-verification.yml b/.github/workflows/post-deploy-verification.yml new file mode 100644 index 000000000..55b88b05e --- /dev/null +++ b/.github/workflows/post-deploy-verification.yml @@ -0,0 +1,207 @@ +# Type: standalone | Triggered by the CI Pipeline finishing successfully. +# +# This is a SEPARATE pipeline from the deploy. It fires only after the entire +# "CI Pipeline" run (build -> e2e -> deploy -> release -> docs-sync) completes +# with conclusion == success on prod -- not after any single step. It then +# verifies that managed + enterprise orgs are executing workflows correctly on +# the just-shipped build. +# +# It runs INSIDE the cluster: the /api/internal/* routes are HMAC-authed and +# WAF-blocked through the public hostname, so it signs the request the same way +# deploy/scripts/digest-cron.sh does and curls the in-pod service URL via a +# one-shot pod -- mirroring the deploy health check. +name: post-deploy-verification + +on: + workflow_run: + workflows: ["CI Pipeline"] + types: + - completed + +permissions: + contents: read + id-token: write + +# One verification per target at a time. cancel-in-progress is false so a second +# pipeline's verification queues rather than killing an in-flight check. +concurrency: + group: post-deploy-verification-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + verify: + # Only when the whole CI Pipeline went green on prod. A failed/cancelled CI + # run, or a successful run on any other branch, is skipped entirely. + if: > + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == 'prod' + runs-on: ubuntu-latest + environment: prod + env: + REGION: ${{ vars.TO_REGION }} + CLUSTER_NAME: ${{ vars.TO_CLUSTER_NAME }} + NAMESPACE: keeperhub + SERVICE_NAME: keeperhub + SSM_ENV_PREFIX: techops-prod + # The job polls the stateless check endpoint once a minute for at most + # MONITOR_MINUTES, watching only executions on the new build (anchored at + # the loop start). It stops early the moment an org is flagged, so the + # common green path costs ~1 poll, and the worst case is capped at 5 min + # of runner time. An org is flagged when it has any system_error, or when + # it ran at least MIN_EXECUTIONS and its error rate exceeds MAX_ERROR_RATE. + MONITOR_MINUTES: "5" + POLL_INTERVAL_SECONDS: "60" + MIN_EXECUTIONS: "1" + MAX_ERROR_RATE: "0.5" + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ env.REGION }} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig --name "${CLUSTER_NAME}" --region "${REGION}" + + - name: Fetch internal-service HMAC secret + run: | + SECRET=$(aws ssm get-parameter \ + --name "/eks/${SSM_ENV_PREFIX}/keeperhub/internal-service-hmac-secret" \ + --with-decryption --query "Parameter.Value" --output text \ + --region "${REGION}") + echo "::add-mask::$SECRET" + echo "INTERNAL_SERVICE_HMAC_SECRET=$SECRET" >> "$GITHUB_ENV" + + - name: Monitor the new build (in-cluster, poll once a minute) + id: verify + env: + # Scopes the endpoint's PagerDuty dedup key so every poll of this + # deploy collapses into one incident per org. + DEPLOY_ID: ${{ github.event.workflow_run.head_sha }} + run: | + set -uo pipefail + + CALLER="scheduler" + METHOD="GET" + PATHNAME="/api/internal/post-deploy-verification" + + # Anchor the window at the loop start so every poll counts only + # executions that began after the deploy -- i.e. runs on the new build. + SINCE=$(date +%s) + DEADLINE=$(( SINCE + MONITOR_MINUTES * 60 )) + + ATTEMPT=0 + PROBLEM_FOUND=0 + GOT_SIGNAL=0 + CHECKED="?" + PROBLEMS="?" + + while true; do + ATTEMPT=$(( ATTEMPT + 1 )) + + # Re-sign per poll: X-KH-Timestamp must be within the verifier's + # replay window. The verifier signs over url.pathname only, so the + # query (since/thresholds) does not affect the signature. + TIMESTAMP=$(date +%s) + BODY_DIGEST=$(printf '' | openssl dgst -sha256 | awk '{print $2}') + SIGNING_STRING=$(printf '%s\n%s\n%s\n%s\n%s' "$METHOD" "$PATHNAME" "$CALLER" "$BODY_DIGEST" "$TIMESTAMP") + SIGNATURE=$(printf '%s' "$SIGNING_STRING" | openssl dgst -sha256 -hmac "$INTERNAL_SERVICE_HMAC_SECRET" | awk '{print $2}') + + # Mark the last poll: only then can the endpoint conclude "no + # executions in the window" (a due schedule may fire at the end). + if [ $(( $(date +%s) + POLL_INTERVAL_SECONDS )) -ge "$DEADLINE" ]; then + FINAL=1 + else + FINAL=0 + fi + + URL="http://${SERVICE_NAME}-common.${NAMESPACE}.svc.cluster.local:3000${PATHNAME}?since=${SINCE}&until=${DEADLINE}&final=${FINAL}&deployId=${DEPLOY_ID}&minExecutions=${MIN_EXECUTIONS}&maxErrorRate=${MAX_ERROR_RATE}" + + # The secret never enters the pod -- only the precomputed signature does. + RESPONSE=$(kubectl run "post-deploy-verify-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${ATTEMPT}" \ + --namespace "${NAMESPACE}" \ + --restart=Never \ + --attach \ + --rm \ + --quiet \ + --image=curlimages/curl:8.11.0 \ + --env="URL=${URL}" \ + --env="CALLER=${CALLER}" \ + --env="TIMESTAMP=${TIMESTAMP}" \ + --env="SIGNATURE=${SIGNATURE}" \ + --command -- sh -c ' + curl -sS --max-time 30 \ + -H "X-KH-Caller: ${CALLER}" \ + -H "X-KH-Timestamp: ${TIMESTAMP}" \ + -H "X-KH-Signature: ${SIGNATURE}" \ + "$URL" + ' 2>/dev/null) || true + + # PUBLIC repo: Actions logs are world-readable. Surface only + # non-identifying aggregates. Per-org slugs/names/error text stay in + # internal logs (Loki); the raw endpoint error is never echoed. + OK=$(printf '%s' "$RESPONSE" | jq -r '.ok // empty' 2>/dev/null) + PC=$(printf '%s' "$RESPONSE" | jq -r '.problemCount // empty' 2>/dev/null) + CO=$(printf '%s' "$RESPONSE" | jq -r '.checkedOrgs // empty' 2>/dev/null) + TE=$(printf '%s' "$RESPONSE" | jq -r '.totalExecutions // empty' 2>/dev/null) + + if [ -n "$OK" ]; then + GOT_SIGNAL=1 + CHECKED="$CO" + PROBLEMS="$PC" + echo "Attempt ${ATTEMPT}: ok=${OK}, checkedOrgs=${CO}, problemCount=${PC}, totalExecutions=${TE}" + if [ "$OK" != "true" ]; then + PROBLEM_FOUND=1 + echo "Problem detected on the new build; stopping monitoring early." + break + fi + else + echo "Attempt ${ATTEMPT}: no valid response (transient); will retry." + fi + + if [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "Monitored ${MONITOR_MINUTES}m with no problems." + break + fi + sleep "${POLL_INTERVAL_SECONDS}" + done + + if [ "$PROBLEM_FOUND" -eq 1 ]; then + RESULT="false" + elif [ "$GOT_SIGNAL" -eq 1 ]; then + RESULT="true" + else + # Never got a parseable response in the whole window -- cannot + # confirm health, so do not greenlight. + RESULT="error" + fi + + { + echo "ok=$RESULT" + echo "checked=$CHECKED" + echo "problems=$PROBLEMS" + } >> "$GITHUB_OUTPUT" + + # A confirmed problem (ok=false) fails the job red; the endpoint has + # already paged the affected clients via PagerDuty. Paging is server-side, + # so this step carries no slugs or routing keys -- just aggregate counts; + # per-org detail lives in PagerDuty and internal logs. + - name: Fail on confirmed problems + if: always() && steps.verify.outputs.ok == 'false' + env: + CHECKED: ${{ steps.verify.outputs.checked }} + PROBLEMS: ${{ steps.verify.outputs.problems }} + run: | + echo "Flagged ${PROBLEMS} of ${CHECKED} managed/enterprise org(s) on the new build. Paged via PagerDuty; details in internal logs." + echo "::error::Post-deploy verification flagged managed/enterprise org problems (count only; PagerDuty + internal logs have specifics)." + exit 1 + + # No signal (ok=error): the check never got a parseable response -- a + # transient cluster/image-pull blip, or (pre-KEEP-834) OIDC not yet + # authorized to run in-cluster. That is not a confirmed regression and + # pages nobody, so warn instead of failing the deploy verification red. + - name: Warn when verification could not complete + if: always() && steps.verify.outputs.ok == 'error' + run: | + echo "::warning::Post-deploy verification could not reach the cluster or got no signal; nothing was confirmed and no client was paged. See the Actions run and internal logs." diff --git a/app/api/internal/post-deploy-verification/route.ts b/app/api/internal/post-deploy-verification/route.ts new file mode 100644 index 000000000..24ef3868f --- /dev/null +++ b/app/api/internal/post-deploy-verification/route.ts @@ -0,0 +1,522 @@ +/** + * Post-deployment verification for managed and enterprise organizations. + * + * Reads recent workflow execution outcomes for the white-glove cohort (the + * env-sourced managed slugs from getManagedOrgSlugs() plus any org on an + * `enterprise` subscription) and reports whether any of them regressed after a + * deploy. Authenticated as an internal service (HMAC) and invoked from the + * post-deploy-verification CI job, which runs inside the cluster against the + * in-pod service URL (the public hostname is WAF-blocked for /api/internal/*). + * + * On a flagged org this pages that client's own PagerDuty service (per-client + * routing key, env sourced), deduped to one incident per org per deploy. + * + * The cohort is the managed + enterprise set, selected directly. Judgement is + * on what actually executed in the window (every trigger type: scheduled, + * block, webhook, event, manual), NOT on a future schedule -- so a run that + * errors is caught even after its schedule's nextRunAt has advanced out of the + * window. The schedule is consulted only to decide whether a run was EXPECTED + * (for the no-executions case). + * + * An org is flagged as a problem when: + * - it had a schedule due in the window (`nextRunAt` in [`since`, `until`]) + * and produced no executions -- a silent no-run (P2), OR + * - it produced any `system_error` executions (P2), OR + * - it ran at least `minExecutions` and its error rate exceeds `maxErrorRate` + * (P3). + * + * An org whose next scheduled run is after the window (or that has no due + * schedule) and produced nothing is out of context: it passes green rather than + * failing the pipeline. + * + * This is a stateless single check (one query per request); the CI job owns the + * timing and polls it. Window and thresholds are query-param overridable so the + * job can tune them without a redeploy: `since`/`until` (unix seconds, the + * window bounds), `lookbackMinutes` (fallback when `since`/`until` are absent), + * `minExecutions`, and `maxErrorRate`. + */ +import { and, desc, eq, gte, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { NextResponse } from "next/server"; +import { + type PagerDutySeverity, + triggerPagerDutyAlert, +} from "@/lib/alerting/pagerduty"; +import { db } from "@/lib/db"; +import { + organization, + organizationSubscriptions, + workflowExecutions, + workflowSchedules, + workflows, +} from "@/lib/db/schema"; +import { ERROR_STATUSES } from "@/lib/errors/execution-status"; +import { HttpStatus } from "@/lib/http-status"; +import { authenticateInternalService } from "@/lib/internal-service-auth"; +import { ErrorCategory, logSystemError, logSystemWarn } from "@/lib/logging"; +import { + getManagedOrgSlugs, + getPagerDutyRoutingKey, +} from "@/lib/orgs/managed-clients"; + +const DEFAULT_LOOKBACK_MINUTES = 60; +const DEFAULT_MIN_EXECUTIONS = 1; +const DEFAULT_MAX_ERROR_RATE = 0.5; +const SAMPLE_ERRORS_PER_ORG = 3; +const MAX_SAMPLE_ERROR_LENGTH = 300; +const PAGERDUTY_SOURCE = "post-deploy-verification"; + +// Internal P-levels. A missed execution or a platform (system_error) fault is a +// P2 (a managed/enterprise client's automation silently broke); a user-error +// rate spike is a P3 (more likely the client's own input/config than the +// deploy). PagerDuty severity: P2 -> critical, P3 -> warning. +type AlertSeverity = "P2" | "P3"; + +function pagerDutySeverity(severity: AlertSeverity): PagerDutySeverity { + return severity === "P2" ? "critical" : "warning"; +} + +type TargetOrg = { + id: string; + slug: string; + name: string; + plan: string; + /** True when the org had an enabled schedule due to fire in the window. */ + dueInWindow: boolean; +}; + +type OrgVerification = { + organizationId: string; + slug: string; + name: string; + plan: string; + total: number; + success: number; + userErrors: number; + systemErrors: number; + errorRate: number; + /** Was a scheduled run expected in the window (drives the no-executions P2). */ + dueInWindow: boolean; + isProblem: boolean; + severity: AlertSeverity | null; + reasons: string[]; + sampleErrors: string[]; +}; + +function parseNumberParam( + value: string | null, + fallback: number, + bounds?: { min?: number; max?: number } +): number { + const parsed = value ? Number.parseFloat(value) : Number.NaN; + let result = Number.isFinite(parsed) ? parsed : fallback; + if (bounds?.min !== undefined) { + result = Math.max(bounds.min, result); + } + if (bounds?.max !== undefined) { + result = Math.min(bounds.max, result); + } + return result; +} + +async function resolveTargetOrgs( + windowStart: Date, + windowEnd: Date +): Promise { + // Cohort = managed slugs (env-sourced) plus any enterprise-plan org, selected + // DIRECTLY -- not gated on a schedule. Errors are then judged on what actually + // executed (any trigger type), so a run that errors is caught even after its + // schedule's nextRunAt has advanced out of the window, and webhook/event/ + // manual-only orgs are covered too. When no managed slugs are configured the + // cohort is enterprise-plan orgs only. + const managedSlugs = getManagedOrgSlugs(); + const cohort = await db + .selectDistinct({ + id: organization.id, + slug: organization.slug, + name: organization.name, + plan: organizationSubscriptions.plan, + }) + .from(organization) + .leftJoin( + organizationSubscriptions, + eq(organizationSubscriptions.organizationId, organization.id) + ) + .where( + and( + isNull(organization.deactivatedAt), + managedSlugs.length > 0 + ? or( + inArray(organization.slug, managedSlugs), + eq(organizationSubscriptions.plan, "enterprise") + ) + : eq(organizationSubscriptions.plan, "enterprise") + ) + ); + + if (cohort.length === 0) { + return []; + } + + // Which cohort orgs had an enabled schedule due to fire in the window (an + // enabled, non-deleted, non-deactivated workflow whose nextRunAt is in + // [windowStart, windowEnd])? Only these are eligible for the "no executions" + // P2 -- an org whose next run is after the window is out of context and, if + // it produced nothing, passes green rather than failing the pipeline. A miss + // is caught precisely: if the run never happened nextRunAt has not advanced, + // so it stays in-window here. + const cohortIds = cohort.map((org) => org.id); + const dueRows = await db + .selectDistinct({ organizationId: workflows.organizationId }) + .from(workflowSchedules) + .innerJoin(workflows, eq(workflows.id, workflowSchedules.workflowId)) + .where( + and( + inArray(workflows.organizationId, cohortIds), + eq(workflows.enabled, true), + isNull(workflows.deletedAt), + isNull(workflows.deactivatedAt), + eq(workflowSchedules.enabled, true), + gte(workflowSchedules.nextRunAt, windowStart), + lte(workflowSchedules.nextRunAt, windowEnd) + ) + ); + const dueOrgIds = new Set(dueRows.map((row) => row.organizationId)); + + return cohort.map((row) => ({ + id: row.id, + slug: row.slug, + name: row.name, + plan: row.plan ?? "free", + dueInWindow: dueOrgIds.has(row.id), + })); +} + +async function fetchSampleErrors( + problemOrgIds: string[], + since: Date +): Promise> { + const byOrg = new Map(); + if (problemOrgIds.length === 0) { + return byOrg; + } + + const rows = await db + .select({ + organizationId: workflows.organizationId, + error: workflowExecutions.error, + status: workflowExecutions.status, + }) + .from(workflowExecutions) + .innerJoin(workflows, eq(workflows.id, workflowExecutions.workflowId)) + .where( + and( + inArray(workflows.organizationId, problemOrgIds), + gte(workflowExecutions.startedAt, since), + inArray(workflowExecutions.status, [...ERROR_STATUSES]) + ) + ) + .orderBy(desc(workflowExecutions.startedAt)) + .limit(problemOrgIds.length * SAMPLE_ERRORS_PER_ORG * 4); + + for (const row of rows) { + const orgId = row.organizationId; + if (!orgId) { + continue; + } + const existing = byOrg.get(orgId) ?? []; + if (existing.length >= SAMPLE_ERRORS_PER_ORG) { + continue; + } + const message = (row.error ?? "(no error message)").slice( + 0, + MAX_SAMPLE_ERROR_LENGTH + ); + existing.push(`[${row.status}] ${message}`); + byOrg.set(orgId, existing); + } + + return byOrg; +} + +/** + * Page each flagged org's own PagerDuty service (per-client routing key, env + * sourced). The dedup key is per (deploy, org) so the verification poll loop + * collapses into one incident per org per deploy. Orgs without a routing key + * (and no default) are skipped -- they still surface in the response + Loki. + */ +async function pageProblemOrgs( + problems: OrgVerification[], + deployId: string, + windowStart: string +): Promise { + await Promise.all( + problems.map((org) => { + const routingKey = getPagerDutyRoutingKey(org.slug); + if (!(routingKey && org.severity)) { + return Promise.resolve(false); + } + return triggerPagerDutyAlert({ + routingKey, + dedupKey: `post-deploy/${deployId || windowStart}/${org.organizationId}`, + severity: pagerDutySeverity(org.severity), + source: PAGERDUTY_SOURCE, + summary: `[${org.severity}] Post-deploy regression for ${org.slug}: ${org.reasons.join("; ")}`, + customDetails: { + org_slug: org.slug, + org_name: org.name, + plan: org.plan, + total: org.total, + user_errors: org.userErrors, + system_errors: org.systemErrors, + reasons: org.reasons, + sample_errors: org.sampleErrors, + window_start: windowStart, + }, + }); + }) + ); +} + +export async function GET(request: Request): Promise { + const auth = await authenticateInternalService(request); + if (!auth.authenticated) { + return NextResponse.json( + { error: auth.error ?? "Unauthorized" }, + { status: auth.status } + ); + } + + const url = new URL(request.url); + const lookbackMinutes = parseNumberParam( + url.searchParams.get("lookbackMinutes"), + DEFAULT_LOOKBACK_MINUTES, + { min: 1 } + ); + // Clamp so an out-of-range value can't widen the check: minExecutions < 1 + // would make the error-rate branch evaluate on every org, and a maxErrorRate + // outside [0,1] is meaningless for a ratio. + const minExecutions = parseNumberParam( + url.searchParams.get("minExecutions"), + DEFAULT_MIN_EXECUTIONS, + { min: 1 } + ); + const maxErrorRate = parseNumberParam( + url.searchParams.get("maxErrorRate"), + DEFAULT_MAX_ERROR_RATE, + { min: 0, max: 1 } + ); + + // `final=1` on the CI loop's last poll: only then can "no executions in the + // window" be concluded (a due schedule may fire at the very end of the + // window). `deployId` (commit SHA) scopes the PagerDuty dedup key so all + // polls of one deploy collapse into a single incident per org. + const isFinalPoll = + url.searchParams.get("final") === "1" || + url.searchParams.get("final") === "true"; + const deployId = url.searchParams.get("deployId") ?? ""; + + // Stateless single check: one query per request. The CI job owns the timing + // (it polls this endpoint). `since`/`until` (unix seconds) bound the + // monitoring window: executions are counted from `since` (so a poll sees only + // runs on the new build), and a schedule due within [`since`, `until`] marks + // an org as expected-to-run (the no-executions P2). Both fall back to the + // backward lookback window when omitted. + const sinceParam = url.searchParams.get("since"); + const sinceEpoch = sinceParam ? Number.parseInt(sinceParam, 10) : Number.NaN; + const since = Number.isFinite(sinceEpoch) + ? new Date(sinceEpoch * 1000) + : new Date(Date.now() - lookbackMinutes * 60 * 1000); + + const untilParam = url.searchParams.get("until"); + const untilEpoch = untilParam ? Number.parseInt(untilParam, 10) : Number.NaN; + const until = Number.isFinite(untilEpoch) + ? new Date(untilEpoch * 1000) + : new Date(since.getTime() + lookbackMinutes * 60 * 1000); + + try { + const windowStart = since.toISOString(); + const windowEnd = until.toISOString(); + const targetOrgs = await resolveTargetOrgs(since, until); + + if (targetOrgs.length === 0) { + return NextResponse.json({ + ok: true, + windowStart, + windowEnd, + minExecutions, + maxErrorRate, + checkedOrgs: 0, + problemCount: 0, + totalExecutions: 0, + orgs: [], + problems: [], + generatedAt: new Date().toISOString(), + }); + } + + const orgIds = targetOrgs.map((org) => org.id); + const statsRows = await db + .select({ + organizationId: workflows.organizationId, + total: sql`COUNT(*)`, + success: sql`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'success')`, + userErrors: sql`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'error')`, + systemErrors: sql`COUNT(*) FILTER (WHERE ${workflowExecutions.status} = 'system_error')`, + }) + .from(workflowExecutions) + .innerJoin(workflows, eq(workflows.id, workflowExecutions.workflowId)) + .where( + and( + inArray(workflows.organizationId, orgIds), + gte(workflowExecutions.startedAt, since) + ) + ) + .groupBy(workflows.organizationId); + + const statsByOrg = new Map( + statsRows.map((row) => [ + row.organizationId, + { + total: Number(row.total), + success: Number(row.success), + userErrors: Number(row.userErrors), + systemErrors: Number(row.systemErrors), + }, + ]) + ); + + const orgs: OrgVerification[] = targetOrgs.map((org) => { + const stats = statsByOrg.get(org.id) ?? { + total: 0, + success: 0, + userErrors: 0, + systemErrors: 0, + }; + const errorCount = stats.userErrors + stats.systemErrors; + const errorRate = stats.total > 0 ? errorCount / stats.total : 0; + + const reasons: string[] = []; + let severity: AlertSeverity | null = null; + + // No executions: only when a run was actually EXPECTED (a schedule was due + // in the window) and the org produced nothing -- its automation silently + // did not run. Only conclusive on the final poll. P2. An org whose next + // run is after the window is out of context: it stays green here. + if (isFinalPoll && org.dueInWindow && stats.total === 0) { + reasons.push( + "no executions in the monitoring window (a schedule was due)" + ); + severity = "P2"; + } + // Platform/infra faults are the strongest signal of a deploy regression, + // judged on what actually ran (any trigger type). P2. + if (stats.systemErrors > 0) { + reasons.push(`${stats.systemErrors} system_error execution(s)`); + severity = "P2"; + } + // User-error rate spike: more likely the client's own input/config than + // the deploy. P3, and never downgrades an already-P2 org. + if (stats.total >= minExecutions && errorRate > maxErrorRate) { + reasons.push( + `error rate ${(errorRate * 100).toFixed(1)}% exceeds ${( + maxErrorRate * 100 + ).toFixed(1)}% over ${stats.total} execution(s)` + ); + severity = severity ?? "P3"; + } + + return { + organizationId: org.id, + slug: org.slug, + name: org.name, + plan: org.plan, + total: stats.total, + success: stats.success, + userErrors: stats.userErrors, + systemErrors: stats.systemErrors, + errorRate, + dueInWindow: org.dueInWindow, + isProblem: reasons.length > 0, + severity, + reasons, + sampleErrors: [], + }; + }); + + const problemOrgIds = orgs + .filter((org) => org.isProblem) + .map((org) => org.organizationId); + const sampleErrors = await fetchSampleErrors(problemOrgIds, since); + for (const org of orgs) { + if (org.isProblem) { + org.sampleErrors = sampleErrors.get(org.organizationId) ?? []; + } + } + + const problems = orgs.filter((org) => org.isProblem); + const totalExecutions = orgs.reduce((sum, org) => sum + org.total, 0); + + // The per-org detail (slugs, names, sample error text) is customer data and + // must not surface in the public-repo CI logs or an external Discord channel. + // Emit it to internal observability (Loki) instead, where operators look up + // specifics; the CI job only ever reports aggregate counts. + if (problems.length > 0) { + logSystemWarn( + ErrorCategory.WORKFLOW_ENGINE, + "Post-deploy verification flagged managed/enterprise org errors", + undefined, + { + problem_count: String(problems.length), + checked_orgs: String(orgs.length), + window_start: windowStart, + problems: JSON.stringify( + problems.map((org) => ({ + slug: org.slug, + plan: org.plan, + total: org.total, + userErrors: org.userErrors, + systemErrors: org.systemErrors, + reasons: org.reasons, + sampleErrors: org.sampleErrors, + })) + ), + } + ); + + // Page each flagged org's PagerDuty service. Never throws; a paging + // failure is logged but does not fail the verification response. + await pageProblemOrgs(problems, deployId, windowStart); + } + + return NextResponse.json({ + ok: problems.length === 0, + windowStart, + windowEnd, + minExecutions, + maxErrorRate, + checkedOrgs: orgs.length, + problemCount: problems.length, + totalExecutions, + orgs, + problems, + generatedAt: new Date().toISOString(), + }); + } catch (error) { + logSystemError( + ErrorCategory.DATABASE, + "Failed to run post-deploy verification", + error, + { endpoint: "/api/internal/post-deploy-verification", operation: "get" } + ); + return NextResponse.json( + { + ok: false, + error: + error instanceof Error + ? error.message + : "Failed to run post-deploy verification", + }, + { status: HttpStatus.INTERNAL_SERVER_ERROR } + ); + } +} diff --git a/deploy/keeperhub-stack/prod/values.yaml b/deploy/keeperhub-stack/prod/values.yaml index 9e0d9d04b..da9deda7e 100644 --- a/deploy/keeperhub-stack/prod/values.yaml +++ b/deploy/keeperhub-stack/prod/values.yaml @@ -481,6 +481,19 @@ app: type: parameterStore name: kh-admin-secret parameter_name: /eks/techops-prod/keeperhub/kh-admin-secret + # Managed-client cohort + per-client PagerDuty routing. Single source of + # truth (JSON dict {"":{"pagerdutyRoutingKey":"..."}}) read by the + # post-deploy verification route; keeps client slugs out of the source. + MANAGED_ORGS_CONFIG: + type: parameterStore + name: managed-orgs + parameter_name: /eks/techops-prod/keeperhub/managed-orgs + # Fallback PagerDuty routing key for enterprise-plan orgs without a + # dedicated service entry in MANAGED_ORGS_CONFIG. + DEFAULT_PAGERDUTY_ROUTING_KEY: + type: parameterStore + name: default-pagerduty-routing-key + parameter_name: /eks/techops-prod/keeperhub/default-pagerduty-routing-key externalSecrets: clusterSecretStoreName: techops-prod @@ -1074,6 +1087,13 @@ metricsCollector: NODE_ENV: type: kv value: "production" + # Managed-client slugs (same dict the app reads) so the per-workflow + # managed-client error gauge stays scoped. No PagerDuty key here -- the + # collector never pages. + MANAGED_ORGS_CONFIG: + type: parameterStore + name: managed-orgs + parameter_name: /eks/techops-prod/keeperhub/managed-orgs externalSecrets: clusterSecretStoreName: techops-prod diff --git a/deploy/keeperhub-stack/staging/values.yaml b/deploy/keeperhub-stack/staging/values.yaml index 1d1c7df6f..854493039 100644 --- a/deploy/keeperhub-stack/staging/values.yaml +++ b/deploy/keeperhub-stack/staging/values.yaml @@ -483,6 +483,19 @@ app: type: parameterStore name: kh-admin-secret parameter_name: /eks/techops-staging/keeperhub/kh-admin-secret + # Managed-client cohort + per-client PagerDuty routing. Single source of + # truth (JSON dict {"":{"pagerdutyRoutingKey":"..."}}) read by the + # post-deploy verification route; keeps client slugs out of the source. + MANAGED_ORGS_CONFIG: + type: parameterStore + name: managed-orgs + parameter_name: /eks/techops-staging/keeperhub/managed-orgs + # Fallback PagerDuty routing key for enterprise-plan orgs without a + # dedicated service entry in MANAGED_ORGS_CONFIG. + DEFAULT_PAGERDUTY_ROUTING_KEY: + type: parameterStore + name: default-pagerduty-routing-key + parameter_name: /eks/techops-staging/keeperhub/default-pagerduty-routing-key externalSecrets: clusterSecretStoreName: techops-staging @@ -1076,6 +1089,13 @@ metricsCollector: NODE_ENV: type: kv value: "staging" + # Managed-client slugs (same dict the app reads) so the per-workflow + # managed-client error gauge stays scoped. No PagerDuty key here -- the + # collector never pages. + MANAGED_ORGS_CONFIG: + type: parameterStore + name: managed-orgs + parameter_name: /eks/techops-staging/keeperhub/managed-orgs externalSecrets: clusterSecretStoreName: techops-staging diff --git a/lib/alerting/pagerduty.ts b/lib/alerting/pagerduty.ts new file mode 100644 index 000000000..fba97ea1f --- /dev/null +++ b/lib/alerting/pagerduty.ts @@ -0,0 +1,77 @@ +/** + * Minimal PagerDuty Events API v2 client (trigger-only). + * + * Used by post-deploy verification to page the owning client's PagerDuty + * service when a managed/enterprise org regresses after a deploy. Routing keys + * are per-client (see lib/orgs/managed-clients.ts) so each org's incident lands + * on its own service. + * + * `dedupKey` collapses repeated triggers for the same (deploy, org, reason) + * into a single incident across the verification poll loop, so a 5-minute + * window does not open five incidents. + */ +import { ErrorCategory, logInfo, logSystemError } from "@/lib/logging"; + +const PAGERDUTY_ENQUEUE_URL = "https://events.pagerduty.com/v2/enqueue"; + +/** Our internal P-levels mapped onto PagerDuty Events v2 severities. */ +export type PagerDutySeverity = "critical" | "error" | "warning" | "info"; + +export type PagerDutyTrigger = { + routingKey: string; + dedupKey: string; + summary: string; + severity: PagerDutySeverity; + source: string; + customDetails?: Record; +}; + +/** + * Fire a trigger event. Returns true on a 2xx from PagerDuty. Never throws: + * a paging failure must not break the verification request that called it. + */ +export async function triggerPagerDutyAlert( + event: PagerDutyTrigger +): Promise { + try { + const response = await fetch(PAGERDUTY_ENQUEUE_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + routing_key: event.routingKey, + event_action: "trigger", + dedup_key: event.dedupKey, + payload: { + summary: event.summary, + severity: event.severity, + source: event.source, + custom_details: event.customDetails, + }, + }), + }); + + if (!response.ok) { + logSystemError( + ErrorCategory.EXTERNAL_SERVICE, + "[PagerDuty] Trigger rejected", + new Error(`PagerDuty returned ${response.status}`), + { dedup_key: event.dedupKey, status: String(response.status) } + ); + return false; + } + + logInfo("[PagerDuty] Triggered alert", { + dedup_key: event.dedupKey, + severity: event.severity, + }); + return true; + } catch (error) { + logSystemError( + ErrorCategory.EXTERNAL_SERVICE, + "[PagerDuty] Failed to send trigger", + error, + { dedup_key: event.dedupKey } + ); + return false; + } +} diff --git a/lib/errors/classify.ts b/lib/errors/classify.ts index 3d62bf33b..fcca5afe1 100644 --- a/lib/errors/classify.ts +++ b/lib/errors/classify.ts @@ -16,9 +16,8 @@ import { ErrorCategory } from "@/lib/logging"; * null for user failures (which surface their raw message). * * The classifier is intentionally pattern-driven against real production - * messages observed for managed clients (Sky/Ajna) so the resulting - * `error_type` label on `workflow_executions` lets the SLI alert filter - * out user-config noise. + * messages observed for managed clients so the resulting `error_type` label on + * `workflow_executions` lets the SLI alert filter out user-config noise. * * Default for unmatched messages is WORKFLOW_ENGINE / errorType="system" / * code=DEFAULT_SYSTEM_ERROR_CODE. That defaults to "treat unknown as system" diff --git a/lib/errors/finalize-error.ts b/lib/errors/finalize-error.ts index 14585be34..e4f1d279e 100644 --- a/lib/errors/finalize-error.ts +++ b/lib/errors/finalize-error.ts @@ -15,7 +15,7 @@ import { ANONYMOUS_ORG_SLUG } from "@/lib/metrics/db-metrics"; * `workflow_executions` row has been written with status='error'. * * Resolves the org slug for the workflow so the counter series is scoped - * for the SLA alert (Sky/Ajna). Falls back to ANONYMOUS_ORG_SLUG for + * for the managed-client SLA alert. Falls back to ANONYMOUS_ORG_SLUG for * personal workflows so personal failures still emit a series. * * Safe to call after the DB write succeeded. Errors are caught and dropped diff --git a/lib/logging.ts b/lib/logging.ts index b82369e70..af1bd699f 100644 --- a/lib/logging.ts +++ b/lib/logging.ts @@ -134,7 +134,7 @@ function buildErrPayload( * Loki labels, not just as text inside the message string. * * The human-readable `msg` field retains the original message and the - * `[org:Sky][exec:xyz]` tag so `kubectl logs` greps still work. + * `[org:][exec:xyz]` tag so `kubectl logs` greps still work. */ function buildStructuredPayload(args: { message: string; diff --git a/lib/metrics/METRICS_REFERENCE.md b/lib/metrics/METRICS_REFERENCE.md index eba5ca4ca..7a1fa90d3 100644 --- a/lib/metrics/METRICS_REFERENCE.md +++ b/lib/metrics/METRICS_REFERENCE.md @@ -84,7 +84,7 @@ Error metrics tracking failures and exceptions. | Metric Name | Description | Labels | Target | Source | |-------------|-------------|--------|--------|--------| | `workflow.execution.errors` | Failed workflow executions | - | < 5% | DB | -| `workflow.errors.by_workflow` | Errored executions in the **last hour** (rolling 1h window), managed orgs only. Powers the Sky/Ajna managed-client error alerts; the alert reads it directly (no offset). Not cumulative. | `workflow_id`, `org_slug`, `error_type` | - | DB | +| `workflow.errors.by_workflow` | Errored executions in the **last hour** (rolling 1h window), managed orgs only. Powers the managed-client error alerts; the alert reads it directly (no offset). Not cumulative. | `workflow_id`, `org_slug`, `error_type` | - | DB | | `workflow.step.errors` | Failed step executions | `step_type` | < 10% | DB | | `plugin.action.errors` | Failed plugin actions | `plugin_name`, `action_name`, `error_type` | < 20% | API | | `api.errors.total` | API errors (webhook failures) | `endpoint`, `status_code`, `error_type` | count | API | @@ -374,7 +374,7 @@ max by (status) (keeperhub_workflow_executions_total{...}) sum by (status) (keeperhub_workflow_executions_total{...}) ``` -`keeperhub_workflow_executions_total` and `keeperhub_workflow_execution_errors_total` are labeled by `org_slug` so dashboards/alerts can scope to managed clients. Personal/anonymous workflows are emitted under `org_slug="_anonymous"` so the sum across `org_slug` for a given status equals the unfiltered per-status total. To filter to managed clients, add `org_slug=~"techops-services|ajna"` (or the inverse `!~` for user workflows). +`keeperhub_workflow_executions_total` and `keeperhub_workflow_execution_errors_total` are labeled by `org_slug` so dashboards/alerts can scope to managed clients. Personal/anonymous workflows are emitted under `org_slug="_anonymous"` so the sum across `org_slug` for a given status equals the unfiltered per-status total. To filter to managed clients, add `org_slug=~"$managed_slugs"` (or the inverse `!~` for user workflows). **Metrics requiring `max()` aggregation:** @@ -422,11 +422,11 @@ sum by (status) ( # Error rate over last hour, scoped to managed orgs 100 * sum(max by (org_slug) ( - delta(keeperhub_workflow_execution_errors_total{org_slug=~"techops-services|ajna"}[1h]) + delta(keeperhub_workflow_execution_errors_total{org_slug=~"$managed_slugs"}[1h]) )) / clamp_min( sum(max by (status, org_slug) ( - delta(keeperhub_workflow_executions_total{org_slug=~"techops-services|ajna"}[1h]) + delta(keeperhub_workflow_executions_total{org_slug=~"$managed_slugs"}[1h]) )), 1 ) diff --git a/lib/metrics/db-metrics.ts b/lib/metrics/db-metrics.ts index cb62659d4..df625cd51 100644 --- a/lib/metrics/db-metrics.ts +++ b/lib/metrics/db-metrics.ts @@ -50,6 +50,7 @@ import { } from "@/lib/db/schema"; import { ERROR_STATUSES } from "@/lib/errors/execution-status"; import { ErrorCategory, logSystemWarn } from "@/lib/logging"; +import { getManagedOrgSlugs } from "@/lib/orgs/managed-clients"; import type { BillingStatus } from "./types"; // Label value used for workflow executions whose workflow has no organization @@ -59,13 +60,10 @@ import type { BillingStatus } from "./types"; // workflows still produce a series rather than silently dropping increments. export const ANONYMOUS_ORG_SLUG = "_anonymous"; -// Org slugs for the managed clients (Sky, Ajna) whose per-workflow error series -// power the managed-client user-error alerts. The per-workflow gauge is scoped -// to these slugs so `workflow_id` never becomes an unbounded label across the -// whole user base — only managed workflows that have errored emit a series. -// Mirrors `local.managed_org_slugs_regex` in the infra Grafana alert config; -// adding a managed org requires updating both lists. -export const MANAGED_ORG_SLUGS = ["techops-services", "ajna"] as const; +// The per-workflow error gauge is scoped to the managed-client slugs (from +// `getManagedOrgSlugs()`, env-sourced) so `workflow_id` never becomes an +// unbounded label across the whole user base — only managed workflows that have +// errored emit a series. // Histogram bucket boundaries in milliseconds (must match prometheus.ts) const WORKFLOW_DURATION_BUCKETS = [ @@ -160,9 +158,7 @@ export async function getWorkflowStatsFromDb(): Promise { .from(workflowExecutions) .innerJoin(workflows, eq(workflowExecutions.workflowId, workflows.id)) .leftJoin(organization, eq(workflows.organizationId, organization.id)) - .where( - gte(workflowExecutions.startedAt, sql`now() - interval '30 days'`) - ) + .where(gte(workflowExecutions.startedAt, sql`now() - interval '30 days'`)) .groupBy( workflowExecutions.status, organization.slug, @@ -292,11 +288,18 @@ export type WorkflowErrorsByWorkflow = Array<{ * last hour keeps the row set tiny so the query is an index range scan on * idx_workflow_executions_error_completed_at and finishes in milliseconds. * - * Scoped to MANAGED_ORG_SLUGS to bound `workflow_id` cardinality. errorType is - * the `workflow_executions.error_type` column, projected to "unknown" for - * rows that predate classification so every series carries a populated label. + * Scoped to the managed-client slugs to bound `workflow_id` cardinality. + * errorType is the `workflow_executions.error_type` column, projected to + * "unknown" for rows that predate classification so every series carries a + * populated label. */ export async function getWorkflowErrorsByWorkflowFromDb(): Promise { + const managedSlugs = getManagedOrgSlugs(); + if (managedSlugs.length === 0) { + // No managed cohort configured -> emit no managed series (keeps the gauge + // bounded and avoids a pointless query). + return []; + } try { const rows = await db .select({ @@ -312,7 +315,7 @@ export async function getWorkflowErrorsByWorkflowFromDb(): Promise= now() - interval '1 hour'`, - inArray(organization.slug, [...MANAGED_ORG_SLUGS]) + inArray(organization.slug, managedSlugs) ) ) .groupBy( diff --git a/lib/orgs/managed-clients.ts b/lib/orgs/managed-clients.ts new file mode 100644 index 000000000..ca744b106 --- /dev/null +++ b/lib/orgs/managed-clients.ts @@ -0,0 +1,56 @@ +/** + * Managed-client cohort + alert routing, sourced from env -- never hardcoded. + * + * `MANAGED_ORGS_CONFIG` is a JSON object keyed by org slug, injected at deploy + * from AWS Parameter Store (SecureString) so client identities and their + * PagerDuty routing keys never live in source (this repo is public). Shape: + * {"":{"pagerdutyRoutingKey":""}} + * + * Consumers: the metrics scraper (managed-client error gauge, scoped to these + * slugs to bound `workflow_id` cardinality) and the post-deploy verification + * route (cohort selection + per-client PagerDuty routing). Kept dependency-free + * so it can be imported from API routes, the metrics scraper, and scripts. + * + * Empty / unset config => no managed slugs: the gauge emits no managed series + * and the verification cohort falls back to enterprise-plan orgs only. Parsed + * per call (the JSON is tiny) so a value change takes effect without caching. + */ + +export type ManagedOrgConfig = { + pagerdutyRoutingKey?: string; +}; + +function parseManagedOrgs(): Record { + const raw = process.env.MANAGED_ORGS_CONFIG; + if (!raw) { + return {}; + } + try { + const parsed: unknown = JSON.parse(raw); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // Malformed config behaves like "no managed orgs" rather than crashing the + // caller (a metrics scrape or the verification route). + } + return {}; +} + +/** Slugs of the managed-client cohort. Empty when unconfigured. */ +export function getManagedOrgSlugs(): string[] { + return Object.keys(parseManagedOrgs()); +} + +/** + * PagerDuty Events v2 routing key for a slug. Falls back to + * `DEFAULT_PAGERDUTY_ROUTING_KEY` (for enterprise-plan orgs without a dedicated + * service). Undefined when neither is set -- callers then skip paging. + */ +export function getPagerDutyRoutingKey(slug: string): string | undefined { + return ( + parseManagedOrgs()[slug]?.pagerdutyRoutingKey ?? + process.env.DEFAULT_PAGERDUTY_ROUTING_KEY ?? + undefined + ); +} diff --git a/scripts/backfill-error-classification.ts b/scripts/backfill-error-classification.ts index 67baabc16..cba00028d 100644 --- a/scripts/backfill-error-classification.ts +++ b/scripts/backfill-error-classification.ts @@ -1,22 +1,24 @@ /** - * KEEP-545: one-time backfill of `error_category` and `error_type` - * columns on `workflow_executions` rows for managed-client orgs (Sky and - * Ajna) over the last 90 days. + * One-time backfill of `error_category` and `error_type` columns on + * `workflow_executions` rows for the managed-client orgs over the last 90 days. * * The DB schema migration that added these columns leaves them null on - * historical rows. The SLA alert post-PR-2 watches the new counter, not - * the historical column, so backfill is not required for the alert. The - * reason to run this is to produce an immediate inventory of "real - * engineering failures vs user-config noise" inside the managed-client - * scope without writing one-off SQL. + * historical rows. The SLA alert watches the new counter, not the historical + * column, so backfill is not required for the alert. The reason to run this is + * to produce an immediate inventory of "real engineering failures vs + * user-config noise" inside the managed-client scope without writing one-off + * SQL. * - * Idempotent: skips rows that already have both classification columns - * set. Safe to re-run. + * Idempotent: skips rows that already have both classification columns set. + * Safe to re-run. + * + * Org scope defaults to the managed cohort (`getManagedOrgSlugs()`, sourced + * from MANAGED_ORGS_CONFIG); pass `--orgs` to override. * * Usage: * pnpm tsx scripts/backfill-error-classification.ts --dry-run * pnpm tsx scripts/backfill-error-classification.ts - * pnpm tsx scripts/backfill-error-classification.ts --orgs sky,ajna --days 30 + * pnpm tsx scripts/backfill-error-classification.ts --orgs slug-a,slug-b --days 30 * * Output: * - Progress lines per batch (`processed=N updated=M skipped=K`) @@ -27,8 +29,9 @@ import { and, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import { db } from "@/lib/db"; import { organization, workflowExecutions, workflows } from "@/lib/db/schema"; import { classifyExecutionError } from "@/lib/errors/classify"; +import { getManagedOrgSlugs } from "@/lib/orgs/managed-clients"; -const DEFAULT_ORGS = ["techops-services", "ajna"]; +const DEFAULT_ORGS = getManagedOrgSlugs(); const DEFAULT_DAYS = 90; const BATCH_SIZE = 500; diff --git a/tests/unit/db-metrics-cache.test.ts b/tests/unit/db-metrics-cache.test.ts index 0241d8738..65bafb2fc 100644 --- a/tests/unit/db-metrics-cache.test.ts +++ b/tests/unit/db-metrics-cache.test.ts @@ -140,10 +140,10 @@ const CACHE_LOOKUP_HIT_RE = /keeperhub_db_metrics_cache_lookups_total\{result="hit"\}\s+\d+/; const REFRESH_SUCCESS_RE = /keeperhub_db_metrics_refresh_total\{outcome="success"\}\s+\d+/; -const ERRORS_BY_WORKFLOW_SKY_RE = - /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_sky_1"[^}]*org_slug="techops-services"[^}]*error_type="user"[^}]*\}\s+7/; -const ERRORS_BY_WORKFLOW_AJNA_RE = - /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_ajna_1"[^}]*org_slug="ajna"[^}]*error_type="system"[^}]*\}\s+2/; +const ERRORS_BY_WORKFLOW_A_RE = + /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_a_1"[^}]*org_slug="managed-a"[^}]*error_type="user"[^}]*\}\s+7/; +const ERRORS_BY_WORKFLOW_B_RE = + /keeperhub_workflow_errors_by_workflow\{[^}]*workflow_id="wf_b_1"[^}]*org_slug="managed-b"[^}]*error_type="system"[^}]*\}\s+2/; const ERRORS_BY_CATEGORY_SYSTEM_RE = /keeperhub_system_errors_by_category\{[^}]*error_category="network_rpc"[^}]*error_type="system"[^}]*\}\s+5/; const ERRORS_BY_CATEGORY_UNKNOWN_RE = @@ -392,14 +392,14 @@ describe("keeperhub_workflow_errors_by_workflow gauge", () => { it("emits one series per (workflow_id, org_slug, error_type) from the DB query", async () => { dbMocks.getWorkflowErrorsByWorkflowFromDb.mockResolvedValue([ { - workflowId: "wf_sky_1", - orgSlug: "techops-services", + workflowId: "wf_a_1", + orgSlug: "managed-a", errorType: "user", count: 7, }, { - workflowId: "wf_ajna_1", - orgSlug: "ajna", + workflowId: "wf_b_1", + orgSlug: "managed-b", errorType: "system", count: 2, }, @@ -408,13 +408,18 @@ describe("keeperhub_workflow_errors_by_workflow gauge", () => { await updateDbMetrics(); const out = await getDbMetrics(); - expect(out).toMatch(ERRORS_BY_WORKFLOW_SKY_RE); - expect(out).toMatch(ERRORS_BY_WORKFLOW_AJNA_RE); + expect(out).toMatch(ERRORS_BY_WORKFLOW_A_RE); + expect(out).toMatch(ERRORS_BY_WORKFLOW_B_RE); }); it("clears stale series when a workflow stops appearing in the query", async () => { dbMocks.getWorkflowErrorsByWorkflowFromDb.mockResolvedValueOnce([ - { workflowId: "wf_gone", orgSlug: "ajna", errorType: "user", count: 3 }, + { + workflowId: "wf_gone", + orgSlug: "managed-b", + errorType: "user", + count: 3, + }, ]); await updateDbMetrics(); expect(await getDbMetrics()).toContain('workflow_id="wf_gone"');