From 8f0312fd7527072ebdd6d5b48a8d513895c12562 Mon Sep 17 00:00:00 2001 From: Shreyansh Sancheti <43677304+shreyanshjain7174@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:22:17 +0530 Subject: [PATCH] feat(demo): add honest provider-backed booth flow Add prepare and present modes for the Clawdlinux summit demo. The flow uses real OpenAI routing through LiteLLM, separately checks Anthropic, records genuine token cost, proves webhook mutation without claiming local runsc execution, shows NetworkPolicy presence without claiming packet enforcement, and verifies a clearly labeled prior-run HMAC audit artifact offline. Add safe ignored-.env credential parsing, Kubernetes Secret handoff, provider idempotency keys, generation-scoped routing intent, cost deduplication, cert-manager preparation, focused smoke and Helm tests, and current-versus-target demo documentation. Signed-off-by: Shreyansh Sancheti <43677304+shreyanshjain7174@users.noreply.github.com> --- CHANGELOG.md | 7 + _staging/booth/README.md | 54 +- _staging/booth/rehearsal-checklist.md | 168 ++--- api/v1alpha1/agentworkload_types.go | 5 + charts/charts/litellm/templates/_helpers.tpl | 2 +- .../charts/litellm/templates/configmap.yaml | 8 + charts/charts/litellm/templates/secret.yaml | 5 +- charts/charts/litellm/values.yaml | 6 + .../tests/litellm_existing_secret_test.yaml | 78 +++ charts/values.schema.json | 11 +- charts/values.yaml | 15 +- cmd/main_test.go | 2 +- config/crd/agentworkload_crd.yaml | 2 + ...agentic.clawdlinux.org_agentworkloads.yaml | 5 + docs/PITCH-SCRIPT.md | 66 +- examples/research-agent.yaml | 39 ++ .../controller/agentworkload_controller.go | 120 +++- .../agentworkload_controller_finops_test.go | 290 ++++++++- pkg/finops/interface.go | 7 +- pkg/finops/memory_reporter.go | 43 +- pkg/finops/memory_reporter_test.go | 52 +- pkg/finops/noop.go | 3 +- pkg/llm/mock_provider.go | 3 +- pkg/llm/provider.go | 10 +- pkg/llm/provider_test.go | 30 + pkg/llm/routing.go | 3 +- pkg/llm/routing_test.go | 5 +- scripts/demo-booth.sh | 603 +++++++++++++++++- scripts/record-demo.sh | 14 +- tests/smoke/run_smoke.sh | 8 + tests/smoke/test_demo_booth_cli.sh | 335 ++++++++++ 31 files changed, 1760 insertions(+), 239 deletions(-) create mode 100644 charts/tests/litellm_existing_secret_test.yaml create mode 100644 examples/research-agent.yaml create mode 100644 pkg/llm/provider_test.go create mode 100644 tests/smoke/test_demo_booth_cli.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cbbc0c..2b2ae04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added a working booth prepare/present flow with real provider and cost proof. + +### Changed +- Changed the internal Go `CostReporter.RecordUsage` and `Provider.CallModel` interfaces to carry operation IDs for idempotency. This breaks out-of-tree implementations. +- Added the optional `litellm.anthropicKey` chart value. Use `litellm.existingSecret` for booth and production deployments. + ## [0.2.0] - 2026-04-25 Production-ready agent orchestration release with multi-agent demos, pluggable workflows, CLI onboarding, and landing page overhaul. diff --git a/_staging/booth/README.md b/_staging/booth/README.md index 473db91..03e03ba 100644 --- a/_staging/booth/README.md +++ b/_staging/booth/README.md @@ -1,18 +1,38 @@ -# Booth staging +# Booth evidence boundaries -Fallback assets for the Clawdlinux attestation demo. Use these if the live run -hiccups. Do not present from these by default. The live demo is the demo. +The booth flow has 6 evidence items. State the boundary before each step. -## attestation-fallback.jsonl +- **REAL:** The AgentWorkload makes one OpenAI-routed call through in-cluster LiteLLM. +- **REAL:** Its status reports genuine nonzero tokens. Its cost metric and annotation are nonzero. +- **REAL, SEPARATE CHECK:** A LiteLLM request checks Anthropic reachability. +- **SIMULATION / CONFIGURATION PROOF:** A server-side dry-run shows gVisor mutation. +- **NETWORKPOLICY OBJECT PRESENCE ONLY:** The script lists the NetworkPolicy object. +- **PRIOR-RUN ARTIFACT:** The checked-in HMAC-signed audit fixture verifies offline. -A real, hash-chained, HMAC-signed audit artifact. 6 entries simulating one -regulated agent run: manifest emit, LLM call, an allowed egress, a denied -egress, a human approval, and a state change to Succeeded. Produced with the -production `pkg/audit` recorder, so it verifies with the shipped `audit-verify`. +The current AgentWorkload does not generate the audit fixture. Audit recording +is not wired into reconciliation yet. +The `--present` path does not execute OPA allow or deny evaluation. +OPA remains in the legacy default flow and target product contract. -This is genuine evidence, not a mock. It verifies clean and rejects tampering. +## Credential loading -### Verify it offline (the money shot) +`scripts/demo-booth.sh --prepare` reads provider keys from exported variables first. +Missing keys can load from the repo-root `.env` file. + +Set `DEMO_ENV_FILE=/path/to/file` to use another file. +The parser accepts only `OPENAI_API_KEY` and `ANTHROPIC_API_KEY` definitions. +It supports optional `export` prefixes and quoted values. +It does not source the file or evaluate its contents. + +Preparation prints each variable name as `available` and reports its source. +It never prints credential values. + +## Prior-run audit fixture + +`attestation-fallback.jsonl` is a real hash-chained, HMAC-signed artifact. +It was produced earlier with `pkg/audit`. It contains 6 audit entries. + +### Verify it offline ```bash audit-verify --source jsonl \ @@ -20,9 +40,9 @@ audit-verify --source jsonl \ --key booth-demo-2026=bmluZXZpZ2lsLWJvb3RoLWRlbW8tYXR0ZXN0YXRpb24ta2V5LTMyYg== ``` -Expect: `PASS — chain is intact and all checkpoints match.` +Expect a pass showing the chain is intact. -### Tamper demo (prove it is real evidence) +### Optional tamper proof ```bash sed 's/"actor":"policy-analyst"/"actor":"ghost-actor"/' \ @@ -31,12 +51,10 @@ audit-verify --source jsonl --path /tmp/tampered.jsonl \ --key booth-demo-2026=bmluZXZpZ2lsLWJvb3RoLWRlbW8tYXR0ZXN0YXRpb24ta2V5LTMyYg== ``` -Expect: `FAIL at seq=4: ... entry_hash mismatch`. One altered field and the -chain breaks. That is what an auditor wants to see. +Expect a failure at the changed entry. The script removes its temporary file. ## Notes -- The key here is a demo key, committed on purpose so the fallback is - self-contained. Never use this key for a real run. -- The verify works fully offline. No cluster, no network, no external - dependencies. That is the air-gapped review-room story. +- The committed key is only for this fixture. +- Never use it for a real workload. +- Verification works offline without the cluster. diff --git a/_staging/booth/rehearsal-checklist.md b/_staging/booth/rehearsal-checklist.md index 5cd1d12..d903e00 100644 --- a/_staging/booth/rehearsal-checklist.md +++ b/_staging/booth/rehearsal-checklist.md @@ -1,119 +1,75 @@ # Booth rehearsal checklist -Cold-run gate for the Clawdlinux attestation demo. Run this 5 times on the actual -booth laptop, offline, before the summit. Assume venue wifi is dead. Tick every -gate. If any gate fails twice, the demo is not booth-ready. - -Driver script: `scripts/demo-booth.sh` (560 lines). This checklist mirrors its -phases and adds explicit pass/fail gates the script does not assert for you. - -## 0. One-time offline prep (do once, before rehearsals) - -- [ ] Laptop has `kubectl`, `helm`, and `kind` (or `k3s`) installed and on PATH. -- [ ] Seed the kind node-image cache with the same command the demo script uses: - `kind create cluster --name clawdlinux-demo --wait 120s` - This pulls the exact `kindest/node` tag for your installed `kind` binary. - Without this, wifi-off `kind create cluster` fails immediately. -- [ ] Pre-pull every image the demo needs, then point kind at the local cache so - no phase pulls from the network: - - [ ] Keep `clawdlinux-demo` running while you load images. - - [ ] For each image the chart and samples use: `docker pull ` then - `kind load docker-image --name clawdlinux-demo`. - - [ ] Re-run the demo with wifi OFF. If any phase blocks on ImagePull, the - cache is incomplete. Fix before the summit. -- [ ] `audit-verify` built and on PATH: `go build -o ~/bin/audit-verify ./cmd/audit-verify`. -- [ ] Fallback artifact staged: `_staging/booth/attestation-fallback.jsonl` - (see `_staging/booth/README.md`). Confirm it still verifies offline. - -## 1. Prerequisites (script: "Checking prerequisites") - -- [ ] PASS gate: script prints "kubectl and helm found" and finds kind or k3s. -- [ ] FAIL signal: "kind or k3s is required" or a missing manifest. Stop, fix PATH - or repo checkout. -- [ ] Confirm present: `config/samples/agentworkload_demo_{allow,deny,swarm}.yaml`. - -## 2. Cluster prep (script: "Preparing Kubernetes cluster") - -- [ ] PASS gate: "Created kind cluster clawdlinux-demo" (or "Reusing ...") within - 120s. -- [ ] FAIL signal: kind create hangs past 120s. Usually a stale Docker state. - Run `scripts/demo-booth.sh --cleanup` and retry. - -## 3. Operator ready (script: "Waiting for operator pod") - -- [ ] PASS gate: operator pod reaches Running inside the 180s deadline. -- [ ] FAIL signal: deadline hit. Check `kubectl -n agentic-system describe pod`. - Almost always an un-cached image. Go back to step 0. - -## 4. Deployment profile (script: "Installing ... profile") - -- [ ] Decide before you start: `--profile platform` (Argo and shared services) - or `--profile lean` (operator and governance controls only). - Rehearse the one you will present. -- [ ] PASS gate: helm install completes within `HELM_TIMEOUT` (default 180s). -- [ ] FAIL signal: helm timeout. Raise `HELM_TIMEOUT=300s` only if the laptop is - slow; if it is an image pull, fix the cache, do not just raise the timeout. - -## 5. Workload + runtime view (script: "AgentWorkload status", "Agent pod runtime view") - -- [ ] PASS gate: the demo AgentWorkload shows a populated status; an agent pod is - listed with its runtime view. - -## 6. Cost metric (script: "Checking cost metric") - -- [ ] PASS gate: a per-workload cost metric is reported (non-empty). -- [ ] This is the FinOps talking point. If empty, say so, do not improvise. - -## 7. gVisor sandbox (script: "Checking gVisor RuntimeClass") - -- [ ] PASS gate: the labeled pod shows `runtimeClassName: gvisor`. - -## 8. Egress seal — the core demo (script: "Checking NetworkPolicy", "OPA allow", "OPA deny") - -- [ ] PASS gate (allow): `agentworkload_demo_allow.yaml` runs; egress scoped to - the FQDN allowlist. -- [ ] PASS gate (deny): `agentworkload_demo_deny.yaml` is blocked at the boundary. -- [ ] Talk track: "default deny, allowlist per workload, enforced at the network - layer, not in app code." -- [ ] FAIL signal: deny case is NOT blocked. This breaks the whole pitch. Do not - present until fixed. - -## 9. Attestation artifact — the money shot - -- [ ] During the run, capture the audit/attestation output to the evidence dir. -- [ ] Verify it offline in front of the prospect: - `audit-verify --source jsonl --path .jsonl --key =` -- [ ] PASS gate: "PASS — chain is intact". -- [ ] Tamper demo (optional, strong): edit one field, re-run audit-verify, show - it FAIL with an entry_hash mismatch. Proves the artifact is real evidence. -- [ ] If the live run hiccups, fall back to `_staging/booth/attestation-fallback.jsonl` - and run the same verify + tamper demo. Never debug live on a prospect's time. - -## 10. Multi-agent swarm (optional scenario, needs `--with-swarm`) - -- [ ] Only if presenting the swarm: run with `--with-swarm` and the platform profile. -- [ ] PASS gate: 3-agent swarm reaches a terminal state without manual nudging. +Run this flow 5 times on the booth Mac. The real provider steps require network +access. Keep the prior-run verifier available when venue network access fails. + +Driver script: `scripts/demo-booth.sh`. + +## 0. One-time prep + +- [ ] Install `kubectl`, `helm`, `kind`, Docker, `curl`, and Go. +- [ ] Build the current operator image using the configured demo image tag. +- [ ] Build `bin/agentctl` with `make build-agentctl`. +- [ ] Build `bin/audit-verify` with `make obs-build-verifier`. +- [ ] Confirm `_staging/booth/attestation-fallback.jsonl` verifies offline. +- [ ] Keep the repo-root `.env` ignored and readable only by the booth operator. + +## 1. Prepare with real credentials + +- [ ] Export both provider keys or define them in the repo-root `.env` file. +- [ ] Set `DEMO_ENV_FILE` when the credential file lives elsewhere. +- [ ] Run `scripts/demo-booth.sh --prepare`. +- [ ] Confirm both variable names report `available` without showing values. +- [ ] Confirm the reported source is `environment` or the expected file. +- [ ] Confirm exported `LITELLM_MASTER_KEY` values with LF or CR are rejected. +- [ ] Confirm context is `kind-clawdlinux-demo`. +- [ ] Confirm cert-manager, webhook, operator, and LiteLLM become ready. +- [ ] Confirm the script never prints key values. +- [ ] Unset exported provider variables after preparation on shared terminals. + +## 2. Present the 5-7 minute flow + +- [ ] Run `scripts/demo-booth.sh --present`. +- [ ] **REAL:** AgentWorkload reaches `Completed` after one OpenAI-routed call. +- [ ] **REAL:** Routing condition shows genuine input and output token counts. +- [ ] **REAL:** Cost annotation is greater than zero. +- [ ] **REAL:** `clawdlinux_agent_cost_dollars` is greater than zero. +- [ ] **REAL:** Separate Anthropic request reports provider reachability. +- [ ] Do not claim the AgentWorkload used Anthropic. +- [ ] **SIMULATION / CONFIGURATION PROOF:** Dry-run injects `runtimeClassName=gvisor`. +- [ ] State that no gVisor pod was scheduled on kind/macOS. +- [ ] **NETWORKPOLICY OBJECT PRESENCE ONLY:** Show the NetworkPolicy object. +- [ ] Say packet enforcement requires an enforcing CNI. +- [ ] Do not claim all runtime pods carry sandbox labels. +- [ ] State that `--present` does not execute OPA allow or deny evaluation. + +## 3. Prior-run audit closer + +- [ ] State that the current workload did not create the fixture. +- [ ] **PRIOR-RUN ARTIFACT:** Verify the checked-in JSONL file offline. +- [ ] Optionally run `scripts/demo-booth.sh --present --tamper-audit`. +- [ ] Confirm the altered temporary artifact fails verification. +- [ ] State that audit recording is not wired into reconciliation yet. ## 11. Teardown - [ ] `scripts/demo-booth.sh --cleanup` deletes the kind cluster. - [ ] Confirm `kind get clusters` no longer lists `clawdlinux-demo`. -## Rehearsal log (fill this in — 5 cold runs, wifi off) +## Rehearsal log -| Run | Date | Wifi off? | All gates pass? | First failure (phase) | Time to green | Notes | -|-----|------|-----------|-----------------|-----------------------|---------------|-------| -| 1 | | | | | | | -| 2 | | | | | | | -| 3 | | | | | | | -| 4 | | | | | | | -| 5 | | | | | | | +| Run | Date | Network stable? | All gates pass? | First failure | Total time | Notes | +|-----|------|-----------------|-----------------|---------------|------------|-------| +| 1 | | | | | | | +| 2 | | | | | | | +| 3 | | | | | | | +| 4 | | | | | | | +| 5 | | | | | | | -Booth-ready bar: 5 consecutive cold runs, wifi off, all gates green, under your -demo time budget. Anything less and you are gambling on the venue. +Booth-ready bar: 5 consecutive runs under 7 minutes with every truth label spoken. ## Optional: record a backup -- [ ] `scripts/demo-booth.sh --profile platform --with-swarm --record` captures terminal output with - script(1). Keep one clean recording as a last-resort fallback if the laptop - itself dies. +- [ ] `scripts/record-demo.sh` records the present path. +- [ ] Set `DEMO_TAMPER_AUDIT=true` to include the tamper proof. +- [ ] Review the recording for accidental key output before sharing it. diff --git a/api/v1alpha1/agentworkload_types.go b/api/v1alpha1/agentworkload_types.go index ca95fd6..117a544 100644 --- a/api/v1alpha1/agentworkload_types.go +++ b/api/v1alpha1/agentworkload_types.go @@ -358,6 +358,11 @@ type AgentWorkloadStatus struct { // +optional WorkflowArtifacts map[string]string `json:"workflowArtifacts,omitempty"` + // modelRoutingOperationID is the stable idempotency key for the current + // workload generation's model call. + // +optional + ModelRoutingOperationID string `json:"modelRoutingOperationID,omitempty"` + // conditions represent the current state of the AgentWorkload resource. // Each condition has a unique type and reflects the status of a specific aspect of the resource. // diff --git a/charts/charts/litellm/templates/_helpers.tpl b/charts/charts/litellm/templates/_helpers.tpl index 8e01536..1bb4723 100644 --- a/charts/charts/litellm/templates/_helpers.tpl +++ b/charts/charts/litellm/templates/_helpers.tpl @@ -17,7 +17,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Resolve the Secret name that holds OPENAI_API_KEY / LITELLM_MASTER_KEY. +Resolve the Secret that holds OPENAI_API_KEY, ANTHROPIC_API_KEY, and LITELLM_MASTER_KEY. If .Values.existingSecret is set, use it (BYO secret -- recommended for production). Otherwise fall back to the chart-managed Secret (dev/demo only). */}} diff --git a/charts/charts/litellm/templates/configmap.yaml b/charts/charts/litellm/templates/configmap.yaml index 9bcf162..149faa8 100644 --- a/charts/charts/litellm/templates/configmap.yaml +++ b/charts/charts/litellm/templates/configmap.yaml @@ -11,6 +11,14 @@ data: master_key: "os.environ/LITELLM_MASTER_KEY" database_url: null model_list: + - model_name: clawdlinux-openai + litellm_params: + model: openai/gpt-4o-mini + api_key: "os.environ/OPENAI_API_KEY" + - model_name: clawdlinux-anthropic + litellm_params: + model: anthropic/claude-haiku-4-5-20251001 + api_key: "os.environ/ANTHROPIC_API_KEY" - model_name: gpt-4o litellm_params: model: openai/gpt-4o diff --git a/charts/charts/litellm/templates/secret.yaml b/charts/charts/litellm/templates/secret.yaml index f761864..2a31741 100644 --- a/charts/charts/litellm/templates/secret.yaml +++ b/charts/charts/litellm/templates/secret.yaml @@ -5,8 +5,8 @@ Production deployments should set .Values.existingSecret to a Secret pre-created plaintext keys never enter `helm get values`, shell history, or the helm release Secret. */ -}} {{- if not .Values.existingSecret }} -{{- if and (or .Values.openaiKey .Values.masterKey) (not .Values.allowPlaintextKeys) }} -{{- fail "A plaintext Helm key value is set (litellm.openaiKey and/or litellm.masterKey). Use litellm.existingSecret to reference a pre-created Secret, or set litellm.allowPlaintextKeys=true to acknowledge the risk (dev/demo only)." }} +{{- if and (or .Values.openaiKey .Values.anthropicKey .Values.masterKey) (not .Values.allowPlaintextKeys) }} +{{- fail "A plaintext Helm key value is set (litellm.openaiKey, litellm.anthropicKey, and/or litellm.masterKey). Use litellm.existingSecret to reference a pre-created Secret, or set litellm.allowPlaintextKeys=true to acknowledge the risk (dev/demo only)." }} {{- end }} apiVersion: v1 kind: Secret @@ -18,5 +18,6 @@ metadata: type: Opaque stringData: OPENAI_API_KEY: {{ .Values.openaiKey | default "" | quote }} + ANTHROPIC_API_KEY: {{ .Values.anthropicKey | default "" | quote }} LITELLM_MASTER_KEY: {{ .Values.masterKey | default (randAlphaNum 32) | quote }} {{- end }} diff --git a/charts/charts/litellm/values.yaml b/charts/charts/litellm/values.yaml index 24fe5a0..b1c620d 100644 --- a/charts/charts/litellm/values.yaml +++ b/charts/charts/litellm/values.yaml @@ -14,7 +14,13 @@ resources: cpu: "1000m" memory: "512Mi" openaiKey: "" +anthropicKey: "" masterKey: "" +# Name of a pre-created Secret. LiteLLM reads OPENAI_API_KEY, +# ANTHROPIC_API_KEY, and LITELLM_MASTER_KEY from it via envFrom. +# Recommended for booth and production deployments. +existingSecret: "" +allowPlaintextKeys: false models: [] # LiteLLM Routing Configuration diff --git a/charts/tests/litellm_existing_secret_test.yaml b/charts/tests/litellm_existing_secret_test.yaml new file mode 100644 index 0000000..09b2735 --- /dev/null +++ b/charts/tests/litellm_existing_secret_test.yaml @@ -0,0 +1,78 @@ +# Run with: helm unittest charts +suite: LiteLLM existing provider secret +templates: + - charts/litellm/templates/configmap.yaml + - charts/litellm/templates/deployment.yaml + - charts/litellm/templates/secret.yaml +tests: + - it: renders OpenAI and Anthropic aliases from one existing Secret + set: + litellm: + enabled: true + existingSecret: clawdlinux-demo-litellm + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'model_name: clawdlinux-openai[\s\S]*model: openai/gpt-4o-mini[\s\S]*api_key: "os.environ/OPENAI_API_KEY"' + template: charts/litellm/templates/configmap.yaml + - matchRegex: + path: data["config.yaml"] + pattern: 'model_name: clawdlinux-anthropic[\s\S]*model: anthropic/claude-haiku-4-5-20251001[\s\S]*api_key: "os.environ/ANTHROPIC_API_KEY"' + template: charts/litellm/templates/configmap.yaml + - documentIndex: 0 + equal: + path: spec.template.spec.containers[0].envFrom[0].secretRef.name + value: clawdlinux-demo-litellm + template: charts/litellm/templates/deployment.yaml + - hasDocuments: + count: 0 + template: charts/litellm/templates/secret.yaml + + - it: renders all provider keys in the chart-managed Secret + set: + litellm: + enabled: true + allowPlaintextKeys: true + openaiKey: openai-test-key + anthropicKey: anthropic-test-key + masterKey: master-test-key + asserts: + - equal: + path: stringData.OPENAI_API_KEY + value: openai-test-key + template: charts/litellm/templates/secret.yaml + - equal: + path: stringData.ANTHROPIC_API_KEY + value: anthropic-test-key + template: charts/litellm/templates/secret.yaml + - equal: + path: stringData.LITELLM_MASTER_KEY + value: master-test-key + template: charts/litellm/templates/secret.yaml + + - it: keeps the default chart-managed Secret valid with empty provider keys + set: + litellm: + enabled: true + asserts: + - equal: + path: stringData.OPENAI_API_KEY + value: "" + template: charts/litellm/templates/secret.yaml + - equal: + path: stringData.ANTHROPIC_API_KEY + value: "" + template: charts/litellm/templates/secret.yaml + - isNotEmpty: + path: stringData.LITELLM_MASTER_KEY + template: charts/litellm/templates/secret.yaml + + - it: rejects a plaintext Anthropic key without risk acknowledgement + set: + litellm: + enabled: true + anthropicKey: anthropic-test-key + asserts: + - failedTemplate: + errorMessage: A plaintext Helm key value is set (litellm.openaiKey, litellm.anthropicKey, and/or litellm.masterKey). Use litellm.existingSecret to reference a pre-created Secret, or set litellm.allowPlaintextKeys=true to acknowledge the risk (dev/demo only). + template: charts/litellm/templates/secret.yaml diff --git a/charts/values.schema.json b/charts/values.schema.json index 9a131fd..c7cebd2 100644 --- a/charts/values.schema.json +++ b/charts/values.schema.json @@ -115,9 +115,18 @@ "service": { "$ref": "#/$defs/service" }, "openaiKey": { "type": "string", - "description": "OpenAI API key for LiteLLM proxy — required at install time" + "description": "OpenAI API key for chart-managed development installs" + }, + "anthropicKey": { + "type": "string", + "description": "Optional Anthropic API key for chart-managed development installs" }, "masterKey": { "type": "string" }, + "existingSecret": { + "type": "string", + "description": "Pre-created Secret containing OpenAI, Anthropic, and LiteLLM master keys" + }, + "allowPlaintextKeys": { "type": "boolean" }, "models": { "type": "array", "items": { "type": "object" } diff --git a/charts/values.yaml b/charts/values.yaml index 10c0f23..52799d5 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -242,19 +242,22 @@ litellm: type: ClusterIP port: 4000 openaiKey: "" # REQUIRED when litellm.enabled=true (dev only -- prefer existingSecret) + anthropicKey: "" # Optional Anthropic API key (dev only -- prefer existingSecret) masterKey: "" # LiteLLM master API key (leave blank to auto-generate) # PRODUCTION: set existingSecret to the name of a pre-created Secret containing - # OPENAI_API_KEY and LITELLM_MASTER_KEY. When set, openaiKey/masterKey are ignored - # and the chart will NOT create a Secret -- keys never enter Helm values or release Secret. + # OPENAI_API_KEY, ANTHROPIC_API_KEY, and LITELLM_MASTER_KEY. Workloads may also + # reference an api-key entry in the same Secret. When set, openaiKey, + # anthropicKey, and masterKey are ignored. The chart will NOT create a Secret. + # Keys never enter Helm values. This is recommended for booth and production use. existingSecret: "" - # Setting allowPlaintextKeys=true acknowledges the risk of providing openaiKey or masterKey - # via Helm values. Required for chart install when either key is non-empty and - # existingSecret is unset. + # Setting allowPlaintextKeys=true acknowledges the risk of providing openaiKey, + # anthropicKey, or masterKey via Helm values. Required when any key is non-empty + # and existingSecret is unset. allowPlaintextKeys: false models: [] # Extra model definitions passed to config.yaml # Example: # models: - # - model_name: gpt-4o + # - model_name: custom-openai # litellm_params: # model: openai/gpt-4o # api_key: "os.environ/OPENAI_API_KEY" diff --git a/cmd/main_test.go b/cmd/main_test.go index 01392fa..57a52ca 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -17,7 +17,7 @@ func TestFinOpsMetricsEndpointContainsClawdlinuxCostMetric(t *testing.T) { t.Parallel() reporter := finops.NewMemoryCostReporter() - if err := reporter.RecordUsage(context.Background(), "demo-workload", "demo-ns", "openai/gpt-4o-mini", 1000, 500); err != nil { + if err := reporter.RecordUsage(context.Background(), "main-cost-reporter", "demo-workload", "demo-ns", "openai/gpt-4o-mini", 1000, 500); err != nil { t.Fatalf("record usage: %v", err) } diff --git a/config/crd/agentworkload_crd.yaml b/config/crd/agentworkload_crd.yaml index dbc806c..3d6be44 100644 --- a/config/crd/agentworkload_crd.yaml +++ b/config/crd/agentworkload_crd.yaml @@ -162,6 +162,8 @@ spec: format: date-time argoPhase: type: string + modelRoutingOperationID: + type: string workflowArtifacts: type: object additionalProperties: diff --git a/config/crd/bases/agentic.clawdlinux.org_agentworkloads.yaml b/config/crd/bases/agentic.clawdlinux.org_agentworkloads.yaml index e487ea1..b3731d0 100644 --- a/config/crd/bases/agentic.clawdlinux.org_agentworkloads.yaml +++ b/config/crd/bases/agentic.clawdlinux.org_agentworkloads.yaml @@ -500,6 +500,11 @@ spec: description: lastReconcileTime is the last time the resource was reconciled format: date-time type: string + modelRoutingOperationID: + description: |- + modelRoutingOperationID is the stable idempotency key for the current + workload generation's model call. + type: string phase: description: 'phase is the current lifecycle phase: Pending, Running, Completed, Failed' diff --git a/docs/PITCH-SCRIPT.md b/docs/PITCH-SCRIPT.md index d96f7ec..b8abe3d 100644 --- a/docs/PITCH-SCRIPT.md +++ b/docs/PITCH-SCRIPT.md @@ -7,7 +7,7 @@ Companion to the current co-founder deck and [DEMO-PLAN.md](DEMO-PLAN.md). Total Times are targets. If she interrupts with questions, good, let her. The deck is a spine, not a cage. **Slide 1, Title (30s).** -"Clawdlinux provides regulated controls for AI agents on Kubernetes. One sentence version: every agent run leaves a signed, offline-verifiable record, and runs inside a network seal it cannot escape. I am not building another agent framework. I am building the thing that lets a bank say yes to one." +"Clawdlinux is building regulated controls for AI agents on Kubernetes. The target contract is offline-verifiable attestation with packet-enforced egress controls. Today's booth proof is narrower, and I will label every boundary. I am not building another agent framework. I am building the control plane that lets a bank review one." **Slide 2, Problem (90s).** "Agents work now. What does not work is getting them past a security review. These are not my numbers." Walk the rings left to right. Land on: "Gartner says 40 percent of these projects die, and names inadequate risk controls as a reason. Only 11 percent of production agents pass an independent security bar. The demand curve and the controls curve have a gap. That gap is the company." @@ -16,13 +16,13 @@ Times are targets. If she interrupts with questions, good, let her. The deck is "Four things happened at once. Strong open runtimes became available, so we no longer need to rebuild that layer. Incidents went from hypothetical to a bank self-reporting to its regulator. Fed guidance explicitly excludes agentic AI, so auditors are improvising. The buyers who need verifiable evidence most are also the ones that often cannot use a hosted control plane. We start with self-hosted and air-gapped deployments." **Slide 4, What we built (90s).** -"Our wedge combines two controls in one self-hosted deployment. First, attestation: every LLM call, tool call, and approval goes into a hash chain, HMAC-signed, and a small CLI recomputes the whole chain offline. The auditor does not have to trust me, or you, or the customer. Second, the egress seal: the agent reaches only the endpoints its manifest declares, enforced by Cilium at the network layer. Not a system prompt asking the model to behave. gVisor and OPA wrap around those." Pause. "Key design decision: same contract whether the workload is our CRD, someone's existing pods, or a third-party runtime. We wrap, we do not replace." +"Our target product is an in-cluster governance and attestation plane. The booth proves one OpenAI-routed model call through LiteLLM. It shows genuine tokens, nonzero cost evidence, and a separate Anthropic reachability check. It also shows webhook mutation simulation, NetworkPolicy object presence, and prior-run HMAC audit verification. The current path does not execute OPA. It does not prove packet enforcement, a scheduled gVisor workload, or current-run audit generation." **Slide 5, How a run works (60s).** -Walk the six steps quickly. The line that matters: "Notice policy is decided before anything runs, and evidence is produced whether or not anyone asks for it. Compliance is a byproduct of running, not a project after running." +Walk the six target steps quickly. Say: "This slide describes the target product contract. Policy allow or deny happens before execution in that contract. The current `--present` path does not execute OPA. It also does not produce current-run attestation." **Slide 6, Use case (90s).** -"Concrete customer: a fund's platform team wants agents analyzing filings and market data. Infosec banned agent SaaS, so the project is stuck. With us: they helm install one bundle inside their perimeter, point LiteLLM at whatever model they already approved, and label their existing agent pods. Three people get unblocked: platform lead ships, security reviews a YAML diff instead of a prompt, compliance hands the auditor a snapshot and a binary instead of a week of log archaeology." +"Target customer: a fund's platform team wants agents analyzing filings and market data. Infosec banned agent SaaS, so the project is stuck. In the target deployment, they install one bundle inside their perimeter. They point LiteLLM at an approved model and label existing agent pods. Platform ships the workload. Security reviews the controls. Compliance receives the product attestation package." **Slide 7, What exists (60s).** "Everything on this slide is in a public repo you can read tonight. 180 commits since February. The ACP numbers are measured, checked-in benchmarks, not projections: 64 to 97 percent context reduction against raw MCP, one round trip instead of up to 21." @@ -38,27 +38,63 @@ Walk the six steps quickly. The line that matters: "Notice policy is decided bef Transition straight into the demo. No gap. -## Part 2: What the demo actually is, technically +## Part 2: Current booth proof and target product contract -`scripts/demo-booth.sh` is a gate script: it provisions and then proves five controls, printing PASS/FAIL evidence for each. Know what each phase does so you can narrate it instead of reading it. +Prepare once with both provider keys exported or stored in the ignored repo-root `.env` file: -**Phase 0, setup.** Checks kubectl, helm, kind (or reuses k3s). Creates kind cluster `clawdlinux-demo`. The platform profile installs the operator, Argo, and shared services via `tests/harness/setup.sh`. `--profile lean` installs the CRD, operator, NetworkPolicy, and gVisor sandbox while omitting Argo and shared services. The lean profile starts faster and does not depend on Argo scheduling; use it when the laptop or wifi is the risk. +```bash +scripts/demo-booth.sh --prepare +``` -**Phase 1, OPA allow/deny.** Applies `opa-allow-demo` (objective: "analyze quarterly revenue data") and waits for it to reach a running phase. Then applies `opa-deny-demo` (objective: "delete all production volumes") and waits for phase `PolicyDenied`, then prints the deny reason from the CRD status condition. What this proves: policy is evaluated by the operator against the manifest before execution, and the decision plus the reason is recorded on the Kubernetes object itself. The model is not in the loop. +Set `DEMO_ENV_FILE=/path/to/file` to use another credential file. +Exported variables override file values. +The script parses only the 2 provider key names. +It never sources the file or prints values. -**Phase 2, cost.** Curls the operator's metrics endpoint from inside the pod and greps for `clawdlinux_agent_cost_dollars`. What this proves: per-workload cost is a first-class metric, not a spreadsheet. +Preparation creates kind cluster `clawdlinux-demo`. It installs pinned cert-manager. +It creates one runtime Secret without printing values. Helm receives only the Secret name. -**Phase 3, gVisor.** Confirms the `gvisor` RuntimeClass exists, then does a server-side dry-run of a pod labeled `agentic.clawdlinux.org/runtime-sandbox: gvisor` and shows that the mutating webhook injected `runtimeClassName: gvisor`. What this proves: isolation is applied by admission control based on a label. No agent code changes. +Run the 5-7 minute path: -**Phase 4, NetworkPolicy.** Confirms the egress policy objects are installed in the namespace. What this proves: the seal exists as a Kubernetes-native object security can review. (On kind, Cilium FQDN enforcement is not fully live; do not claim packet-level blocking in this environment, claim policy generation and show the object.) +```bash +scripts/demo-booth.sh --present +``` -**Phase 5, swarm (`--with-swarm` only).** Applies `demo-competitive-swarm`: three agents (competitor-scraper, llm-synthesizer, report-generator) compiled into an Argo Workflow DAG. What this proves: the same controls wrap a multi-agent pipeline, and orchestration is delegated to Argo rather than reinvented. +### CURRENT BOOTH PROOF -**The audit tamper closer uses the signed fallback artifact.** Follow [`_staging/booth/README.md`](../_staging/booth/README.md): run `audit-verify --source jsonl --path _staging/booth/attestation-fallback.jsonl --key ` for PASS, use the documented `sed` command to create `/tmp/tampered.jsonl`, then verify that file for FAIL. Rehearse this end to end at least once before the call. If you cannot get a clean PASS/FAIL cycle working, cut it from the live demo and show the `pkg/audit` design instead; a fumbled trust demo is worse than none. +Say each proof label aloud. Do not blend target behavior into the current demo. -**Flags cheat sheet.** `--profile platform|lean` selects the deployment profile, `--with-swarm` adds the Argo swarm scenario, `--record` captures the terminal with script(1), and `--cleanup` deletes the cluster. Evidence lands in `tests/harness/evidence/booth-/`. +**REAL LLM CALL AND COST.** `examples/research-agent.yaml` uses one incident investigator. +Every task category maps to `clawdlinux-openai`. LiteLLM routes that alias to OpenAI. +The status condition shows genuine token counts. The annotation and metric show nonzero cost. -**What the demo does NOT show. Say so if asked.** No real LLM traffic unless an endpoint is configured; the allow workload exercises lifecycle, not inference quality. No packet-level egress blocking on kind. No SPIFFE, no multi-cluster, no SOC 2. Saying "that part is roadmap" out loud is what makes the rest believable. +**REAL, separate Anthropic check.** A small request uses `clawdlinux-anthropic` through LiteLLM. +It proves provider reachability. Do not claim the AgentWorkload used Anthropic. + +**WEBHOOK MUTATION SIMULATION.** A server-side dry-run shows webhook mutation. +It injects `runtimeClassName=gvisor`. No gVisor pod is scheduled on kind/macOS. + +**NETWORKPOLICY OBJECT PRESENCE ONLY.** The script shows the generated NetworkPolicy object. +Say: "Packet enforcement requires an enforcing CNI." This iteration installs no Cilium. + +**PRIOR-RUN HMAC AUDIT FIXTURE.** The checked-in JSONL fixture passes HMAC verification offline. +The current workload did not generate it. Do not call it a current-run artifact or asymmetric signature. + +Add `--tamper-audit` to alter a temporary copy and show verification failure. +Use `scripts/record-demo.sh` to record the present path. + +**OPA IS LEGACY OR TARGET ONLY.** The legacy default flow retains its OPA allow/deny development path. +The target product contract also includes policy evaluation. `--present` executes neither path. + +**What this demo does not prove.** It does not prove packet enforcement on kind. +It does not prove a scheduled gVisor workload. It does not prove current-run attestation. +It does not prove every runtime pod has a sandbox label. It does not prove an OPA decision or policy gate. + +### TARGET PRODUCT CONTRACT + +The target product emits offline-verifiable, tamper-evident attestation for each governed run. +The target product applies packet-enforced egress controls through a supported enforcing CNI. +These are product acceptance criteria. They are not outcomes from the current booth path. ## Part 3: Turning the demo into signal and customers diff --git a/examples/research-agent.yaml b/examples/research-agent.yaml new file mode 100644 index 0000000..0c42cac --- /dev/null +++ b/examples/research-agent.yaml @@ -0,0 +1,39 @@ +apiVersion: agentic.clawdlinux.org/v1alpha1 +kind: AgentWorkload +metadata: + name: booth-incident-investigation + namespace: agentic-system + labels: + app.kubernetes.io/name: booth-incident-investigation + app.kubernetes.io/part-of: clawdlinux-booth-demo + demo.clawdlinux.org/evidence: real-openai-via-litellm + annotations: + demo.clawdlinux.org/provider-proof: "OpenAI is called through in-cluster LiteLLM. Anthropic is checked separately." +spec: + objective: | + Investigate a production checkout latency incident from these facts: + p95 rose from 280ms to 2.4s after a database connection-pool change, + error rate reached 7%, and rollback restored normal service in 6 minutes. + Produce a concise incident assessment with the likely cause, evidence, + immediate containment, and 3 follow-up checks. Do not invent missing telemetry. + workloadType: kubernetes + agents: + - incident-investigator + collaborationMode: solo + opaPolicy: strict + autoApproveThreshold: "0.95" + modelStrategy: cost-aware + taskClassifier: default + providers: + - name: litellm + type: openai-compatible + endpoint: http://clawdlinux-demo-litellm.agentic-system.svc.cluster.local:4000/v1 + apiKeySecret: + name: clawdlinux-demo-litellm + key: api-key + modelMapping: + validation: litellm/clawdlinux-openai + analysis: litellm/clawdlinux-openai + reasoning: litellm/clawdlinux-openai + timeouts: + execution: 180 diff --git a/internal/controller/agentworkload_controller.go b/internal/controller/agentworkload_controller.go index 1e7eb65..3d7763e 100644 --- a/internal/controller/agentworkload_controller.go +++ b/internal/controller/agentworkload_controller.go @@ -20,6 +20,7 @@ package controller import ( "context" + "crypto/sha256" "encoding/json" "fmt" "os" @@ -58,6 +59,8 @@ const maxActionsInStatus = 100 // so the workflow in `argo-workflows` namespace would otherwise leak. const AgentWorkloadFinalizer = "agentic.clawdlinux.org/finalizer" +const modelRoutingPendingCondition = "ModelRoutingPending" + // AgentWorkloadReconciler reconciles a AgentWorkload object type AgentWorkloadReconciler struct { client.Client @@ -249,8 +252,20 @@ func (r *AgentWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // ========== MODEL ROUTING (Phase 3) with Retry (Phase 5) ========== - // Handle cost-aware model routing if enabled + // Provider delivery is at least once. A persisted operation ID gives each + // retry the same upstream idempotency key when the provider supports it. if workload.Spec.ModelStrategy != nil && *workload.Spec.ModelStrategy == "cost-aware" { + if modelRoutingSucceededForGeneration(&workload) { + log.Info("model routing already completed for workload generation", "generation", workload.Generation) + return ctrl.Result{}, nil + } + + operationID, err := r.persistModelRoutingIntent(ctx, &workload) + if err != nil { + log.Error(err, "failed to persist model routing intent") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + type routeResult struct { response *llm.ModelResponse routingInfo *llm.RoutingInfo @@ -265,7 +280,7 @@ func (r *AgentWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Reques }) response := result.response routingInfo := result.routingInfo - err := retryInfo.LastErr + err = retryInfo.LastErr if err != nil { log.Error(err, "model routing failed after retries", @@ -305,6 +320,15 @@ func (r *AgentWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Reques if workload.Status.Conditions == nil { workload.Status.Conditions = []metav1.Condition{} } + workload.Status.ModelRoutingOperationID = operationID + workload.Status.Conditions = upsertCondition(workload.Status.Conditions, metav1.Condition{ + Type: modelRoutingPendingCondition, + Status: metav1.ConditionFalse, + ObservedGeneration: workload.Generation, + Reason: "RoutingCompleted", + Message: fmt.Sprintf("Model routing operation %s completed", operationID), + LastTransitionTime: metav1.Now(), + }) condition := metav1.Condition{ Type: "ModelRoutingSucceeded", @@ -319,19 +343,7 @@ func (r *AgentWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Reques LastTransitionTime: metav1.Now(), } - // Replace or append condition - foundIdx := -1 - for i, c := range workload.Status.Conditions { - if c.Type == "ModelRoutingSucceeded" { - foundIdx = i - break - } - } - if foundIdx >= 0 { - workload.Status.Conditions[foundIdx] = condition - } else { - workload.Status.Conditions = append(workload.Status.Conditions, condition) - } + workload.Status.Conditions = upsertCondition(workload.Status.Conditions, condition) workload.Status.Phase = "Completed" @@ -766,6 +778,55 @@ func (r *AgentWorkloadReconciler) cleanupViaRuntime(ctx context.Context, workloa // routeAndCallModel handles cost-aware model routing for instructions // It classifies the task, selects the appropriate model/provider, and calls it // Returns the model response and routing metadata for tracking +func modelRoutingSucceededForGeneration(workload *agenticv1alpha1.AgentWorkload) bool { + for _, condition := range workload.Status.Conditions { + if condition.Type == "ModelRoutingSucceeded" && + condition.Status == metav1.ConditionTrue && + condition.ObservedGeneration == workload.Generation { + return true + } + } + return false +} + +func modelRoutingOperationID(workload *agenticv1alpha1.AgentWorkload) string { + source := fmt.Sprintf("%s/%s/%s/%d", workload.Namespace, workload.Name, workload.UID, workload.Generation) + digest := sha256.Sum256([]byte(source)) + return fmt.Sprintf("agentworkload-g%d-%x", workload.Generation, digest[:12]) +} + +func persistedModelRoutingOperationID(workload *agenticv1alpha1.AgentWorkload) string { + if workload.Status.ModelRoutingOperationID != "" { + return workload.Status.ModelRoutingOperationID + } + return modelRoutingOperationID(workload) +} + +func (r *AgentWorkloadReconciler) persistModelRoutingIntent( + ctx context.Context, + workload *agenticv1alpha1.AgentWorkload, +) (string, error) { + operationID := modelRoutingOperationID(workload) + if workload.Status.ModelRoutingOperationID == operationID { + return operationID, nil + } + + workload.Status.ModelRoutingOperationID = operationID + workload.Status.Conditions = upsertCondition(workload.Status.Conditions, metav1.Condition{ + Type: modelRoutingPendingCondition, + Status: metav1.ConditionTrue, + ObservedGeneration: workload.Generation, + Reason: "RoutingIntentPersisted", + Message: fmt.Sprintf("Model routing operation %s is pending", operationID), + LastTransitionTime: metav1.Now(), + }) + + if err := r.Status().Update(ctx, workload); err != nil { + return "", fmt.Errorf("persist routing operation %s: %w", operationID, err) + } + return operationID, nil +} + func (r *AgentWorkloadReconciler) routeAndCallModel( ctx context.Context, workload *agenticv1alpha1.AgentWorkload, @@ -825,6 +886,7 @@ func (r *AgentWorkloadReconciler) routeAndCallModel( workload.Namespace, &workload.Spec, instructions, + persistedModelRoutingOperationID(workload), ) if err != nil { @@ -832,21 +894,19 @@ func (r *AgentWorkloadReconciler) routeAndCallModel( return nil, routingInfo, err } - go func() { - recordCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - if recordErr := r.CostReporter.RecordUsage( - recordCtx, - workload.Name, - workload.Namespace, - routingInfo.ProviderName+"/"+routingInfo.ModelName, - int64(routingInfo.InputTokens), - int64(routingInfo.OutputTokens), - ); recordErr != nil { - log.Error(recordErr, "failed to record usage") - } - }() + recordCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if recordErr := r.CostReporter.RecordUsage( + recordCtx, + persistedModelRoutingOperationID(workload), + workload.Name, + workload.Namespace, + routingInfo.ProviderName+"/"+routingInfo.ModelName, + int64(routingInfo.InputTokens), + int64(routingInfo.OutputTokens), + ); recordErr != nil { + log.Error(recordErr, "failed to record usage") + } if annotateErr := r.updateWorkloadCostAnnotation(ctx, workload); annotateErr != nil { log.Error(annotateErr, "failed to update workload cost annotation") diff --git a/internal/controller/agentworkload_controller_finops_test.go b/internal/controller/agentworkload_controller_finops_test.go index 9a2c9e4..86366e3 100644 --- a/internal/controller/agentworkload_controller_finops_test.go +++ b/internal/controller/agentworkload_controller_finops_test.go @@ -4,6 +4,10 @@ import ( "context" "errors" "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" "testing" "time" @@ -14,6 +18,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" agenticv1alpha1 "github.com/Clawdlinux/agentic-operator-core/api/v1alpha1" + "github.com/Clawdlinux/agentic-operator-core/pkg/finops" + "github.com/Clawdlinux/agentic-operator-core/pkg/resilience" ) type listErrorClient struct { @@ -28,6 +34,51 @@ func (c *listErrorClient) List(ctx context.Context, list client.ObjectList, opts return c.Client.List(ctx, list, opts...) } +type failStatusUpdateClient struct { + client.Client + mu sync.Mutex + updateCount int + failOn int +} + +func (c *failStatusUpdateClient) Status() client.SubResourceWriter { + return &failStatusUpdateWriter{SubResourceWriter: c.Client.Status(), client: c} +} + +type failStatusUpdateWriter struct { + client.SubResourceWriter + client *failStatusUpdateClient +} + +func (w *failStatusUpdateWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + w.client.mu.Lock() + w.client.updateCount++ + updateCount := w.client.updateCount + w.client.mu.Unlock() + + if updateCount == w.client.failOn { + return errors.New("injected status update failure") + } + return w.SubResourceWriter.Update(ctx, obj, opts...) +} + +type capturedIdempotencyKeys struct { + mu sync.Mutex + values []string +} + +func (c *capturedIdempotencyKeys) add(value string) { + c.mu.Lock() + defer c.mu.Unlock() + c.values = append(c.values, value) +} + +func (c *capturedIdempotencyKeys) snapshot() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.values...) +} + type capturingValidator struct { err error calls int @@ -46,18 +97,34 @@ func (v *capturingValidator) RequiresWorkloadCount() bool { } type stubCostReporter struct { - checkBudgetErr error - recordCh chan struct{} - costToday float64 + mu sync.Mutex + checkBudgetErr error + recordCh chan struct{} + costToday float64 + costAfterRecord float64 + releaseRecordOnCostLookup chan struct{} + releaseOnce sync.Once } -func (s *stubCostReporter) RecordUsage(ctx context.Context, workloadName, namespace, model string, promptTokens, completionTokens int64) error { +func (s *stubCostReporter) RecordUsage(ctx context.Context, operationID, workloadName, namespace, model string, promptTokens, completionTokens int64) error { _ = ctx + _ = operationID _ = workloadName _ = namespace _ = model _ = promptTokens _ = completionTokens + if s.releaseRecordOnCostLookup != nil { + select { + case <-s.releaseRecordOnCostLookup: + case <-time.After(50 * time.Millisecond): + } + } + s.mu.Lock() + if s.costAfterRecord > 0 { + s.costToday = s.costAfterRecord + } + s.mu.Unlock() select { case s.recordCh <- struct{}{}: default: @@ -76,6 +143,11 @@ func (s *stubCostReporter) WorkloadCostToday(ctx context.Context, workloadName, _ = ctx _ = workloadName _ = namespace + s.mu.Lock() + defer s.mu.Unlock() + if s.releaseRecordOnCostLookup != nil { + s.releaseOnce.Do(func() { close(s.releaseRecordOnCostLookup) }) + } return s.costToday, nil } @@ -152,6 +224,206 @@ func TestReconcile_ValidatorReceivesConcurrentWorkloadCount(t *testing.T) { } } +func TestReconcile_CostAwareRoutingRunsOncePerSuccessfulGeneration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := newControllerTestScheme(t) + mockServer, requestCount := newCountingOpenAIServer(t, 0) + workload, providerSecret := newCostAwareReconcileWorkload("routing-once", mockServer.URL) + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&agenticv1alpha1.AgentWorkload{}). + WithObjects( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: workload.Namespace}}, + providerSecret, + workload, + ). + Build() + reporter := finops.NewMemoryCostReporter() + reconciler := &AgentWorkloadReconciler{Client: k8sClient, Scheme: scheme, CostReporter: reporter} + request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(workload)} + + for attempt := 1; attempt <= 2; attempt++ { + if _, err := reconciler.Reconcile(ctx, request); err != nil { + t.Fatalf("reconcile %d returned error: %v", attempt, err) + } + } + + if got := requestCount.Load(); got != 1 { + t.Fatalf("provider request count = %d, want 1", got) + } + usage := reporter.GetUsage(workload.Name, workload.Namespace) + if usage == nil { + t.Fatal("expected one usage record") + } + if usage.RequestCount != 1 { + t.Fatalf("usage record count = %d, want 1", usage.RequestCount) + } +} + +func TestReconcile_CostAwareRoutingReusesOperationIDAfterStatusFailure(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := newControllerTestScheme(t) + mockServer, requestCount, idempotencyKeys := newCapturingOpenAIServer(t, 0) + workload, providerSecret := newCostAwareReconcileWorkload("routing-status-retry", mockServer.URL) + + baseClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&agenticv1alpha1.AgentWorkload{}). + WithObjects( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: workload.Namespace}}, + providerSecret, + workload, + ). + Build() + k8sClient := &failStatusUpdateClient{Client: baseClient, failOn: 2} + reporter := finops.NewMemoryCostReporter() + retryCfg := resilience.RetryConfig{MaxRetries: 0, InitialBackoff: time.Millisecond, MaxBackoff: time.Millisecond} + reconciler := &AgentWorkloadReconciler{ + Client: k8sClient, + Scheme: scheme, + CostReporter: reporter, + RetryConfig: &retryCfg, + } + request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(workload)} + + if _, err := reconciler.Reconcile(ctx, request); err != nil { + t.Fatalf("first reconcile returned error: %v", err) + } + + persisted := &agenticv1alpha1.AgentWorkload{} + if err := baseClient.Get(ctx, request.NamespacedName, persisted); err != nil { + t.Fatalf("load workload after injected failure: %v", err) + } + if persisted.Status.ModelRoutingOperationID == "" { + t.Fatal("expected routing operation ID to persist before provider call") + } + + if _, err := reconciler.Reconcile(ctx, request); err != nil { + t.Fatalf("retry reconcile returned error: %v", err) + } + + if got := requestCount.Load(); got != 2 { + t.Fatalf("provider request count = %d, want 2", got) + } + keys := idempotencyKeys.snapshot() + if len(keys) != 2 { + t.Fatalf("Idempotency-Key count = %d, want 2", len(keys)) + } + if keys[0] == "" || keys[0] != keys[1] { + t.Fatalf("Idempotency-Key values = %q, want the same non-empty key", keys) + } + usage := reporter.GetUsage(workload.Name, workload.Namespace) + if usage == nil || usage.RequestCount != 1 { + t.Fatalf("usage after provider retry = %#v, want one record", usage) + } +} + +func TestReconcile_CostAwareRoutingRetriesAfterFailedGeneration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + scheme := newControllerTestScheme(t) + mockServer, requestCount := newCountingOpenAIServer(t, 1) + workload, providerSecret := newCostAwareReconcileWorkload("routing-retry", mockServer.URL) + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&agenticv1alpha1.AgentWorkload{}). + WithObjects( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: workload.Namespace}}, + providerSecret, + workload, + ). + Build() + reporter := finops.NewMemoryCostReporter() + retryCfg := resilience.RetryConfig{MaxRetries: 0, InitialBackoff: time.Millisecond, MaxBackoff: time.Millisecond} + reconciler := &AgentWorkloadReconciler{ + Client: k8sClient, + Scheme: scheme, + CostReporter: reporter, + RetryConfig: &retryCfg, + } + request := reconcile.Request{NamespacedName: client.ObjectKeyFromObject(workload)} + + if _, err := reconciler.Reconcile(ctx, request); err != nil { + t.Fatalf("failed reconcile returned error: %v", err) + } + if _, err := reconciler.Reconcile(ctx, request); err != nil { + t.Fatalf("retry reconcile returned error: %v", err) + } + + if got := requestCount.Load(); got != 2 { + t.Fatalf("provider request count = %d, want 2", got) + } + usage := reporter.GetUsage(workload.Name, workload.Namespace) + if usage == nil || usage.RequestCount != 1 { + t.Fatalf("successful usage record count = %v, want 1", usage) + } +} + +func newCostAwareReconcileWorkload(name, endpoint string) (*agenticv1alpha1.AgentWorkload, *corev1.Secret) { + strategy := "cost-aware" + classifier := "default" + objective := "Analyze quarterly revenue data and identify top trends." + secretKey := "api-key" + namespace := "test-routing" + + workload := &agenticv1alpha1.AgentWorkload{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Generation: 7}, + Spec: agenticv1alpha1.AgentWorkloadSpec{ + ModelStrategy: &strategy, + TaskClassifier: &classifier, + Objective: &objective, + Providers: []agenticv1alpha1.LLMProvider{{ + Name: "mock-openai", + Type: "openai-compatible", + Endpoint: &endpoint, + APIKeySecret: &agenticv1alpha1.SecretKeyRef{Name: "provider-secret", Key: &secretKey}, + }}, + ModelMapping: map[string]string{ + "validation": "mock-openai/gpt-3.5-turbo", + "analysis": "mock-openai/gpt-4", + "reasoning": "mock-openai/gpt-4-turbo", + }, + }, + } + providerSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "provider-secret", Namespace: namespace}, + Data: map[string][]byte{"api-key": []byte("test-token")}, + } + return workload, providerSecret +} + +func newCountingOpenAIServer(t *testing.T, failures int64) (*httptest.Server, *atomic.Int64) { + t.Helper() + server, requestCount, _ := newCapturingOpenAIServer(t, failures) + return server, requestCount +} + +func newCapturingOpenAIServer(t *testing.T, failures int64) (*httptest.Server, *atomic.Int64, *capturedIdempotencyKeys) { + t.Helper() + + requestCount := &atomic.Int64{} + idempotencyKeys := &capturedIdempotencyKeys{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + idempotencyKeys.add(r.Header.Get("Idempotency-Key")) + requestNumber := requestCount.Add(1) + if requestNumber <= failures { + http.Error(w, "upstream unavailable", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":21,"completion_tokens":8}}`)) + })) + t.Cleanup(server.Close) + return server, requestCount, idempotencyKeys +} + func TestRouteAndCallModel_BudgetErrorShortCircuitsRouting(t *testing.T) { t.Parallel() @@ -237,7 +509,11 @@ func TestRouteAndCallModel_RecordsUsageAndUpdatesCostAnnotation(t *testing.T) { &agenticv1alpha1.AgentWorkload{ObjectMeta: workload.ObjectMeta, Spec: workload.Spec}, ).Build() - reporter := &stubCostReporter{recordCh: make(chan struct{}, 1), costToday: 1.25} + reporter := &stubCostReporter{ + recordCh: make(chan struct{}, 1), + costAfterRecord: 1.25, + releaseRecordOnCostLookup: make(chan struct{}), + } reconciler := &AgentWorkloadReconciler{Client: k8sClient, Scheme: scheme, CostReporter: reporter} current := &agenticv1alpha1.AgentWorkload{} @@ -256,7 +532,7 @@ func TestRouteAndCallModel_RecordsUsageAndUpdatesCostAnnotation(t *testing.T) { select { case <-reporter.recordCh: case <-time.After(2 * time.Second): - t.Fatalf("expected RecordUsage to be invoked asynchronously") + t.Fatalf("expected RecordUsage to be invoked") } updated := &agenticv1alpha1.AgentWorkload{} @@ -268,7 +544,7 @@ func TestRouteAndCallModel_RecordsUsageAndUpdatesCostAnnotation(t *testing.T) { t.Fatalf("expected cost annotation map to be set") } - want := fmt.Sprintf("%.6f", reporter.costToday) + want := fmt.Sprintf("%.6f", reporter.costAfterRecord) got := updated.Annotations["agentworkload.clawdlinux.io/cost-usd-today"] if got != want { t.Fatalf("expected cost annotation %q, got %q", want, got) diff --git a/pkg/finops/interface.go b/pkg/finops/interface.go index 53e72d5..1bd67a3 100644 --- a/pkg/finops/interface.go +++ b/pkg/finops/interface.go @@ -8,8 +8,11 @@ import "context" // The OSS default implementation is a no-op that logs cost data // but does not enforce limits or report externally. type CostReporter interface { - // RecordUsage records token usage for a workload. Non-blocking. - RecordUsage(ctx context.Context, workloadName, namespace, model string, + // RecordUsage records token usage for a workload before cost queries run. + // It must provide read-after-write visibility before returning. The controller + // calls WorkloadCostToday next to build the workload cost annotation. Repeated + // calls with the same non-empty operationID must not count usage twice. + RecordUsage(ctx context.Context, operationID, workloadName, namespace, model string, promptTokens, completionTokens int64) error // CheckBudget returns an error if the workload has exceeded its diff --git a/pkg/finops/memory_reporter.go b/pkg/finops/memory_reporter.go index d9ee14f..4550367 100644 --- a/pkg/finops/memory_reporter.go +++ b/pkg/finops/memory_reporter.go @@ -38,10 +38,13 @@ type WorkloadUsage struct { // MemoryCostReporter implements CostReporter with in-memory tracking. type MemoryCostReporter struct { - mu sync.RWMutex - usage map[string]*WorkloadUsage // key: "namespace/workloadName" - pricing map[string]ModelPricing // key: "provider/model" - budget map[string]float64 // key: "namespace/workloadName" → max USD + mu sync.RWMutex + usage map[string]*WorkloadUsage // key: "namespace/workloadName" + // Process-local and unbounded by design for the OSS/demo reporter. Production + // reporters must persist, deduplicate, and expire operation IDs. + recordedOperations map[string]struct{} // key: deterministic operation ID + pricing map[string]ModelPricing // key: "provider/model" + budget map[string]float64 // key: "namespace/workloadName" → max USD // Prometheus metrics costGauge *prometheus.GaugeVec @@ -75,21 +78,23 @@ func NewMemoryCostReporter() *MemoryCostReporter { ) return &MemoryCostReporter{ - usage: make(map[string]*WorkloadUsage), - pricing: defaultPricing(), - budget: make(map[string]float64), - costGauge: costGauge, - costDollarsGauge: costDollarsGauge, - tokensCount: tokensCount, + usage: make(map[string]*WorkloadUsage), + recordedOperations: make(map[string]struct{}), + pricing: defaultPricing(), + budget: make(map[string]float64), + costGauge: costGauge, + costDollarsGauge: costDollarsGauge, + tokensCount: tokensCount, } } func defaultPricing() map[string]ModelPricing { return map[string]ModelPricing{ // OpenAI - "openai/gpt-4o-mini": {InputPer1KTokens: 0.00015, OutputPer1KTokens: 0.0006}, - "openai/gpt-4o": {InputPer1KTokens: 0.005, OutputPer1KTokens: 0.015}, - "openai/gpt-4-turbo": {InputPer1KTokens: 0.01, OutputPer1KTokens: 0.03}, + "openai/gpt-4o-mini": {InputPer1KTokens: 0.00015, OutputPer1KTokens: 0.0006}, + "openai/gpt-4o": {InputPer1KTokens: 0.005, OutputPer1KTokens: 0.015}, + "openai/gpt-4-turbo": {InputPer1KTokens: 0.01, OutputPer1KTokens: 0.03}, + "litellm/clawdlinux-openai": {InputPer1KTokens: 0.00015, OutputPer1KTokens: 0.0006}, // Anthropic "anthropic/claude-sonnet": {InputPer1KTokens: 0.003, OutputPer1KTokens: 0.015}, "anthropic/claude-haiku": {InputPer1KTokens: 0.00025, OutputPer1KTokens: 0.00125}, @@ -126,13 +131,20 @@ func (m *MemoryCostReporter) estimateCost(model string, promptTokens, completion (float64(completionTokens) / 1000.0 * p.OutputPer1KTokens) } -// RecordUsage records token usage and updates estimated cost. -func (m *MemoryCostReporter) RecordUsage(ctx context.Context, workloadName, namespace, model string, +// RecordUsage records token usage and updates estimated cost once per operation ID. +func (m *MemoryCostReporter) RecordUsage(ctx context.Context, operationID, workloadName, namespace, model string, promptTokens, completionTokens int64) error { m.mu.Lock() defer m.mu.Unlock() + if operationID != "" { + if _, recorded := m.recordedOperations[operationID]; recorded { + return nil + } + m.recordedOperations[operationID] = struct{}{} + } + k := m.key(workloadName, namespace) u := m.getOrCreateUsage(k) cost := m.estimateCost(model, promptTokens, completionTokens) @@ -153,6 +165,7 @@ func (m *MemoryCostReporter) RecordUsage(ctx context.Context, workloadName, name "finops: cost recorded", "workload", workloadName, "namespace", namespace, + "operationID", operationID, "model", model, "promptTokens", promptTokens, "completionTokens", completionTokens, diff --git a/pkg/finops/memory_reporter_test.go b/pkg/finops/memory_reporter_test.go index edaa1a0..7880d8f 100644 --- a/pkg/finops/memory_reporter_test.go +++ b/pkg/finops/memory_reporter_test.go @@ -12,7 +12,7 @@ func TestMemoryCostReporter_RecordAndQuery(t *testing.T) { ctx := context.Background() // Record some usage - err := r.RecordUsage(ctx, "test-workload", "test-ns", "openai/gpt-4o-mini", 1000, 500) + err := r.RecordUsage(ctx, "record-and-query", "test-workload", "test-ns", "openai/gpt-4o-mini", 1000, 500) if err != nil { t.Fatalf("RecordUsage failed: %v", err) } @@ -46,6 +46,48 @@ func TestMemoryCostReporter_RecordAndQuery(t *testing.T) { } } +func TestMemoryCostReporter_DuplicateOperationIDDoesNotDoubleCount(t *testing.T) { + reporter := NewMemoryCostReporter() + ctx := context.Background() + + for attempt := 0; attempt < 2; attempt++ { + if err := reporter.RecordUsage(ctx, "operation-123", "test-workload", "test-ns", "openai/gpt-4o-mini", 1000, 500); err != nil { + t.Fatalf("RecordUsage attempt %d failed: %v", attempt+1, err) + } + } + + usage := reporter.GetUsage("test-workload", "test-ns") + if usage == nil { + t.Fatal("expected usage record") + } + if usage.RequestCount != 1 { + t.Fatalf("request count = %d, want 1", usage.RequestCount) + } + if usage.TotalPromptTokens != 1000 { + t.Fatalf("prompt tokens = %d, want 1000", usage.TotalPromptTokens) + } + if usage.TotalCompletionTokens != 500 { + t.Fatalf("completion tokens = %d, want 500", usage.TotalCompletionTokens) + } +} + +func TestMemoryCostReporter_LiteLLMOpenAIAliasPricing(t *testing.T) { + reporter := NewMemoryCostReporter() + ctx := context.Background() + + if err := reporter.RecordUsage(ctx, "litellm-alias", "booth-demo", "agentic-system", "litellm/clawdlinux-openai", 1000, 500); err != nil { + t.Fatalf("RecordUsage failed: %v", err) + } + + cost, err := reporter.WorkloadCostToday(ctx, "booth-demo", "agentic-system") + if err != nil { + t.Fatalf("WorkloadCostToday failed: %v", err) + } + if cost < 0.0004 || cost > 0.0005 { + t.Fatalf("expected GPT-4o-mini alias cost near $0.00045, got $%.6f", cost) + } +} + func TestMemoryCostReporter_BudgetEnforcement(t *testing.T) { r := NewMemoryCostReporter() ctx := context.Background() @@ -54,7 +96,7 @@ func TestMemoryCostReporter_BudgetEnforcement(t *testing.T) { r.SetBudget("budget-test", "test-ns", 0.001) // First call should be under budget - err := r.RecordUsage(ctx, "budget-test", "test-ns", "openai/gpt-4o", 1000, 1000) + err := r.RecordUsage(ctx, "budget-enforcement", "budget-test", "test-ns", "openai/gpt-4o", 1000, 1000) if err != nil { t.Fatalf("RecordUsage failed: %v", err) } @@ -74,7 +116,7 @@ func TestMemoryCostReporter_NoBudgetUnlimited(t *testing.T) { ctx := context.Background() // Record large usage with no budget set - err := r.RecordUsage(ctx, "no-budget", "test-ns", "openai/gpt-4o", 100000, 100000) + err := r.RecordUsage(ctx, "no-budget", "no-budget", "test-ns", "openai/gpt-4o", 100000, 100000) if err != nil { t.Fatalf("RecordUsage failed: %v", err) } @@ -103,7 +145,7 @@ func TestMemoryCostReporter_OllamaFree(t *testing.T) { r := NewMemoryCostReporter() ctx := context.Background() - err := r.RecordUsage(ctx, "ollama-test", "test-ns", "ollama/gemma3:1b", 5000, 3000) + err := r.RecordUsage(ctx, "ollama-free", "ollama-test", "test-ns", "ollama/gemma3:1b", 5000, 3000) if err != nil { t.Fatalf("RecordUsage failed: %v", err) } @@ -119,7 +161,7 @@ func TestMemoryCostReporter_PrometheusCollectorEmitsClawdlinuxCostMetric(t *test r := NewMemoryCostReporter() ctx := context.Background() - if err := r.RecordUsage(ctx, "demo-workload", "demo-ns", "openai/gpt-4o-mini", 1000, 500); err != nil { + if err := r.RecordUsage(ctx, "prometheus-metric", "demo-workload", "demo-ns", "openai/gpt-4o-mini", 1000, 500); err != nil { t.Fatalf("RecordUsage failed: %v", err) } diff --git a/pkg/finops/noop.go b/pkg/finops/noop.go index ccf77e5..8d0a2dc 100644 --- a/pkg/finops/noop.go +++ b/pkg/finops/noop.go @@ -15,9 +15,10 @@ func NewNoOpCostReporter() *NoOpCostReporter { return &NoOpCostReporter{} } -func (n *NoOpCostReporter) RecordUsage(ctx context.Context, workloadName, namespace, model string, promptTokens, completionTokens int64) error { +func (n *NoOpCostReporter) RecordUsage(ctx context.Context, operationID, workloadName, namespace, model string, promptTokens, completionTokens int64) error { logf.FromContext(ctx).V(1).Info( "finops: no-op reporter, cost not enforced", + "operationID", operationID, "workload", workloadName, "namespace", namespace, "model", model, diff --git a/pkg/llm/mock_provider.go b/pkg/llm/mock_provider.go index a9dd1b7..e859a03 100644 --- a/pkg/llm/mock_provider.go +++ b/pkg/llm/mock_provider.go @@ -37,7 +37,8 @@ func (p *MockOpenAIProvider) Type() string { } // CallModel returns a mock response for testing -func (p *MockOpenAIProvider) CallModel(ctx context.Context, model string, prompt string) (*ModelResponse, error) { +func (p *MockOpenAIProvider) CallModel(ctx context.Context, operationID, model, prompt string) (*ModelResponse, error) { + _ = operationID // Track call count p.callCount[model]++ diff --git a/pkg/llm/provider.go b/pkg/llm/provider.go index 3a29e39..978be56 100644 --- a/pkg/llm/provider.go +++ b/pkg/llm/provider.go @@ -16,8 +16,9 @@ import ( // Provider defines the interface for LLM providers type Provider interface { - // CallModel sends a prompt to the model and returns the response - CallModel(ctx context.Context, model string, prompt string) (*ModelResponse, error) + // CallModel sends a prompt to the model and returns the response. Providers + // receive a stable operation ID for upstream idempotency support. + CallModel(ctx context.Context, operationID, model, prompt string) (*ModelResponse, error) // Name returns the provider name Name() string @@ -74,7 +75,7 @@ func (p *OpenAICompatibleProvider) Type() string { } // CallModel sends a request to the OpenAI-compatible API -func (p *OpenAICompatibleProvider) CallModel(ctx context.Context, model string, prompt string) (*ModelResponse, error) { +func (p *OpenAICompatibleProvider) CallModel(ctx context.Context, operationID, model, prompt string) (*ModelResponse, error) { // Cloudflare Workers AI requires model names prefixed with "@cf/" // If provider is Cloudflare and model doesn't already have the prefix, add it. if (strings.Contains(p.name, "cloudflare") || strings.Contains(p.name, "workers-ai")) && @@ -104,6 +105,9 @@ func (p *OpenAICompatibleProvider) CallModel(ctx context.Context, model string, req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", p.apiKey)) + if operationID != "" { + req.Header.Set("Idempotency-Key", operationID) + } // Send request client := &http.Client{} diff --git a/pkg/llm/provider_test.go b/pkg/llm/provider_test.go new file mode 100644 index 0000000..d1da02d --- /dev/null +++ b/pkg/llm/provider_test.go @@ -0,0 +1,30 @@ +package llm + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestOpenAICompatibleProvider_PropagatesIdempotencyKey(t *testing.T) { + t.Parallel() + + const operationID = "agentworkload-7-abc123" + var gotIdempotencyKey string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + gotIdempotencyKey = request.Header.Get("Idempotency-Key") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":2,"completion_tokens":1}}`)) + })) + t.Cleanup(server.Close) + + provider := NewOpenAICompatibleProvider("test-provider", server.URL, "test-token") + if _, err := provider.CallModel(context.Background(), operationID, "test-model", "test prompt"); err != nil { + t.Fatalf("CallModel failed: %v", err) + } + + if gotIdempotencyKey != operationID { + t.Fatalf("Idempotency-Key = %q, want %q", gotIdempotencyKey, operationID) + } +} diff --git a/pkg/llm/routing.go b/pkg/llm/routing.go index 7a2490c..0673c9d 100644 --- a/pkg/llm/routing.go +++ b/pkg/llm/routing.go @@ -34,6 +34,7 @@ func (mr *ModelRouter) RouteAndCall( namespace string, spec *v1alpha1.AgentWorkloadSpec, instructions string, + operationID string, ) (*ModelResponse, *RoutingInfo, error) { routingInfo := &RoutingInfo{} @@ -132,7 +133,7 @@ func (mr *ModelRouter) RouteAndCall( // Call the model with tracing callCtx, callSpan := StartModelCallSpan(ctx, providerName, modelName) - response, err := provider.CallModel(callCtx, modelName, instructions) + response, err := provider.CallModel(callCtx, operationID, modelName, instructions) if err != nil { SetModelCallAttributes(callSpan, 0, 0, false) callSpan.End() diff --git a/pkg/llm/routing_test.go b/pkg/llm/routing_test.go index 4b35694..7fa4cc6 100644 --- a/pkg/llm/routing_test.go +++ b/pkg/llm/routing_test.go @@ -125,6 +125,7 @@ func TestModelRouterWithMockProvider(t *testing.T) { "test-routing", spec, tc.objective, + "routing-test-operation", ) // We expect errors because there's no real API @@ -183,7 +184,7 @@ func TestModelRouterProviderNotFound(t *testing.T) { }, } - _, _, err := router.RouteAndCall(ctx, client, "default", spec, objective) + _, _, err := router.RouteAndCall(ctx, client, "default", spec, objective, "provider-not-found") // Error is expected because the provider in the mapping is missing if err == nil { @@ -212,7 +213,7 @@ func TestModelRouterMissingModelMapping(t *testing.T) { // No ModelMapping specified } - _, _, err := router.RouteAndCall(ctx, client, "default", spec, objective) + _, _, err := router.RouteAndCall(ctx, client, "default", spec, objective, "missing-model-mapping") if err == nil { t.Errorf("expected error when modelMapping is empty") diff --git a/scripts/demo-booth.sh b/scripts/demo-booth.sh index f3206ef..3efd2e8 100755 --- a/scripts/demo-booth.sh +++ b/scripts/demo-booth.sh @@ -11,6 +11,14 @@ NS_SHARED="${NS_SHARED:-shared-services}" HELM_TIMEOUT="${HELM_TIMEOUT:-180s}" OPERATOR_IMAGE="${OPERATOR_IMAGE:-ghcr.io/clawdlinux/agentic-operator-core/agentic-operator}" OPERATOR_IMAGE_TAG="${OPERATOR_IMAGE_TAG:-latest}" +CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.17.2}" +DEMO_RELEASE="${DEMO_RELEASE:-clawdlinux-demo}" +DEMO_SECRET="${DEMO_SECRET:-clawdlinux-demo-litellm}" +DEMO_ENV_FILE="${DEMO_ENV_FILE:-${REPO_ROOT}/.env}" +RESEARCH_MANIFEST="${REPO_ROOT}/examples/research-agent.yaml" +AUDIT_FIXTURE="${REPO_ROOT}/_staging/booth/attestation-fallback.jsonl" +# Committed demo fixture key. Never use it as a production secret. +AUDIT_DEMO_KEY="booth-demo-2026=bmluZXZpZ2lsLWJvb3RoLWRlbW8tYXR0ZXN0YXRpb24ta2V5LTMyYg==" ALLOW_MANIFEST="${REPO_ROOT}/config/samples/agentworkload_demo_allow.yaml" DENY_MANIFEST="${REPO_ROOT}/config/samples/agentworkload_demo_deny.yaml" @@ -21,6 +29,8 @@ DEMO_PROFILE="${DEMO_PROFILE:-platform}" WITH_SWARM=false RECORD=false CLEANUP=false +DEMO_MODE=legacy +TAMPER_AUDIT=false ORIGINAL_ARGS=("$@") if [[ ( -t 1 || "${FORCE_COLOR:-}" == "1" ) && -z "${NO_COLOR:-}" ]]; then @@ -46,6 +56,9 @@ Usage: $(basename "$0") [OPTIONS] Runs the Clawdlinux booth demo gate. Options: + --prepare Create and prepare the real-provider kind demo cluster. + --present Run the 5-7 minute real-provider booth presentation. + --tamper-audit Tamper with the prior-run audit fixture and prove rejection. --cluster NAME kind cluster name. Default: ${CLUSTER_NAME} --profile NAME Deployment profile: platform or lean. Default: ${DEMO_PROFILE} --with-swarm Run the optional Argo multi-agent swarm scenario. @@ -61,6 +74,8 @@ Environment: NS_OPERATOR Operator namespace. Default: ${NS_OPERATOR} OPERATOR_IMAGE Lean-profile operator image repository. Default: ${OPERATOR_IMAGE} OPERATOR_IMAGE_TAG Lean-profile operator image tag. Default: ${OPERATOR_IMAGE_TAG} + CERT_MANAGER_VERSION Pinned cert-manager chart version. Default: ${CERT_MANAGER_VERSION} + DEMO_ENV_FILE Credential file. Default: ${REPO_ROOT}/.env FORCE_COLOR Set to 1 to preserve ANSI colors through a recorder or pipe. EOF } @@ -74,6 +89,20 @@ die() { printf '%b[FAIL]%b %s\n' "${RED}" "${RESET}" "$*" >&2; exit 1; } parse_args() { while (($#)); do case "$1" in + --prepare) + [[ "${DEMO_MODE}" == "legacy" || "${DEMO_MODE}" == "prepare" ]] || die "--prepare and --present are mutually exclusive" + DEMO_MODE=prepare + shift + ;; + --present) + [[ "${DEMO_MODE}" == "legacy" || "${DEMO_MODE}" == "present" ]] || die "--prepare and --present are mutually exclusive" + DEMO_MODE=present + shift + ;; + --tamper-audit) + TAMPER_AUDIT=true + shift + ;; --cluster) [[ $# -ge 2 ]] || die "--cluster requires a value" CLUSTER_NAME="$2" @@ -208,6 +237,535 @@ ensure_namespace() { kubectl create namespace "$1" --dry-run=client -o yaml | kubectl apply -f - >/dev/null } +trim_horizontal_space() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s' "${value}" +} + +parse_demo_credential_value() { + local key="$1" + local raw_value="$2" + local source_file="$3" + local line_number="$4" + local value + value="$(trim_horizontal_space "${raw_value}")" + + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* || "${value}" == *'\' ]]; then + die "malformed definition for ${key} in ${source_file}:${line_number}" + fi + if [[ "${value}" == "'"* ]]; then + [[ ${#value} -ge 2 && "${value: -1}" == "'" ]] || die "malformed definition for ${key} in ${source_file}:${line_number}" + value="${value:1:${#value}-2}" + elif [[ "${value}" == '"'* ]]; then + [[ ${#value} -ge 2 && "${value: -1}" == '"' ]] || die "malformed definition for ${key} in ${source_file}:${line_number}" + value="${value:1:${#value}-2}" + fi + + printf '%s' "${value}" +} + +reject_credential_newlines() { + local key="$1" + local value="$2" + local source="$3" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + die "unsafe newline in ${key} from ${source}" + fi +} + +load_demo_credentials() { + local openai_from_environment=false + local anthropic_from_environment=false + local openai_definitions=0 + local anthropic_definitions=0 + local line line_number=0 key raw_value parsed_value + + if [[ ${OPENAI_API_KEY+x} == x ]]; then + reject_credential_newlines "OPENAI_API_KEY" "${OPENAI_API_KEY}" "environment" + openai_from_environment=true + OPENAI_API_KEY_SOURCE="environment" + fi + if [[ ${ANTHROPIC_API_KEY+x} == x ]]; then + reject_credential_newlines "ANTHROPIC_API_KEY" "${ANTHROPIC_API_KEY}" "environment" + anthropic_from_environment=true + ANTHROPIC_API_KEY_SOURCE="environment" + fi + + if [[ ! -e "${DEMO_ENV_FILE}" ]]; then + return 0 + fi + [[ -f "${DEMO_ENV_FILE}" && -r "${DEMO_ENV_FILE}" ]] || die "credential file is not readable: ${DEMO_ENV_FILE}" + + while IFS= read -r line || [[ -n "${line}" ]]; do + ((line_number += 1)) + if [[ "${line}" == *$'\r' ]]; then + line="${line%$'\r'}" + fi + + if [[ "${line}" =~ ^[[:space:]]*(export[[:space:]]+)?(OPENAI_API_KEY|ANTHROPIC_API_KEY)([[:space:]]|=|$) ]]; then + key="${BASH_REMATCH[2]}" + else + continue + fi + + case "${key}" in + OPENAI_API_KEY) + ((openai_definitions += 1)) + ((openai_definitions == 1)) || die "duplicate definition for OPENAI_API_KEY in ${DEMO_ENV_FILE}" + ;; + ANTHROPIC_API_KEY) + ((anthropic_definitions += 1)) + ((anthropic_definitions == 1)) || die "duplicate definition for ANTHROPIC_API_KEY in ${DEMO_ENV_FILE}" + ;; + esac + + if [[ "${line}" =~ ^[[:space:]]*(export[[:space:]]+)?(OPENAI_API_KEY|ANTHROPIC_API_KEY)[[:space:]]*=(.*)$ ]]; then + raw_value="${BASH_REMATCH[3]}" + else + die "malformed definition for ${key} in ${DEMO_ENV_FILE}:${line_number}" + fi + parsed_value="$(parse_demo_credential_value "${key}" "${raw_value}" "${DEMO_ENV_FILE}" "${line_number}")" + + case "${key}" in + OPENAI_API_KEY) + if [[ "${openai_from_environment}" != "true" ]]; then + OPENAI_API_KEY="${parsed_value}" + export OPENAI_API_KEY + OPENAI_API_KEY_SOURCE="${DEMO_ENV_FILE}" + fi + ;; + ANTHROPIC_API_KEY) + if [[ "${anthropic_from_environment}" != "true" ]]; then + ANTHROPIC_API_KEY="${parsed_value}" + export ANTHROPIC_API_KEY + ANTHROPIC_API_KEY_SOURCE="${DEMO_ENV_FILE}" + fi + ;; + esac + done <"${DEMO_ENV_FILE}" +} + +require_real_provider_keys() { + local missing=() + load_demo_credentials + [[ -n "${OPENAI_API_KEY:-}" ]] || missing+=(OPENAI_API_KEY) + [[ -n "${ANTHROPIC_API_KEY:-}" ]] || missing+=(ANTHROPIC_API_KEY) + if ((${#missing[@]})); then + printf '%b[FAIL]%b Real-provider preparation requires: %s\n' "${RED}" "${RESET}" "${missing[*]}" >&2 + printf 'Export the missing values or define them in %s, then rerun.\n' "${DEMO_ENV_FILE}" >&2 + exit 1 + fi + printf 'OPENAI_API_KEY=available\n' + printf 'ANTHROPIC_API_KEY=available\n' + printf 'Credential variable OPENAI_API_KEY loaded from %s\n' "${OPENAI_API_KEY_SOURCE}" + printf 'Credential variable ANTHROPIC_API_KEY loaded from %s\n' "${ANTHROPIC_API_KEY_SOURCE}" +} + +require_demo_context() { + local expected_context="kind-${CLUSTER_NAME}" + local current_context + current_context="$(kubectl config current-context 2>/dev/null || true)" + [[ "${current_context}" == "${expected_context}" ]] || { + die "wrong kubectl context: ${current_context:-none}. Expected ${expected_context}. Run --prepare first." + } +} + +ensure_demo_kind_cluster() { + step "Preparing kind cluster ${CLUSTER_NAME}" + require_command kind + if kind get clusters | grep -Fxq "${CLUSTER_NAME}"; then + ok "Reusing kind cluster ${CLUSTER_NAME}" + else + kind create cluster --name "${CLUSTER_NAME}" --wait 120s + ok "Created kind cluster ${CLUSTER_NAME}" + fi + kubectl config use-context "kind-${CLUSTER_NAME}" >/dev/null + require_demo_context +} + +install_cert_manager() { + step "Installing cert-manager ${CERT_MANAGER_VERSION}" + helm upgrade --install cert-manager oci://quay.io/jetstack/charts/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version "${CERT_MANAGER_VERSION}" \ + --set crds.enabled=true \ + --timeout "${HELM_TIMEOUT}" \ + --wait >/dev/null + kubectl -n cert-manager rollout status deployment/cert-manager --timeout="${HELM_TIMEOUT}" >/dev/null + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout="${HELM_TIMEOUT}" >/dev/null + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout="${HELM_TIMEOUT}" >/dev/null + ok "cert-manager is ready" +} + +create_runtime_provider_secret() { + step "Creating runtime provider Secret" + local master_key + if [[ -n "${LITELLM_MASTER_KEY:-}" ]]; then + reject_credential_newlines "LITELLM_MASTER_KEY" "${LITELLM_MASTER_KEY}" "environment" + master_key="${LITELLM_MASTER_KEY}" + else + require_command openssl + master_key="$(openssl rand -hex 24)" + fi + + { + printf 'OPENAI_API_KEY=%s\n' "${OPENAI_API_KEY}" + printf 'ANTHROPIC_API_KEY=%s\n' "${ANTHROPIC_API_KEY}" + printf 'LITELLM_MASTER_KEY=%s\n' "${master_key}" + printf 'api-key=%s\n' "${master_key}" + } | kubectl -n "${NS_OPERATOR}" create secret generic "${DEMO_SECRET}" \ + --from-env-file=/dev/stdin \ + --dry-run=client \ + -o yaml | kubectl apply -f - >/dev/null + unset master_key + ok "Runtime Secret ${DEMO_SECRET} created without printing key values" +} + +load_local_operator_image() { + local operator_image="${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG}" + if command -v docker >/dev/null 2>&1 && docker image inspect "${operator_image}" >/dev/null 2>&1; then + kind load docker-image "${operator_image}" --name "${CLUSTER_NAME}" >/dev/null + ok "Loaded local operator image ${operator_image}" + else + log "Local operator image not found. Helm will pull ${operator_image}." + fi +} + +wait_for_demo_components() { + step "Waiting for webhook, operator, and LiteLLM" + kubectl -n "${NS_OPERATOR}" wait \ + --for=condition=Ready \ + "certificate/${DEMO_RELEASE}-agentic-operator-webhook-cert" \ + --timeout="${HELM_TIMEOUT}" >/dev/null + + local operator_deployment + operator_deployment="$(kubectl -n "${NS_OPERATOR}" get deployment \ + -l app.kubernetes.io/name=agentic-operator \ + -o jsonpath='{.items[0].metadata.name}')" + [[ -n "${operator_deployment}" ]] || die "operator Deployment not found" + kubectl -n "${NS_OPERATOR}" rollout status "deployment/${operator_deployment}" --timeout="${HELM_TIMEOUT}" >/dev/null + kubectl -n "${NS_OPERATOR}" rollout status "deployment/${DEMO_RELEASE}-litellm" --timeout="${HELM_TIMEOUT}" >/dev/null + + local deadline endpoint_ip + deadline=$(( $(date +%s) + 120 )) + while (( $(date +%s) < deadline )); do + endpoint_ip="$(kubectl -n "${NS_OPERATOR}" get endpoints "${DEMO_RELEASE}-webhook-service" \ + -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)" + [[ -n "${endpoint_ip}" ]] && break + sleep 2 + done + [[ -n "${endpoint_ip:-}" ]] || die "webhook Service has no ready endpoint" + ok "Webhook, operator, and LiteLLM are ready" +} + +prepare_real_demo() { + step "Preparing real-provider booth demo" + require_real_provider_keys + require_command kubectl + require_command helm + ensure_demo_kind_cluster + install_cert_manager + ensure_namespace "${NS_OPERATOR}" + create_runtime_provider_secret + + step "Applying CRDs before the Helm release" + kubectl apply -f "${REPO_ROOT}/config/crd/bases" >/dev/null + kubectl wait --for=condition=Established crd/agentworkloads.agentic.clawdlinux.org --timeout=60s >/dev/null + ok "AgentWorkload CRD established" + + load_local_operator_image + + step "Installing Clawdlinux booth components" + helm upgrade --install "${DEMO_RELEASE}" "${REPO_ROOT}/charts" \ + --namespace "${NS_OPERATOR}" \ + --create-namespace \ + --set-string license.key=dev-license \ + --set argo.enabled=false \ + --set browserless.enabled=false \ + --set minio.enabled=false \ + --set postgresql.enabled=false \ + --set clawdlinuxObservability.enabled=false \ + --set networkPolicy.enabled=true \ + --set ciliumPolicy.enabled=false \ + --set global.runtimeSandbox.enabled=true \ + --set global.runtimeSandbox.createRuntimeClass=true \ + --set agenticOperator.webhook.enabled=true \ + --set-string agentic-operator.env.ENABLE_WEBHOOKS=true \ + --set-string agentic-operator.env.AGENTIC_COST_TRACKING=memory \ + --set agentic-operator.image.repository="${OPERATOR_IMAGE}" \ + --set agentic-operator.image.tag="${OPERATOR_IMAGE_TAG}" \ + --set agentic-operator.image.pullPolicy=IfNotPresent \ + --set litellm.enabled=true \ + --set litellm.replicaCount=1 \ + --set-string litellm.existingSecret="${DEMO_SECRET}" \ + --timeout "${HELM_TIMEOUT}" \ + --wait >/dev/null + + wait_for_demo_components + ok "Preparation complete. Run: $(basename "$0") --present" +} + +assert_runtime_secret_shape() { + local secret_keys required_key + secret_keys="$(kubectl -n "${NS_OPERATOR}" get secret "${DEMO_SECRET}" \ + -o go-template='{{range $key, $value := .data}}{{$key}}{{"\n"}}{{end}}')" + for required_key in OPENAI_API_KEY ANTHROPIC_API_KEY LITELLM_MASTER_KEY api-key; do + grep -Fxq "${required_key}" <<<"${secret_keys}" || die "Secret ${DEMO_SECRET} is missing key ${required_key}. Run --prepare again." + done + ok "Runtime Secret has all required key names" +} + +apply_research_workload() { + step "REAL: OpenAI-routed AgentWorkload through in-cluster LiteLLM" + kubectl -n "${NS_OPERATOR}" delete agentworkload booth-incident-investigation \ + --ignore-not-found --wait=true --timeout=30s >/dev/null + + local agentctl_command="" + local agentctl_source="" + if [[ -x "${REPO_ROOT}/bin/agentctl" ]]; then + agentctl_command="${REPO_ROOT}/bin/agentctl" + agentctl_source="repo-local agentctl" + elif command -v agentctl >/dev/null 2>&1; then + agentctl_command="$(command -v agentctl)" + agentctl_source="agentctl from PATH" + fi + + if [[ -n "${agentctl_command}" ]]; then + if "${agentctl_command}" apply -f "${RESEARCH_MANIFEST}"; then + ok "Applied with ${agentctl_source}" + return + fi + warn "${agentctl_source} failed. Falling back to kubectl." + fi + + kubectl apply -f "${RESEARCH_MANIFEST}" >/dev/null + if [[ -n "${agentctl_command}" ]]; then + ok "agentctl failed. Applied with kubectl." + else + ok "agentctl unavailable. Applied with kubectl." + fi +} + +wait_for_research_completion() { + local deadline phase + deadline=$(( $(date +%s) + 240 )) + while (( $(date +%s) < deadline )); do + phase="$(kubectl -n "${NS_OPERATOR}" get agentworkload booth-incident-investigation \ + -o jsonpath='{.status.phase}' 2>/dev/null || true)" + case "${phase}" in + Completed) + ok "AgentWorkload reached Completed" + return + ;; + Failed|PolicyDenied) + die "AgentWorkload reached terminal phase ${phase}" + ;; + esac + sleep 2 + done + die "AgentWorkload did not reach Completed. Last phase: ${phase:-empty}" +} + +assert_nonzero_routing_tokens() { + local routing_message="$1" + if [[ ! "${routing_message}" =~ input:([0-9]+)[[:space:]]tokens,[[:space:]]output:([0-9]+)[[:space:]]tokens ]] || + ((10#${BASH_REMATCH[1]} <= 0 || 10#${BASH_REMATCH[2]} <= 0)); then + die "routing condition has missing or zero token counts" + fi +} + +show_real_routing_and_cost() { + step "REAL: Routing, token, and cost evidence" + local routing_message cost_annotation metric_output metric_line metric_value + routing_message="$(kubectl -n "${NS_OPERATOR}" get agentworkload booth-incident-investigation \ + -o jsonpath='{range .status.conditions[?(@.type=="ModelRoutingSucceeded")]}{.message}{end}')" + [[ -n "${routing_message}" ]] || die "ModelRoutingSucceeded condition is missing" + assert_nonzero_routing_tokens "${routing_message}" + printf 'Model routing: %s\n' "${routing_message}" + + cost_annotation="$(kubectl -n "${NS_OPERATOR}" get agentworkload booth-incident-investigation \ + -o go-template='{{index .metadata.annotations "agentworkload.clawdlinux.io/cost-usd-today"}}')" + awk -v cost="${cost_annotation:-0}" 'BEGIN { exit !(cost + 0 > 0) }' || die "cost annotation is missing or zero" + printf 'Cost annotation: $%s\n' "${cost_annotation}" + + local pod_name local_port port_forward_pid + pod_name="$(operator_pod_name)" + [[ -n "${pod_name}" ]] || die "operator pod not found" + local_port="${DEMO_METRICS_PORT:-18080}" + kubectl -n "${NS_OPERATOR}" port-forward "pod/${pod_name}" "${local_port}:8080" >/dev/null 2>&1 & + port_forward_pid=$! + metric_output="" + for _ in {1..20}; do + metric_output="$(curl -fsS --max-time 2 "http://127.0.0.1:${local_port}/metrics" 2>/dev/null || true)" + [[ -n "${metric_output}" ]] && break + sleep 1 + done + kill "${port_forward_pid}" >/dev/null 2>&1 || true + wait "${port_forward_pid}" 2>/dev/null || true + + metric_line="$(printf '%s\n' "${metric_output}" | grep '^clawdlinux_agent_cost_dollars{' | grep 'workload="booth-incident-investigation"' | head -1 || true)" + [[ -n "${metric_line}" ]] || die "clawdlinux_agent_cost_dollars metric is missing for the booth workload" + metric_value="$(awk '{print $NF}' <<<"${metric_line}")" + awk -v cost="${metric_value:-0}" 'BEGIN { exit !(cost + 0 > 0) }' || die "clawdlinux_agent_cost_dollars is zero" + printf 'Cost metric: %s\n' "${metric_line}" +} + +verify_anthropic_route() { + step "REAL: Small Anthropic reachability check through LiteLLM" + local litellm_pod + litellm_pod="$(kubectl -n "${NS_OPERATOR}" get pod \ + -l app.kubernetes.io/name=litellm \ + -o jsonpath='{.items[0].metadata.name}')" + [[ -n "${litellm_pod}" ]] || die "LiteLLM pod not found" + + kubectl -n "${NS_OPERATOR}" exec -i "${litellm_pod}" -- python - <<'PY' +import json +import os +import sys +import urllib.error +import urllib.request + +payload = json.dumps({ + "model": "clawdlinux-anthropic", + "messages": [{"role": "user", "content": "Reply with the word reachable."}], + "max_tokens": 8, + "temperature": 0, +}).encode() +request = urllib.request.Request( + "http://127.0.0.1:4000/v1/chat/completions", + data=payload, + headers={ + "Authorization": "Bearer " + os.environ["LITELLM_MASTER_KEY"], + "Content-Type": "application/json", + }, +) +try: + with urllib.request.urlopen(request, timeout=45) as response: + result = json.load(response) +except urllib.error.HTTPError as exc: + print(f"Anthropic route failed with HTTP {exc.code}", file=sys.stderr) + raise SystemExit(1) +except Exception: + print("Anthropic route failed", file=sys.stderr) + raise SystemExit(1) + +usage = result.get("usage", {}) +input_tokens = usage.get("prompt_tokens", 0) +output_tokens = usage.get("completion_tokens", 0) +if not result.get("choices") or input_tokens <= 0 or output_tokens <= 0: + print("Anthropic route returned no usable completion or token counts", file=sys.stderr) + raise SystemExit(1) +print(f"Anthropic provider reachable through LiteLLM. input_tokens={input_tokens} output_tokens={output_tokens}") +PY + ok "Anthropic provider reachable. This was a separate check, not the AgentWorkload provider." +} + +show_gvisor_configuration_proof() { + step "SIMULATION / CONFIGURATION PROOF: gVisor admission mutation" + local runtime_class + runtime_class="$(kubectl -n "${NS_OPERATOR}" apply --dry-run=server -o jsonpath='{.spec.runtimeClassName}' -f - <<'YAML' +apiVersion: v1 +kind: Pod +metadata: + name: booth-gvisor-dry-run + labels: + agentic.clawdlinux.org/runtime-sandbox: gvisor +spec: + restartPolicy: Never + containers: + - name: pause + image: registry.k8s.io/pause:3.10 +YAML +)" + [[ "${runtime_class}" == "gvisor" ]] || die "webhook dry-run did not inject runtimeClassName=gvisor" + printf 'SIMULATION / CONFIGURATION PROOF: server-side dry-run injected runtimeClassName=%s. No pod was scheduled.\n' "${runtime_class}" +} + +show_network_policy_presence() { + step "NetworkPolicy object" + local policy_names + policy_names="$(kubectl -n "${NS_OPERATOR}" get networkpolicy -o name)" + [[ -n "${policy_names}" ]] || die "NetworkPolicy object not found" + printf '%s\n' "${policy_names}" + printf 'NETWORKPOLICY OBJECT PRESENCE ONLY. Packet enforcement requires an enforcing CNI.\n' +} + +find_audit_verifier() { + if [[ -x "${REPO_ROOT}/bin/audit-verify" ]]; then + printf '%s' "${REPO_ROOT}/bin/audit-verify" + return + fi + if command -v audit-verify >/dev/null 2>&1; then + command -v audit-verify + return + fi + require_command go + mkdir -p "${REPO_ROOT}/bin" + go build -o "${REPO_ROOT}/bin/audit-verify" "${REPO_ROOT}/cmd/audit-verify" + printf '%s' "${REPO_ROOT}/bin/audit-verify" +} + +run_prior_run_audit() { + local tamper="${1:-false}" + step "PRIOR-RUN HMAC-SIGNED AUDIT FIXTURE: offline verification" + [[ -f "${AUDIT_FIXTURE}" ]] || die "missing prior-run audit fixture: ${AUDIT_FIXTURE}" + printf 'PRIOR-RUN ARTIFACT: the current AgentWorkload did not generate this file.\n' + + local verifier + verifier="$(find_audit_verifier)" + "${verifier}" --source jsonl --path "${AUDIT_FIXTURE}" --key "${AUDIT_DEMO_KEY}" + + if [[ "${tamper}" != "true" ]]; then + return + fi + + local tampered_file + tampered_file="$(mktemp "${TMPDIR:-/tmp}/clawdlinux-audit-tampered.XXXXXX.jsonl")" + sed 's/"actor":"policy-analyst"/"actor":"tampered-actor"/' "${AUDIT_FIXTURE}" > "${tampered_file}" + if cmp -s "${AUDIT_FIXTURE}" "${tampered_file}"; then + rm -f "${tampered_file}" + die "tamper operation did not change the prior-run fixture" + fi + if "${verifier}" --source jsonl --path "${tampered_file}" --key "${AUDIT_DEMO_KEY}"; then + rm -f "${tampered_file}" + die "tampered audit artifact unexpectedly verified" + fi + rm -f "${tampered_file}" + ok "Tampered prior-run artifact was rejected" +} + +print_present_summary() { + cat <<'EOF' + +CURRENT --present EVIDENCE +- Real OpenAI-routed model call through LiteLLM. +- Genuine input/output tokens plus nonzero cost metric and annotation. +- Separate Anthropic reachability check. +- Webhook mutation simulation/configuration proof for runtimeClassName=gvisor. No pod was scheduled. +- NetworkPolicy object presence only. Packet enforcement was not tested. +- Prior-run HMAC-signed audit fixture verification. Optional tamper failure. +EOF +} + +present_real_demo() { + require_command kubectl + require_command curl + require_demo_context + [[ -f "${RESEARCH_MANIFEST}" ]] || die "missing ${RESEARCH_MANIFEST}" + assert_runtime_secret_shape + apply_research_workload + wait_for_research_completion + show_real_routing_and_cost + verify_anthropic_route + show_gvisor_configuration_proof + show_network_policy_presence + run_prior_run_audit "${TAMPER_AUDIT}" + print_present_summary +} + wait_for_operator() { step "Waiting for operator pod" local deadline phase pod_name @@ -452,7 +1010,7 @@ verify_network_policy() { } run_opa_demo() { - step "Applying OPA allow workload" + step "LEGACY DEFAULT FLOW: Applying OPA allow workload" kubectl -n "${NS_OPERATOR}" delete agentworkload opa-allow-demo --ignore-not-found >/dev/null 2>&1 || true apply_manifest "${ALLOW_MANIFEST}" if ALLOW_PHASE="$(wait_for_allow_phase opa-allow-demo)"; then @@ -466,7 +1024,7 @@ run_opa_demo() { show_workload_evidence show_agent_pods - step "Applying OPA deny workload" + step "LEGACY DEFAULT FLOW: Applying OPA deny workload" kubectl -n "${NS_OPERATOR}" delete agentworkload opa-deny-demo --ignore-not-found >/dev/null 2>&1 || true apply_manifest "${DENY_MANIFEST}" if DENY_PHASE="$(wait_for_deny_phase opa-deny-demo)"; then @@ -557,8 +1115,8 @@ run_swarm_demo() { print_summary() { echo "" - printf '%bBooth Demo Summary%b\n' "${BOLD}" "${RESET}" - printf 'ALLOW path: %s. DENY path: %s. gVisor: %s. NetworkPolicy: %s. Cost: %s. Swarm: %s.\n' \ + printf '%bLegacy Default Flow Summary%b\n' "${BOLD}" "${RESET}" + printf 'ALLOW path: %s. DENY path: %s. gVisor configuration: %s. NetworkPolicy object: %s. Cost: %s. Swarm: %s.\n' \ "${ALLOW_PHASE:-}" \ "${DENY_PHASE:-}" \ "${GVISOR_OK:-no}" \ @@ -567,12 +1125,36 @@ print_summary() { "${SWARM_PHASE:-skipped (use --with-swarm)}" echo "" if [[ "${DENY_PHASE:-}" != "PolicyDenied" ]]; then - warn "DENY needs a reachable HTTPS MCP endpoint to reach OPA. Set DEMO_MCP_ENDPOINT for live booth runs." + warn "Legacy OPA DENY needs a reachable HTTPS MCP endpoint. Set DEMO_MCP_ENDPOINT for development runs." fi } main() { parse_args "$@" + start_recording_if_requested + + if [[ "${CLEANUP}" == "true" ]]; then + cleanup_cluster + return + fi + + case "${DEMO_MODE}" in + prepare) + prepare_real_demo + return + ;; + present) + present_real_demo + return + ;; + legacy) + if [[ "${TAMPER_AUDIT}" == "true" ]]; then + run_prior_run_audit true + return + fi + ;; + esac + case "${DEMO_PROFILE}" in platform|lean) ;; *) @@ -582,13 +1164,6 @@ main() { if [[ "${WITH_SWARM}" == "true" && "${DEMO_PROFILE}" == "lean" ]]; then die "--with-swarm requires --profile platform because the swarm uses Argo" fi - start_recording_if_requested - - if [[ "${CLEANUP}" == "true" ]]; then - cleanup_cluster - return - fi - check_prerequisites ensure_cluster if [[ "${DEMO_PROFILE}" == "lean" ]]; then @@ -603,4 +1178,6 @@ main() { print_summary } -main "$@" \ No newline at end of file +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi \ No newline at end of file diff --git a/scripts/record-demo.sh b/scripts/record-demo.sh index d673592..6eaa46e 100755 --- a/scripts/record-demo.sh +++ b/scripts/record-demo.sh @@ -8,16 +8,12 @@ mkdir -p demos STAMP="$(date +%Y%m%d)" LOG_FILE="demos/demo-${STAMP}.log" CAST_FILE="demos/demo-${STAMP}.cast" -DEMO_PROFILE="${DEMO_PROFILE:-lean}" -case "${DEMO_PROFILE}" in - platform|lean) ;; - *) - echo "Invalid DEMO_PROFILE: ${DEMO_PROFILE} (expected platform or lean)" >&2 - exit 1 - ;; -esac +DEMO_ARGS=(--present) +if [[ "${DEMO_TAMPER_AUDIT:-false}" == "true" ]]; then + DEMO_ARGS+=(--tamper-audit) +fi -printf -v DEMO_RUNNER '%q ' env FORCE_COLOR=1 ./scripts/demo-booth.sh --profile "${DEMO_PROFILE}" +printf -v DEMO_RUNNER '%q ' env FORCE_COLOR=1 ./scripts/demo-booth.sh "${DEMO_ARGS[@]}" printf -v QUOTED_LOG_FILE '%q' "${LOG_FILE}" DEMO_CMD="set -o pipefail; ${DEMO_RUNNER}| tee ${QUOTED_LOG_FILE}" diff --git a/tests/smoke/run_smoke.sh b/tests/smoke/run_smoke.sh index cbfd24a..055d5bb 100644 --- a/tests/smoke/run_smoke.sh +++ b/tests/smoke/run_smoke.sh @@ -36,6 +36,14 @@ pass() { echo " ✅ PASS: $*"; ((PASS+=1)); ((TOTAL+=1)); } fail() { echo " ❌ FAIL: $*"; ((FAIL+=1)); ((TOTAL+=1)); } log() { echo "[smoke] $*"; } +# ── Local CLI contract ─────────────────────────────────────────────────────── +log "Local CLI: booth demo safety and fallback checks" +if bash "${SCRIPT_DIR}/test_demo_booth_cli.sh"; then + pass "Booth demo CLI contract" +else + fail "Booth demo CLI contract" +fi + # ── Test 1: CRD installed ──────────────────────────────────────────────────── log "Test 1: AgentWorkload CRD exists" if kubectl get crd agentworkloads.agentic.clawdlinux.org >/dev/null 2>&1; then diff --git a/tests/smoke/test_demo_booth_cli.sh b/tests/smoke/test_demo_booth_cli.sh new file mode 100644 index 0000000..e4085a6 --- /dev/null +++ b/tests/smoke/test_demo_booth_cli.sh @@ -0,0 +1,335 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +DEMO_SCRIPT="${ROOT_DIR}/scripts/demo-booth.sh" +RECORD_SCRIPT="${ROOT_DIR}/scripts/record-demo.sh" +RESEARCH_MANIFEST="${ROOT_DIR}/examples/research-agent.yaml" +SMOKE_RUNNER="${ROOT_DIR}/tests/smoke/run_smoke.sh" +TEST_TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/demo-booth-cli.XXXXXX")" +trap 'rm -rf "${TEST_TMP_DIR}"' EXIT + +bash -n "${DEMO_SCRIPT}" +bash -n "${RECORD_SCRIPT}" +help_output="$("${DEMO_SCRIPT}" --help)" + +for flag in --prepare --present --tamper-audit; do + if ! grep -Fq -- "${flag}" <<<"${help_output}"; then + printf 'missing booth mode in --help: %s\n' "${flag}" >&2 + exit 1 + fi +done + +present_summary="$(bash -c 'source "$1"; print_present_summary' _ "${DEMO_SCRIPT}")" +present_evidence=( + 'CURRENT --present EVIDENCE' + '- Real OpenAI-routed model call through LiteLLM.' + '- Genuine input/output tokens plus nonzero cost metric and annotation.' + '- Separate Anthropic reachability check.' + '- Webhook mutation simulation/configuration proof for runtimeClassName=gvisor. No pod was scheduled.' + '- NetworkPolicy object presence only. Packet enforcement was not tested.' + '- Prior-run HMAC-signed audit fixture verification. Optional tamper failure.' +) +for evidence_line in "${present_evidence[@]}"; do + if ! grep -Fxq -- "${evidence_line}" <<<"${present_summary}"; then + printf 'present summary is missing evidence boundary: %s\n' "${evidence_line}" >&2 + exit 1 + fi +done +if grep -Eiq 'OPA|policy (decision|gate)|current-run (signed|audit|attestation)' <<<"${present_summary}"; then + printf 'present summary must not claim current-run policy or signed-audit evidence\n' >&2 + exit 1 +fi + +if ! bash -c 'source "$1"; assert_nonzero_routing_tokens "Task classified as research, routed to openai/gpt-4o-mini (input:21 tokens, output:8 tokens)"' _ "${DEMO_SCRIPT}"; then + printf 'present path must accept genuine nonzero routing token counts\n' >&2 + exit 1 +fi +set +e +zero_token_output="$(bash -c 'source "$1"; assert_nonzero_routing_tokens "Task classified as research, routed to openai/gpt-4o-mini (input:0 tokens, output:8 tokens)"' _ "${DEMO_SCRIPT}" 2>&1)" +zero_token_status=$? +set -e +if [[ ${zero_token_status} -eq 0 ]] || ! grep -Fq 'routing condition has missing or zero token counts' <<<"${zero_token_output}"; then + printf 'present path must reject missing or zero routing token counts\n' >&2 + exit 1 +fi + +required_text=( + 'SIMULATION / CONFIGURATION PROOF' + 'NETWORKPOLICY OBJECT PRESENCE ONLY. Packet enforcement requires an enforcing CNI.' + 'PRIOR-RUN ARTIFACT' + 'OPENAI_API_KEY' + 'ANTHROPIC_API_KEY' + 'clawdlinux-demo-litellm' +) + +for text in "${required_text[@]}"; do + if ! grep -Fq -- "${text}" "${DEMO_SCRIPT}"; then + printf 'missing booth safety or credential marker: %s\n' "${text}" >&2 + exit 1 + fi +done + +if grep -Eq '^[[:space:]]*set[[:space:]]+-[^[:space:]]*x' "${DEMO_SCRIPT}"; then + printf 'demo script must not enable shell tracing\n' >&2 + exit 1 +fi + +set +e +prepare_output="$(env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY DEMO_ENV_FILE="${TEST_TMP_DIR}/missing.env" PATH=/usr/bin:/bin "${DEMO_SCRIPT}" --prepare 2>&1)" +prepare_status=$? +set -e +if [[ ${prepare_status} -eq 0 ]]; then + printf 'prepare must fail when provider keys are absent\n' >&2 + exit 1 +fi +if ! grep -Fq 'Real-provider preparation requires: OPENAI_API_KEY ANTHROPIC_API_KEY' <<<"${prepare_output}"; then + printf 'prepare did not fail at the provider key gate\n' >&2 + exit 1 +fi + +env_file="${TEST_TMP_DIR}/credentials.env" +fake_bin="${TEST_TMP_DIR}/fake-bin" +command_log="${TEST_TMP_DIR}/commands.log" +mkdir -p "${fake_bin}" +file_openai='demo-file-openai-not-secret' +environment_openai='demo-environment-openai-not-secret' +file_anthropic='demo-file-anthropic-not-secret' +cat >"${env_file}" <"${fake_bin}/kind" <<'EOF' +#!/usr/bin/env bash +printf 'kind' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +if [[ "${1:-}" == "get" && "${2:-}" == "clusters" ]]; then + printf '%s\n' "${CLUSTER_NAME:-clawdlinux-demo}" +fi +EOF + +cat >"${fake_bin}/helm" <<'EOF' +#!/usr/bin/env bash +printf 'helm' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +EOF + +cat >"${fake_bin}/openssl" <<'EOF' +#!/usr/bin/env bash +printf 'openssl' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +printf 'generated-test-master-key\n' +EOF + +cat >"${fake_bin}/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'kubectl' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +args=" $* " +case "${args}" in + *" config current-context "*) + printf 'kind-%s\n' "${CLUSTER_NAME:-clawdlinux-demo}" + ;; + *" get deployment "*) + printf 'agentic-operator\n' + ;; + *" get endpoints "*) + printf '127.0.0.1\n' + ;; + *" create namespace "*) + printf 'apiVersion: v1\nkind: Namespace\nmetadata:\n name: test\n' + ;; + *" create secret generic "*) + secret_input="$(cat)" + grep -Fqx "OPENAI_API_KEY=${EXPECTED_OPENAI_API_KEY}" <<<"${secret_input}" + grep -Fqx "ANTHROPIC_API_KEY=${EXPECTED_ANTHROPIC_API_KEY}" <<<"${secret_input}" + printf 'apiVersion: v1\nkind: Secret\nmetadata:\n name: test\n' + ;; + *" apply -f - "*) + cat >/dev/null + ;; +esac +EOF +chmod +x "${fake_bin}/kind" "${fake_bin}/helm" "${fake_bin}/openssl" "${fake_bin}/kubectl" + +set +e +loaded_prepare_output="$(env \ + -u ANTHROPIC_API_KEY \ + -u LITELLM_MASTER_KEY \ + OPENAI_API_KEY="${environment_openai}" \ + DEMO_ENV_FILE="${env_file}" \ + FAKE_COMMAND_LOG="${command_log}" \ + EXPECTED_OPENAI_API_KEY="${environment_openai}" \ + EXPECTED_ANTHROPIC_API_KEY="${file_anthropic}" \ + PATH="${fake_bin}:/usr/bin:/bin" \ + "${DEMO_SCRIPT}" --prepare 2>&1)" +loaded_prepare_status=$? +set -e +if [[ ${loaded_prepare_status} -ne 0 ]]; then + printf 'prepare with a safe env file failed:\n%s\n' "${loaded_prepare_output}" >&2 + exit 1 +fi +for status_line in 'OPENAI_API_KEY=available' 'ANTHROPIC_API_KEY=available'; do + if ! grep -Fxq "${status_line}" <<<"${loaded_prepare_output}"; then + printf 'prepare did not report credential availability: %s\n' "${status_line}" >&2 + exit 1 + fi +done +if ! grep -Fq "Credential variable OPENAI_API_KEY loaded from environment" <<<"${loaded_prepare_output}"; then + printf 'prepare did not report environment precedence for OPENAI_API_KEY\n' >&2 + exit 1 +fi +if ! grep -Fq "Credential variable ANTHROPIC_API_KEY loaded from ${env_file}" <<<"${loaded_prepare_output}"; then + printf 'prepare did not report the env file source for ANTHROPIC_API_KEY\n' >&2 + exit 1 +fi +for dummy_value in "${file_openai}" "${environment_openai}" "${file_anthropic}"; do + if grep -Fq "${dummy_value}" <<<"${loaded_prepare_output}" || grep -Fq "${dummy_value}" "${command_log}"; then + printf 'credential value leaked into prepare output or command log\n' >&2 + exit 1 + fi +done + +duplicate_env_file="${TEST_TMP_DIR}/duplicate.env" +cat >"${duplicate_env_file}" <<'EOF' +OPENAI_API_KEY=first-placeholder +export OPENAI_API_KEY='second-placeholder' +ANTHROPIC_API_KEY=anthropic-placeholder +EOF +set +e +duplicate_output="$(env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY DEMO_ENV_FILE="${duplicate_env_file}" PATH=/usr/bin:/bin "${DEMO_SCRIPT}" --prepare 2>&1)" +duplicate_status=$? +set -e +if [[ ${duplicate_status} -eq 0 ]] || ! grep -Fq 'duplicate definition for OPENAI_API_KEY' <<<"${duplicate_output}"; then + printf 'prepare must reject duplicate credential definitions\n' >&2 + exit 1 +fi + +multiline_env_file="${TEST_TMP_DIR}/multiline.env" +cat >"${multiline_env_file}" <<'EOF' +OPENAI_API_KEY="unsafe +newline" +ANTHROPIC_API_KEY=anthropic-placeholder +EOF +set +e +multiline_output="$(env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY DEMO_ENV_FILE="${multiline_env_file}" PATH=/usr/bin:/bin "${DEMO_SCRIPT}" --prepare 2>&1)" +multiline_status=$? +set -e +if [[ ${multiline_status} -eq 0 ]] || ! grep -Fq 'malformed definition for OPENAI_API_KEY' <<<"${multiline_output}"; then + printf 'prepare must reject multiline credential values\n' >&2 + exit 1 +fi + +set +e +environment_newline_output="$(env \ + OPENAI_API_KEY=$'line-one\nline-two' \ + ANTHROPIC_API_KEY=anthropic-placeholder \ + DEMO_ENV_FILE="${TEST_TMP_DIR}/missing.env" \ + PATH=/usr/bin:/bin \ + "${DEMO_SCRIPT}" --prepare 2>&1)" +environment_newline_status=$? +set -e +if [[ ${environment_newline_status} -eq 0 ]] || ! grep -Fq 'unsafe newline in OPENAI_API_KEY from environment' <<<"${environment_newline_output}"; then + printf 'prepare must reject newlines in exported credential values\n' >&2 + exit 1 +fi + +unsafe_master_values=($'line-one\nline-two' $'line-one\rline-two') +for unsafe_master_value in "${unsafe_master_values[@]}"; do + : >"${command_log}" + set +e + master_key_output="$(env \ + OPENAI_API_KEY=openai-placeholder \ + ANTHROPIC_API_KEY=anthropic-placeholder \ + LITELLM_MASTER_KEY="${unsafe_master_value}" \ + DEMO_ENV_FILE="${TEST_TMP_DIR}/missing.env" \ + FAKE_COMMAND_LOG="${command_log}" \ + PATH="${fake_bin}:/usr/bin:/bin" \ + "${DEMO_SCRIPT}" --prepare 2>&1)" + master_key_status=$? + set -e + if [[ ${master_key_status} -eq 0 ]] || ! grep -Fq 'unsafe newline in LITELLM_MASTER_KEY from environment' <<<"${master_key_output}"; then + printf 'prepare must reject LF and CR in exported LITELLM_MASTER_KEY\n' >&2 + exit 1 + fi + if grep -Fq 'create secret generic' "${command_log}"; then + printf 'prepare must validate LITELLM_MASTER_KEY before Secret creation\n' >&2 + exit 1 + fi +done + +fallback_root="${TEST_TMP_DIR}/fallback-root" +fallback_bin="${TEST_TMP_DIR}/fallback-bin" +fallback_log="${TEST_TMP_DIR}/fallback.log" +mkdir -p "${fallback_root}/examples" "${fallback_bin}" +: >"${fallback_root}/examples/research-agent.yaml" +cat >"${fallback_bin}/agentctl" <<'EOF' +#!/usr/bin/env bash +printf 'agentctl' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +exit 23 +EOF +cat >"${fallback_bin}/kubectl" <<'EOF' +#!/usr/bin/env bash +printf 'kubectl' >>"${FAKE_COMMAND_LOG}" +printf ' %q' "$@" >>"${FAKE_COMMAND_LOG}" +printf '\n' >>"${FAKE_COMMAND_LOG}" +EOF +chmod +x "${fallback_bin}/agentctl" "${fallback_bin}/kubectl" + +set +e +fallback_output="$(FAKE_COMMAND_LOG="${fallback_log}" PATH="${fallback_bin}:/usr/bin:/bin" bash -c ' + source "$1" + REPO_ROOT="$2" + RESEARCH_MANIFEST="$2/examples/research-agent.yaml" + NS_OPERATOR="test-routing" + apply_research_workload +' _ "${DEMO_SCRIPT}" "${fallback_root}" 2>&1)" +fallback_status=$? +set -e +if [[ ${fallback_status} -ne 0 ]]; then + printf 'failed agentctl did not fall back to kubectl:\n%s\n' "${fallback_output}" >&2 + exit 1 +fi +if ! grep -Fq 'agentctl apply -f' "${fallback_log}" || ! grep -Fq 'kubectl apply -f' "${fallback_log}"; then + printf 'fallback path did not attempt agentctl then kubectl\n' >&2 + exit 1 +fi + +if ! grep -Fq 'test_demo_booth_cli.sh' "${SMOKE_RUNNER}"; then + printf 'normal smoke runner must include test_demo_booth_cli.sh\n' >&2 + exit 1 +fi + +if ! grep -Fq -- '--present' "${RECORD_SCRIPT}"; then + printf 'record-demo.sh must record the present path\n' >&2 + exit 1 +fi + +set +e +present_output="$(KUBECONFIG=/dev/null "${DEMO_SCRIPT}" --present 2>&1)" +present_status=$? +set -e +if [[ ${present_status} -eq 0 ]] || ! grep -Fq 'wrong kubectl context' <<<"${present_output}"; then + printf 'present must refuse a non-demo kubectl context\n' >&2 + exit 1 +fi + +if grep -Eq '^[[:space:]]*orchestration:' "${RESEARCH_MANIFEST}"; then + printf 'research workload must not declare orchestration\n' >&2 + exit 1 +fi +if [[ "$(grep -Fc 'litellm/clawdlinux-openai' "${RESEARCH_MANIFEST}")" -ne 3 ]]; then + printf 'all 3 task categories must use the LiteLLM OpenAI alias\n' >&2 + exit 1 +fi + +printf 'demo booth CLI flags: PASS\n' \ No newline at end of file