diff --git a/README.md b/README.md index b4afec95..5abb4bf0 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ Blocked: block-ssn-demo Explore working examples for popular frameworks. - [Customer Support Agent](examples/customer_support_agent/) - PII protection, prompt injection defense, and tool controls -- [Steer Action Demo](examples/steer_action_demo/) - allow, deny, warn, and steer decisions in one workflow +- [Steer Action Demo](examples/steer_action_demo/) - observe, deny, and steer decisions in one workflow - [LangChain](examples/langchain/) - protect a SQL agent from dangerous queries - [CrewAI](examples/crewai/) - combine Agent Control with CrewAI guardrails - [AWS Strands](examples/strands_agents/) - protect Strands workflows and tool calls diff --git a/engine/tests/test_core.py b/engine/tests/test_core.py index 045bf297..9c8da751 100644 --- a/engine/tests/test_core.py +++ b/engine/tests/test_core.py @@ -266,9 +266,9 @@ async def test_parallel_evaluation_starts_all_controls(self): """Test that all controls start before any complete (parallel, not sequential).""" # Given: 3 slow controls controls = [ - make_control(1, "slow1", "test-slow", action="log", config_value="1"), - make_control(2, "slow2", "test-slow", action="log", config_value="2"), - make_control(3, "slow3", "test-slow", action="log", config_value="3"), + make_control(1, "slow1", "test-slow", action="observe", config_value="1"), + make_control(2, "slow2", "test-slow", action="observe", config_value="2"), + make_control(3, "slow3", "test-slow", action="observe", config_value="3"), ] engine = ControlEngine(controls) @@ -298,9 +298,9 @@ async def test_parallel_evaluation_faster_than_sequential(self): # Given: 3 slow controls (each takes ~50ms) controls = [ - make_control(1, "slow1", "test-slow", action="log", config_value="1"), - make_control(2, "slow2", "test-slow", action="log", config_value="2"), - make_control(3, "slow3", "test-slow", action="log", config_value="3"), + make_control(1, "slow1", "test-slow", action="observe", config_value="1"), + make_control(2, "slow2", "test-slow", action="observe", config_value="2"), + make_control(3, "slow3", "test-slow", action="observe", config_value="3"), ] engine = ControlEngine(controls) @@ -334,7 +334,7 @@ async def test_cancel_on_deny_cancels_blocking_tasks(self): """Test that blocking tasks are cancelled when another control denies.""" # Given: A blocker (waits forever) and a denier (returns immediately) controls = [ - make_control(1, "blocker", "test-blocker", action="log", config_value="b"), + make_control(1, "blocker", "test-blocker", action="observe", config_value="b"), make_control(2, "denier", "test-deny", action="deny", config_value="d"), ] engine = ControlEngine(controls) @@ -367,10 +367,10 @@ async def test_cancel_on_deny_with_multiple_blockers(self): """Test that multiple blocking tasks are all cancelled.""" # Given: Multiple blockers and one denier controls = [ - make_control(1, "blocker1", "test-blocker", action="log", config_value="1"), - make_control(2, "blocker2", "test-blocker", action="log", config_value="2"), + make_control(1, "blocker1", "test-blocker", action="observe", config_value="1"), + make_control(2, "blocker2", "test-blocker", action="observe", config_value="2"), make_control(3, "denier", "test-deny", action="deny", config_value="d"), - make_control(4, "blocker3", "test-blocker", action="log", config_value="3"), + make_control(4, "blocker3", "test-blocker", action="observe", config_value="3"), ] engine = ControlEngine(controls) @@ -392,11 +392,11 @@ async def test_cancel_on_deny_with_multiple_blockers(self): @pytest.mark.asyncio async def test_no_cancel_on_non_deny_match(self): - """Test that 'log' action match doesn't cancel other tasks.""" - # Given: A slow task and a matcher with action=log (not deny) + """Test that an observe action match doesn't cancel other tasks.""" + # Given: A slow task and a matcher with action=observe (not deny) controls = [ - make_control(1, "slow", "test-slow", action="log", config_value="s"), - make_control(2, "matcher", "test-deny", action="log", config_value="m"), + make_control(1, "slow", "test-slow", action="observe", config_value="s"), + make_control(2, "matcher", "test-deny", action="observe", config_value="m"), ] engine = ControlEngine(controls) @@ -408,10 +408,10 @@ async def test_no_cancel_on_non_deny_match(self): ) result = await engine.process(request) - # Then: Slow task should complete (not cancelled) because action was 'log' + # Then: Slow task should complete (not cancelled) because action was observe assert "slow:s:end" in _execution_log, "Slow task should complete for non-deny match" - # And: Result should still be safe (log doesn't make it unsafe) + # And: Result should still be safe (observe doesn't make it unsafe) assert result.is_safe is True assert result.matches is not None assert len(result.matches) == 1 @@ -454,9 +454,9 @@ async def test_collect_all_completed_results(self): """Test that all completed results are collected.""" # Given: Multiple quick controls controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="1"), - make_control(2, "deny1", "test-deny", action="log", config_value="d"), - make_control(3, "allow2", "test-allow", action="log", config_value="2"), + make_control(1, "allow1", "test-allow", action="observe", config_value="1"), + make_control(2, "deny1", "test-deny", action="observe", config_value="d"), + make_control(3, "allow2", "test-allow", action="observe", config_value="2"), ] engine = ControlEngine(controls) @@ -595,15 +595,15 @@ async def test_evaluator_error_fails_closed_for_deny(self): async def test_error_does_not_affect_other_controls(self): """Test that one control's error doesn't affect others. - Given: Multiple controls, one (log action) throws an error + Given: Multiple controls, one (observe action) throws an error When: The engine processes the request Then: Other controls still execute and their results are collected """ - # Given: A mix of error and working controls (error has log action) + # Given: A mix of error and working controls (error has observe action) controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), - make_control(2, "error_control", "test-error", action="log", config_value="err"), - make_control(3, "deny1", "test-deny", action="log", config_value="d1"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), + make_control(2, "error_control", "test-error", action="observe", config_value="err"), + make_control(3, "deny1", "test-deny", action="observe", config_value="d1"), ] engine = ControlEngine(controls) @@ -621,7 +621,7 @@ async def test_error_does_not_affect_other_controls(self): assert "deny:d1:start" in _execution_log assert "deny:d1:end" in _execution_log - # And: Should be safe (log action fails open) + # And: Should be safe (observe action fails open) assert result.is_safe is True # Confidence is 2/3 (two successful, one errored) assert abs(result.confidence - 2/3) < 0.01 @@ -632,16 +632,16 @@ async def test_error_does_not_affect_other_controls(self): assert result.matches[0].control_name == "deny1" @pytest.mark.asyncio - async def test_error_with_log_action_fails_open(self): - """Test that errored control with log action fails open. + async def test_error_with_observe_action_fails_open(self): + """Test that errored control with observe action fails open. - Given: A control with action=log that throws an error + Given: A control with action=observe that throws an error When: The engine processes the request Then: The request is safe (fail-open for non-deny actions) """ - # Given: Error control with log action (non-deny) + # Given: Error control with observe action (non-deny) controls = [ - make_control(1, "error_log", "test-error", action="log", config_value="el"), + make_control(1, "error_log", "test-error", action="observe", config_value="el"), ] engine = ControlEngine(controls) @@ -653,7 +653,7 @@ async def test_error_with_log_action_fails_open(self): ) result = await engine.process(request) - # Then: Should be safe - log action fails open + # Then: Should be safe - observe action fails open assert result.is_safe is True # But confidence is 0 (no successful evaluations) assert result.confidence == 0.0 @@ -740,12 +740,12 @@ async def test_errors_array_exposes_evaluator_failures(self): When: The engine processes the request Then: All errored evaluations are in the errors array, deny errors fail closed """ - # Given: Mix of working and errored controls (error1 is deny, error2 is log) + # Given: Mix of working and errored controls (error1 is deny, error2 is observe) controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), make_control(2, "error1", "test-error", action="deny", config_value="e1"), - make_control(3, "deny1", "test-deny", action="warn", config_value="d1"), - make_control(4, "error2", "test-error", action="log", config_value="e2"), + make_control(3, "deny1", "test-deny", action="observe", config_value="d1"), + make_control(4, "error2", "test-error", action="observe", config_value="e2"), ] engine = ControlEngine(controls) @@ -790,8 +790,8 @@ async def test_errors_array_empty_when_no_errors(self): """ # Given: All working controls controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), - make_control(2, "deny1", "test-deny", action="warn", config_value="d1"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), + make_control(2, "deny1", "test-deny", action="observe", config_value="d1"), ] engine = ControlEngine(controls) @@ -831,7 +831,7 @@ async def test_confidence_is_full_on_deny_match(self): controls = [ make_control(1, "denier", "test-deny", action="deny", config_value="d"), ] + [ - make_control(i + 2, f"blocker{i}", "test-blocker", action="log", config_value=str(i)) + make_control(i + 2, f"blocker{i}", "test-blocker", action="observe", config_value=str(i)) for i in range(9) ] engine = ControlEngine(controls) @@ -862,10 +862,10 @@ async def test_confidence_excludes_cancelled_tasks(self): # Given: 2 quick allow controls and 2 blockers, plus 1 deny # The deny will cancel the blockers, but allow controls complete controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), - make_control(2, "allow2", "test-allow", action="log", config_value="a2"), - make_control(3, "blocker1", "test-blocker", action="log", config_value="b1"), - make_control(4, "blocker2", "test-blocker", action="log", config_value="b2"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), + make_control(2, "allow2", "test-allow", action="observe", config_value="a2"), + make_control(3, "blocker1", "test-blocker", action="observe", config_value="b1"), + make_control(4, "blocker2", "test-blocker", action="observe", config_value="b2"), make_control(5, "denier", "test-deny", action="deny", config_value="d"), ] engine = ControlEngine(controls) @@ -894,11 +894,11 @@ async def test_confidence_proportional_without_deny_match(self): When: Some controls error but none match with deny Then: Confidence is successful/evaluated ratio """ - # Given: 2 successful log controls, 1 errored log control + # Given: 2 successful observe controls, 1 errored observe control controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), - make_control(2, "allow2", "test-allow", action="log", config_value="a2"), - make_control(3, "error1", "test-error", action="log", config_value="e1"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), + make_control(2, "allow2", "test-allow", action="observe", config_value="a2"), + make_control(3, "error1", "test-error", action="observe", config_value="e1"), ] engine = ControlEngine(controls) @@ -924,9 +924,9 @@ async def test_confidence_zero_when_deny_errors_despite_other_successes(self): """ # Given: 3 successful controls + 1 deny that errors controls = [ - make_control(1, "allow1", "test-allow", action="log", config_value="a1"), - make_control(2, "allow2", "test-allow", action="log", config_value="a2"), - make_control(3, "allow3", "test-allow", action="log", config_value="a3"), + make_control(1, "allow1", "test-allow", action="observe", config_value="a1"), + make_control(2, "allow2", "test-allow", action="observe", config_value="a2"), + make_control(3, "allow3", "test-allow", action="observe", config_value="a3"), make_control(4, "deny_error", "test-error", action="deny", config_value="de"), ] engine = ControlEngine(controls) @@ -1003,7 +1003,7 @@ async def test_step_names_filters_tasks(self): 1, "allow_copy", "test-allow", - action="log", + action="observe", config_value="copy", step_types=["tool"], path="input", @@ -1013,7 +1013,7 @@ async def test_step_names_filters_tasks(self): 2, "allow_aws", "test-allow", - action="log", + action="observe", config_value="aws", step_types=["tool"], path="input", @@ -1040,7 +1040,7 @@ async def test_step_name_regex_filters_tasks(self): 1, "allow_db", "test-allow", - action="log", + action="observe", config_value="db", step_types=["tool"], path="input", @@ -1050,7 +1050,7 @@ async def test_step_name_regex_filters_tasks(self): 2, "allow_web", "test-allow", - action="log", + action="observe", config_value="web", step_types=["tool"], path="input", @@ -1076,7 +1076,7 @@ async def test_or_semantics_names_or_regex(self): 1, "allow_mixed", "test-allow", - action="log", + action="observe", config_value="mixed", step_types=["tool"], path="input", @@ -1103,7 +1103,7 @@ async def test_path_optional_defaults_to_star(self): 1, "payload_echo", "test-payload-echo", - action="log", + action="observe", config_value="p", step_types=["tool"], path=None, # omit path to use "*" @@ -1131,7 +1131,7 @@ def test_invalid_step_name_regex_rejected(self): "selector": {"path": "input"}, "evaluator": EvaluatorSpec(name="test-allow", config={"value": "x"}), }, - action={"decision": "log"}, + action={"decision": "observe"}, ) @@ -1217,7 +1217,7 @@ async def test_timeout_does_not_affect_fast_evaluators(self): """ # Given: One fast allow and one slow timeout controls = [ - make_control(1, "fast", "test-allow", action="log", config_value="f1"), + make_control(1, "fast", "test-allow", action="observe", config_value="f1"), MockControlWithIdentity( id=2, name="slow_timeout", @@ -1233,7 +1233,7 @@ async def test_timeout_does_not_affect_fast_evaluators(self): config={"value": "slow", "timeout_ms": 100}, ), }, - action={"decision": "log"}, # Log, not deny - so fails open + action={"decision": "observe"}, # Observe, not deny - so fails open ), ), ] @@ -1260,7 +1260,7 @@ async def test_timeout_does_not_affect_fast_evaluators(self): assert len(result.errors) == 1 assert result.errors[0].control_name == "slow_timeout" - # And: Should be safe (log action fails open) + # And: Should be safe (observe action fails open) assert result.is_safe is True # Confidence is 0.5 (1 success, 1 error out of 2) assert result.confidence == 0.5 @@ -1319,7 +1319,7 @@ async def evaluate(self, data: Any) -> EvaluatorResult: # Given: 6 controls (more than the limit of 2) controls = [ - make_control(i, f"ctrl{i}", "test-concurrency", action="log", config_value=str(i)) + make_control(i, f"ctrl{i}", "test-concurrency", action="observe", config_value=str(i)) for i in range(6) ] engine = ControlEngine(controls) @@ -1377,7 +1377,7 @@ async def test_or_short_circuit_records_skipped_trace(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1444,7 +1444,7 @@ async def test_composite_results_preserve_decisive_child_metadata(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1492,7 +1492,7 @@ async def test_and_condition_all_children_match_records_full_trace(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1537,7 +1537,7 @@ async def test_not_condition_inverts_child_result(self): "evaluator": {"name": "test-allow", "config": {"value": "child"}}, } }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1579,7 +1579,7 @@ async def test_not_condition_propagates_child_error_trace(self): "evaluator": {"name": "test-error", "config": {"value": "boom"}}, } }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1628,7 +1628,7 @@ async def test_or_condition_all_children_non_match_records_full_trace(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1679,7 +1679,7 @@ async def test_and_error_records_skipped_children_in_trace(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1735,7 +1735,7 @@ async def test_or_error_records_skipped_children_in_trace(self): }, ] }, - action={"decision": "log"}, + action={"decision": "observe"}, ), ) ] @@ -1830,10 +1830,10 @@ async def test_server_context_only_runs_server_controls(self): """ controls = [ make_control_with_execution( - 1, "local_ctrl", "test-allow", action="log", config_value="loc", execution="sdk" + 1, "local_ctrl", "test-allow", action="observe", config_value="loc", execution="sdk" ), make_control_with_execution( - 2, "server_ctrl", "test-allow", action="log", config_value="srv", execution="server" + 2, "server_ctrl", "test-allow", action="observe", config_value="srv", execution="server" ), ] engine = ControlEngine(controls, context="server") @@ -1860,10 +1860,10 @@ async def test_sdk_context_only_runs_sdk_controls(self): """ controls = [ make_control_with_execution( - 1, "local_ctrl", "test-allow", action="log", config_value="loc", execution="sdk" + 1, "local_ctrl", "test-allow", action="observe", config_value="loc", execution="sdk" ), make_control_with_execution( - 2, "server_ctrl", "test-allow", action="log", config_value="srv", execution="server" + 2, "server_ctrl", "test-allow", action="observe", config_value="srv", execution="server" ), ] engine = ControlEngine(controls, context="sdk") @@ -1890,10 +1890,10 @@ async def test_default_context_is_server(self): """ controls = [ make_control_with_execution( - 1, "local_ctrl", "test-allow", action="log", config_value="loc", execution="sdk" + 1, "local_ctrl", "test-allow", action="observe", config_value="loc", execution="sdk" ), make_control_with_execution( - 2, "server_ctrl", "test-allow", action="log", config_value="srv", execution="server" + 2, "server_ctrl", "test-allow", action="observe", config_value="srv", execution="server" ), ] engine = ControlEngine(controls) # No context param @@ -2015,7 +2015,7 @@ async def test_context_filtering_combined_with_step_scoping(self): controls = [ make_control_with_execution( 1, "local_copy", "test-allow", - action="log", + action="observe", config_value="lc", execution="sdk", step_types=["tool"], @@ -2023,7 +2023,7 @@ async def test_context_filtering_combined_with_step_scoping(self): ), make_control_with_execution( 2, "server_copy", "test-allow", - action="log", + action="observe", config_value="sc", execution="server", step_types=["tool"], diff --git a/examples/README.md b/examples/README.md index e748cf5a..63f3c6a3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,7 +15,7 @@ This directory contains runnable examples for Agent Control. Each example has it | DeepEval | Build a custom evaluator using DeepEval GEval metrics. | https://docs.agentcontrol.dev/examples/deepeval | | Galileo Luna-2 | Toxicity detection and content moderation with Galileo Protect. | https://docs.agentcontrol.dev/examples/galileo-luna2 | | LangChain SQL Agent | Protect a SQL agent from dangerous queries with server-side controls. | https://docs.agentcontrol.dev/examples/langchain-sql | -| Steer Action Demo | Banking transfer agent showcasing allow, deny, warn, and steer actions. | https://docs.agentcontrol.dev/examples/steer-action-demo | +| Steer Action Demo | Banking transfer agent showcasing observe, deny, and steer actions. | https://docs.agentcontrol.dev/examples/steer-action-demo | | AWS Strands | Guardrails for AWS Strands agent workflows and tool calls. | https://docs.agentcontrol.dev/examples/aws-strands | | TypeScript SDK | Consumer-style TypeScript example using the published npm package. | https://docs.agentcontrol.dev/examples/typescript_sdk | diff --git a/examples/crewai/README.md b/examples/crewai/README.md index 2812acf1..6d465f31 100644 --- a/examples/crewai/README.md +++ b/examples/crewai/README.md @@ -25,7 +25,7 @@ Tool executes (LLM call, DB query, etc.) CrewAI Guardrails (quality retries) <- LAYER 3: CrewAI validates quality | v -Return to user (or block / steer / warn) +Return to user (or block / steer / observe) ``` **Agent Control** blocks or steers immediately (security). **CrewAI Guardrails** retry with feedback (quality). Controls are defined on the **server** -- update rules without redeploying agents. @@ -77,7 +77,7 @@ uv run --active python -m secure_research_crew.main ## Examples -### 1. [Steering Financial Agent](./steering_financial_agent/) -- Deny, Steer & Warn +### 1. [Steering Financial Agent](./steering_financial_agent/) -- Deny, Steer & Observe Demonstrates all three Agent Control action types in a wire-transfer scenario: @@ -85,7 +85,7 @@ Demonstrates all three Agent Control action types in a wire-transfer scenario: |--------|----------|---------| | **DENY** | Hard block, no recovery | Sanctioned country, fraud detected | | **STEER** | Pause, guide agent to correct, retry | 2FA required, manager approval needed | -| **WARN** | Log for audit, no blocking | New recipient, unusual activity | +| **OBSERVE** | Record for audit, no blocking | New recipient, unusual activity | ```bash cd examples/crewai/steering_financial_agent @@ -131,7 +131,7 @@ A production-quality **3-agent sequential crew** (Researcher, Analyst, Writer) w | # | Scenario | Agent | Evaluator | Action | Result | |---|----------|-------|-----------|--------|--------| -| 1 | Happy path | All 3 | All | allow | Report generated with sources | +| 1 | Happy path | All 3 | All | observe | Report generated with sources | | 2 | SQL injection | Researcher | SQL | deny | "Multiple SQL statements not allowed" | | 3 | Restricted table | Researcher | LIST | deny | salary_data access blocked | | 4 | Missing methodology | Analyst | JSON Schema | steer | Auto-corrected, passes on retry | @@ -189,7 +189,7 @@ uv run --active python -m content_publishing_flow.main | POST-execution checks | Yes | Yes | Yes | Yes | | **deny** action | Yes | Yes | Yes | Yes | | **steer** action | Yes | Yes | Yes | Yes | -| **warn** action | Yes | | | | +| **observe** action | Yes | | | | | Regex evaluator | | Yes | Yes | Yes | | List evaluator | Yes | Yes | Yes | Yes | | JSON evaluator | Yes | Yes | Yes | Yes | diff --git a/examples/crewai/content_publishing_flow/README.md b/examples/crewai/content_publishing_flow/README.md index cffadfe1..f3090c11 100644 --- a/examples/crewai/content_publishing_flow/README.md +++ b/examples/crewai/content_publishing_flow/README.md @@ -129,7 +129,7 @@ Agent Control works at the **tool boundary** within each flow stage: 3. The server evaluates controls matching the step name and stage (pre/post) 4. On **deny**: `ControlViolationError` is raised, the stage handles it 5. On **steer**: `ControlSteerError` provides corrective guidance for retry -6. On **allow**: execution proceeds normally +6. On **observe** or no match: execution proceeds normally This pattern means controls are defined on the **server** (not in code), so you can update rules, add new controls, or change actions without redeploying the agent. diff --git a/examples/crewai/secure_research_crew/README.md b/examples/crewai/secure_research_crew/README.md index 44afa0ce..b4957501 100644 --- a/examples/crewai/secure_research_crew/README.md +++ b/examples/crewai/secure_research_crew/README.md @@ -46,7 +46,7 @@ Each tool's `step_name` matches the `step_names` in its corresponding control sc | # | Scenario | Agent | Control | Evaluator | Action | Expected | |---|----------|-------|---------|-----------|--------|----------| -| 1 | Happy path | All | All | All | allow | Report generated | +| 1 | Happy path | All | All | All | observe | Report generated | | 2 | SQL injection | Researcher | researcher-sql-safety | SQL | deny | Query blocked | | 3 | Restricted table | Researcher | researcher-restricted-tables | LIST | deny | Query blocked | | 4 | Missing methodology | Analyst | analyst-methodology-check | JSON Schema | steer | Auto-corrected, then passes | diff --git a/examples/crewai/steering_financial_agent/README.md b/examples/crewai/steering_financial_agent/README.md index 54d7879c..2fabd7e0 100644 --- a/examples/crewai/steering_financial_agent/README.md +++ b/examples/crewai/steering_financial_agent/README.md @@ -1,6 +1,6 @@ # CrewAI Financial Agent with Steering Controls -Demonstrates all Agent Control action types in a realistic wire-transfer scenario using CrewAI. +Demonstrates Agent Control actions in a realistic wire-transfer scenario using CrewAI. ## Action Types @@ -8,15 +8,15 @@ Demonstrates all Agent Control action types in a realistic wire-transfer scenari |--------|----------|---------| | **DENY** | Hard block, agent cannot recover | Sanctioned country, fraud detected | | **STEER** | Pause execution, guide agent to correct and retry | 2FA required, manager approval needed | -| **WARN** | Log for audit, agent continues uninterrupted | New recipient, unusual activity | +| **OBSERVE** | Record for audit, agent continues uninterrupted | New recipient, unusual activity | -The key difference: **DENY** raises `ControlViolationError` (permanent), **STEER** raises `ControlSteerError` (recoverable), and **WARN** logs silently. +The key difference: **DENY** raises `ControlViolationError` (permanent), **STEER** raises `ControlSteerError` (recoverable), and **OBSERVE** records a non-blocking advisory event. ## Scenarios | # | Scenario | Amount | Controls Triggered | Outcome | |---|----------|--------|--------------------|---------| -| 1 | Small transfer to new vendor | $2,500 | warn (new recipient) | ALLOWED + audit log | +| 1 | Small transfer to new vendor | $2,500 | observe (new recipient) | ALLOWED + audit trail | | 2 | Transfer to North Korea | $500 | deny (sanctioned country) | BLOCKED | | 3 | Large transfer | $15,000 | steer (2FA required) | STEERED -> verified -> ALLOWED | | 4 | Very large transfer | $75,000 | steer (2FA + manager) | STEERED -> approved -> ALLOWED | @@ -28,8 +28,8 @@ The key difference: **DENY** raises `ControlViolationError` (permanent), **STEER - `deny-high-fraud-score` — JSON evaluator, blocks fraud_score > 0.8 - `steer-require-2fa` — JSON evaluator with oneOf schema, steers for 2FA - `steer-require-manager-approval` — JSON evaluator, steers for approval -- `warn-new-recipient` — LIST evaluator, logs unknown recipients -- `warn-pii-in-confirmation` — REGEX evaluator, logs PII in output +- `observe-new-recipient` — LIST evaluator, records unknown recipients +- `observe-pii-in-confirmation` — REGEX evaluator, records PII in output ## Prerequisites diff --git a/examples/crewai/steering_financial_agent/pyproject.toml b/examples/crewai/steering_financial_agent/pyproject.toml index f2bebb4a..87da3107 100644 --- a/examples/crewai/steering_financial_agent/pyproject.toml +++ b/examples/crewai/steering_financial_agent/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "steering-financial-agent" version = "0.1.0" -description = "CrewAI financial agent with steering, deny, and warn actions via Agent Control" +description = "CrewAI financial agent with steering, deny, and observe actions via Agent Control" requires-python = ">=3.12" dependencies = [ "agent-control-sdk>=6.3.0", diff --git a/examples/crewai/steering_financial_agent/setup_controls.py b/examples/crewai/steering_financial_agent/setup_controls.py index 77bf67e0..15126b24 100644 --- a/examples/crewai/steering_financial_agent/setup_controls.py +++ b/examples/crewai/steering_financial_agent/setup_controls.py @@ -1,10 +1,10 @@ """ Setup script for the CrewAI Financial Agent steering example. -Creates controls that demonstrate all three non-blocking action types: +Creates controls that demonstrate the three supported action types: - DENY: Hard-block sanctioned countries and fraud (no recovery) - STEER: Guide the agent through 2FA and manager approval workflows -- WARN: Flag new recipients and unusual hours for audit (no blocking) +- OBSERVE: Flag new recipients and unusual hours for audit (no blocking) Run once before running the agent: uv run --active python setup_controls.py @@ -222,15 +222,15 @@ async def setup(): }, ), # ┌─────────────────────────────────────────────────────────┐ - # │ WARN: New Recipient │ + # │ OBSERVE: New Recipient │ # │ Uses LIST evaluator with "not in" logic. │ - # │ Logs a warning when recipient is unknown — but does │ - # │ NOT block the transfer. Useful for audit trails. │ + # │ Records an advisory event when recipient is unknown │ + # │ but does NOT block the transfer. Useful for audit. │ # └─────────────────────────────────────────────────────────┘ ( - "warn-new-recipient", + "observe-new-recipient", { - "description": "Log warning for transfers to unknown recipients", + "description": "Observe transfers to unknown recipients", "enabled": True, "execution": "server", "scope": { @@ -251,18 +251,18 @@ async def setup(): "case_sensitive": False, }, }, - "action": {"decision": "warn"}, + "action": {"decision": "observe"}, }, ), # ┌─────────────────────────────────────────────────────────┐ - # │ WARN: PII in Output │ + # │ OBSERVE: PII in Output │ # │ Uses REGEX evaluator to detect leaked PII in the │ - # │ transfer confirmation. Logs for compliance review. │ + # │ transfer confirmation. Records for compliance review. │ # └─────────────────────────────────────────────────────────┘ ( - "warn-pii-in-confirmation", + "observe-pii-in-confirmation", { - "description": "Log warning if transfer confirmation contains PII patterns", + "description": "Observe PII patterns in transfer confirmation output", "enabled": True, "execution": "server", "scope": { @@ -277,7 +277,7 @@ async def setup(): "pattern": r"(?:\b\d{3}-\d{2}-\d{4}\b|\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b)" }, }, - "action": {"decision": "warn"}, + "action": {"decision": "observe"}, }, ), ] @@ -322,7 +322,7 @@ async def setup(): print(" - 2FA required for >= $10k (json evaluator)") print(" - Manager approval for >= $50k (json evaluator)") print() - print(" WARN (log for audit, no blocking):") + print(" OBSERVE (audit trail, no blocking):") print(" - New/unknown recipient (list evaluator)") print(" - PII in confirmation output (regex evaluator)") print() diff --git a/examples/crewai/steering_financial_agent/src/steering_financial_agent/main.py b/examples/crewai/steering_financial_agent/src/steering_financial_agent/main.py index c4dd06b9..5e779b49 100644 --- a/examples/crewai/steering_financial_agent/src/steering_financial_agent/main.py +++ b/examples/crewai/steering_financial_agent/src/steering_financial_agent/main.py @@ -1,5 +1,5 @@ """ -CrewAI Financial Agent with Steering, Deny, and Warn actions. +CrewAI Financial Agent with Steering, Deny, and Observe actions. Demonstrates the three key Agent Control action types in a realistic wire-transfer scenario using a CrewAI crew: @@ -7,8 +7,8 @@ DENY - Sanctioned country or fraud score blocks the transfer immediately. STEER - Large transfers pause execution and guide the agent through 2FA verification or manager approval before retrying. - WARN - New recipients and PII in output are logged for audit - without blocking the transfer. + OBSERVE - New recipients and PII in output are recorded for audit + without blocking the transfer. PREREQUISITE: Run setup_controls.py first: @@ -20,7 +20,7 @@ $ uv run --active steering_financial_agent Scenarios: - 1. Small legitimate transfer -> ALLOW (warn on new recipient) + 1. Small legitimate transfer -> OBSERVE (new recipient) 2. Sanctioned country -> DENY (hard block) 3. Large transfer ($15k) -> STEER (2FA required, then allowed) 4. Very large transfer ($75k) -> STEER (manager approval, then allowed) @@ -67,7 +67,7 @@ def verify_server(): "deny-high-fraud-score", "steer-require-2fa", "steer-require-manager-approval", - "warn-new-recipient", + "observe-new-recipient", ] missing = [n for n in required if n not in names] if missing: @@ -104,7 +104,7 @@ def run_scenario(crew, number, title, request, expected): def main(): print("=" * 60) print(" CrewAI Financial Agent") - print(" Steering, Deny & Warn with Agent Control") + print(" Steering, Deny & Observe with Agent Control") print("=" * 60) print() @@ -118,19 +118,19 @@ def main(): crew = create_financial_crew() # ── Scenario 1: Small Legitimate Transfer ─────────────────────── - # Amount < $10k, known country, low fraud → ALLOW - # Unknown recipient → WARN (logged, not blocked) + # Amount < $10k, known country, low fraud → OBSERVE only + # Unknown recipient → OBSERVE (recorded, not blocked) run_scenario( crew, 1, - "Small Legitimate Transfer (ALLOW + WARN)", + "Small Legitimate Transfer (OBSERVE)", { "amount": 2500, "recipient": "New Vendor XYZ", "destination_country": "Germany", "fraud_score": 0.1, }, - "ALLOWED with warning (new recipient logged for audit)", + "Allowed with an observe event (new recipient recorded for audit)", ) # ── Scenario 2: Sanctioned Country ────────────────────────────── @@ -149,26 +149,26 @@ def main(): ) # ── Scenario 3: Large Transfer Requiring 2FA ──────────────────── - # Amount $15k → STEER (2FA required), agent verifies, retries → ALLOW + # Amount $15k → STEER (2FA required), agent verifies, retries successfully run_scenario( crew, 3, - "Large Transfer - 2FA Steering (STEER then ALLOW)", + "Large Transfer - 2FA Steering (STEER then success)", { "amount": 15000, "recipient": "Acme Corp", "destination_country": "United Kingdom", "fraud_score": 0.2, }, - "STEERED (2FA), then ALLOWED after verification", + "STEERED (2FA), then succeeded after verification", ) # ── Scenario 4: Very Large Transfer Requiring Manager Approval ── - # Amount $75k → STEER (2FA + manager approval), agent handles both → ALLOW + # Amount $75k → STEER (2FA + manager approval), agent handles both successfully run_scenario( crew, 4, - "Very Large Transfer - Manager Approval (STEER then ALLOW)", + "Very Large Transfer - Manager Approval (STEER then success)", { "amount": 75000, "recipient": "Global Suppliers Inc", @@ -206,13 +206,13 @@ def main(): STEER 2FA verification (Scenario 3) - pause, verify, retry Manager approval (Scenario 4) - pause, collect + approve, retry - WARN New recipient (Scenario 1) - logged for audit, not blocked - PII in output (if triggered) - logged for compliance, not blocked + OBSERVE New recipient (Scenario 1) - recorded for audit, not blocked + PII in output (if triggered) - recorded for compliance, not blocked Key Differences: DENY = ControlViolationError (agent cannot recover) STEER = ControlSteerError (agent corrects and retries) - WARN = Logged silently (agent continues uninterrupted) + OBSERVE = Recorded silently (agent continues uninterrupted) """) diff --git a/examples/customer_support_agent/demo.sh b/examples/customer_support_agent/demo.sh index f7080f3d..a642489e 100755 --- a/examples/customer_support_agent/demo.sh +++ b/examples/customer_support_agent/demo.sh @@ -247,7 +247,7 @@ show_observability_stats() { echo "Last 5 minutes:" echo "---------------" local stats_5m=$(curl -s "${server_url}/api/v1/observability/stats?agent_name=${agent_name}&time_range=5m") - if echo "$stats_5m" | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Total executions: {d.get('total_executions', 0)}\"); print(f\" Matches: {d.get('total_matches', 0)}\"); print(f\" Non-matches: {d.get('total_non_matches', 0)}\"); print(f\" Errors: {d.get('total_errors', 0)}\"); actions=d.get('action_counts', {}); print(f\" Actions: allow={actions.get('allow', 0)}, deny={actions.get('deny', 0)}, warn={actions.get('warn', 0)}, log={actions.get('log', 0)}\")" 2>/dev/null; then + if echo "$stats_5m" | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Total executions: {d.get('total_executions', 0)}\"); print(f\" Matches: {d.get('total_matches', 0)}\"); print(f\" Non-matches: {d.get('total_non_matches', 0)}\"); print(f\" Errors: {d.get('total_errors', 0)}\"); actions=d.get('action_counts', {}); print(f\" Actions: observe={actions.get('observe', 0)}, deny={actions.get('deny', 0)}, steer={actions.get('steer', 0)}\")" 2>/dev/null; then : else echo " No data or error fetching stats" @@ -259,7 +259,7 @@ show_observability_stats() { echo "Last 1 hour:" echo "------------" local stats_1h=$(curl -s "${server_url}/api/v1/observability/stats?agent_name=${agent_name}&time_range=1h") - if echo "$stats_1h" | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Total executions: {d.get('total_executions', 0)}\"); print(f\" Matches: {d.get('total_matches', 0)}\"); print(f\" Non-matches: {d.get('total_non_matches', 0)}\"); print(f\" Errors: {d.get('total_errors', 0)}\"); actions=d.get('action_counts', {}); print(f\" Actions: allow={actions.get('allow', 0)}, deny={actions.get('deny', 0)}, warn={actions.get('warn', 0)}, log={actions.get('log', 0)}\")" 2>/dev/null; then + if echo "$stats_1h" | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Total executions: {d.get('total_executions', 0)}\"); print(f\" Matches: {d.get('total_matches', 0)}\"); print(f\" Non-matches: {d.get('total_non_matches', 0)}\"); print(f\" Errors: {d.get('total_errors', 0)}\"); actions=d.get('action_counts', {}); print(f\" Actions: observe={actions.get('observe', 0)}, deny={actions.get('deny', 0)}, steer={actions.get('steer', 0)}\")" 2>/dev/null; then : else echo " No data or error fetching stats" diff --git a/examples/customer_support_agent/run_demo.py b/examples/customer_support_agent/run_demo.py index 5c001a72..477fbb95 100644 --- a/examples/customer_support_agent/run_demo.py +++ b/examples/customer_support_agent/run_demo.py @@ -494,9 +494,9 @@ async def run_tool_control_tests(agent: CustomerSupportAgent): print(f" Result: {response}") print() - # Test 5: High priority ticket (warn-high-priority-ticket) + # Test 5: High priority ticket (observe-high-priority-ticket) print("Test 5: High priority ticket creation") - print(" Control: warn-high-priority-ticket (scope.step_name_regex + selector path: input.priority)") + print(" Control: observe-high-priority-ticket (scope.step_name_regex + selector path: input.priority)") print(" Priority: critical") response = await agent.create_support_ticket( subject="Urgent issue", @@ -506,8 +506,8 @@ async def run_tool_control_tests(agent: CustomerSupportAgent): print(f" Result: {response}") print() - # Test 6: Low priority ticket (should not trigger warn) - print("Test 6: Low priority ticket creation (should not warn)") + # Test 6: Low priority ticket (should not trigger observe) + print("Test 6: Low priority ticket creation (should not observe)") print(" Priority: low") response = await agent.create_support_ticket( subject="Question", @@ -519,7 +519,7 @@ async def run_tool_control_tests(agent: CustomerSupportAgent): # Test 7: PII in ticket description print("Test 7: Email in ticket description") - print(" Control: block-pii-in-ticket-description (scope.step_names + selector path: input.description)") + print(" Control: observe-pii-in-ticket-description (scope.step_names + selector path: input.description)") response = await agent.create_support_ticket( subject="Contact request", description="Please email me at secret@company.com", diff --git a/examples/customer_support_agent/setup_demo_controls.py b/examples/customer_support_agent/setup_demo_controls.py index 7e8c0538..efa4d255 100644 --- a/examples/customer_support_agent/setup_demo_controls.py +++ b/examples/customer_support_agent/setup_demo_controls.py @@ -120,10 +120,10 @@ }, }, { - "name": "log-ticket-creation", - "description": "Logs all ticket creation for audit (scope.step_names: exact match)", + "name": "observe-ticket-creation", + "description": "Observes all ticket creation for audit (scope.step_names: exact match)", "definition": { - "description": "Logs all ticket creation for audit (scope.step_names: exact match)", + "description": "Observes all ticket creation for audit (scope.step_names: exact match)", "enabled": True, "execution": "server", "scope": { @@ -133,14 +133,14 @@ }, "condition": { "selector": { - "path": "*", # Log entire payload + "path": "*", # Observe entire payload }, "evaluator": { "name": "regex", "config": {"pattern": r".*"}, # Always matches }, }, - "action": {"decision": "log"}, + "action": {"decision": "observe"}, }, }, # ========================================================================== @@ -174,10 +174,10 @@ }, }, { - "name": "warn-high-priority-ticket", - "description": "Warns on high priority tickets (path: arguments.priority)", + "name": "observe-high-priority-ticket", + "description": "Observes high priority tickets (path: arguments.priority)", "definition": { - "description": "Warns on high priority tickets (path: input.priority)", + "description": "Observes high priority tickets (path: input.priority)", "enabled": True, "execution": "server", "scope": { @@ -200,17 +200,17 @@ }, }, }, - "action": {"decision": "warn"}, + "action": {"decision": "observe"}, }, }, # ========================================================================== # TOOL CALL CONTROLS - using 'path' with nested arguments # ========================================================================== { - "name": "block-pii-in-ticket-description", - "description": "Blocks PII in ticket descriptions (path: arguments.description)", + "name": "observe-pii-in-ticket-description", + "description": "Observes PII in ticket descriptions (path: arguments.description)", "definition": { - "description": "Blocks PII in ticket descriptions (path: input.description)", + "description": "Observes PII in ticket descriptions (path: input.description)", "enabled": True, "execution": "server", "scope": { @@ -230,7 +230,7 @@ }, }, }, - "action": {"decision": "warn"}, + "action": {"decision": "observe"}, }, }, ] diff --git a/examples/deepeval/setup_controls.py b/examples/deepeval/setup_controls.py index faddb15e..d4959c77 100755 --- a/examples/deepeval/setup_controls.py +++ b/examples/deepeval/setup_controls.py @@ -133,7 +133,7 @@ }, }, }, - "action": {"decision": "warn"}, + "action": {"decision": "observe"}, }, }, ] diff --git a/examples/google_adk_callbacks/my_agent/agent.py b/examples/google_adk_callbacks/my_agent/agent.py index 17575065..e8eb65ad 100644 --- a/examples/google_adk_callbacks/my_agent/agent.py +++ b/examples/google_adk_callbacks/my_agent/agent.py @@ -218,10 +218,8 @@ def _handle_result(result: Any) -> None: ) for match in matches: - if match.action == "warn": - print(f"Agent Control warning [{match.control_name}]: {match.result.message}") - elif match.action == "log": - print(f"Agent Control log [{match.control_name}]: {match.result.message}") + if match.action == "observe": + print(f"Agent Control observe [{match.control_name}]: {match.result.message}") def _resolve_agent_step_name(callback_context: CallbackContext) -> str: diff --git a/examples/steer_action_demo/README.md b/examples/steer_action_demo/README.md index 5d9a415c..b386784e 100644 --- a/examples/steer_action_demo/README.md +++ b/examples/steer_action_demo/README.md @@ -1,9 +1,10 @@ # Steer Action Demo -Banking transfer agent showcasing allow, deny, warn, and steer actions. +Banking transfer agent showcasing observe, deny, and steer actions. ## What this example shows +- Observe actions for non-blocking audit trails - Steer actions with corrective guidance - Multi-step approval workflows - Deterministic retry behavior diff --git a/examples/steer_action_demo/autonomous_agent_demo.py b/examples/steer_action_demo/autonomous_agent_demo.py index 6df15091..0b3d488d 100644 --- a/examples/steer_action_demo/autonomous_agent_demo.py +++ b/examples/steer_action_demo/autonomous_agent_demo.py @@ -9,7 +9,7 @@ This agent processes wire transfers while AgentControl enforces: - DENY actions for hard compliance violations (OFAC sanctions, high fraud) - STEER actions for approval workflows (2FA verification, manager approval) -- ALLOW actions for simple, low-risk transfers +- OBSERVE actions for simple, low-risk transfer audit trails ## The Experience @@ -353,14 +353,14 @@ async def process_transfer_node(state: AgentState) -> AgentState: print(f" ✓ ${result['amount']:,.2f} sent to {result['recipient']}") log_trace("audit", f"Transaction logged: ID={result['transaction_id']}, Amount=${result['amount']:,.2f}") - # Note: Warn actions (if any) are logged but don't block execution - # Check if this was a new recipient (which triggers warn control) + # Note: Observe actions (if any) are recorded but don't block execution + # Check if this was a new recipient (which triggers observe control) if request["recipient_name"] not in ["John Smith", "Acme Corp", "Global Suppliers Inc"]: - print(f"\n ⚠️ \033[33mWARN: New recipient detected\033[0m") - print(f" Control: warn-new-recipient") + print(f"\n ℹ️ \033[34mOBSERVE: New recipient detected\033[0m") + print(f" Control: observe-new-recipient") print(f" Note: Transfer to '{request['recipient_name']}' is not in known recipient list") - print(f" Action: Logged for review (non-blocking)") - log_trace("warn", f"New recipient logged for review: {request['recipient_name']}") + print(f" Action: Recorded for review (non-blocking)") + log_trace("observe", f"New recipient recorded for review: {request['recipient_name']}") return { **state, diff --git a/examples/steer_action_demo/setup_controls.py b/examples/steer_action_demo/setup_controls.py index af13c80f..7f1efb62 100644 --- a/examples/steer_action_demo/setup_controls.py +++ b/examples/steer_action_demo/setup_controls.py @@ -1,8 +1,8 @@ """ Setup script for Banking Transaction Agent controls. -Demonstrates all AgentControl action types in a realistic banking scenario: -- ALLOW: Simple transfers that satisfy controls +Demonstrates the Agent Control action types in a realistic banking scenario: +- OBSERVE: Simple transfers that trigger non-blocking audit controls - DENY: Compliance violations (sanctioned countries, fraud) - STEER: Large transfers requiring verification and approval @@ -19,7 +19,7 @@ async def setup_banking_controls(): - """Create banking controls demonstrating allow, deny, and steer actions.""" + """Create banking controls demonstrating observe, deny, and steer actions.""" async with AgentControlClient(base_url=SERVER_URL) as client: # 1. Register Agent agent = Agent( @@ -100,12 +100,12 @@ async def setup_banking_controls(): } # ============================================================================ - # WARN CONTROLS - Log suspicious activity without blocking + # OBSERVE CONTROLS - Record suspicious activity without blocking # ============================================================================ - # Control 3: New Recipient Warning (WARN) + # Control 3: New Recipient Audit (OBSERVE) new_recipient_control = { - "description": "Log warning when transferring to a new recipient", + "description": "Observe transfers to new recipients for audit review", "enabled": True, "execution": "server", "scope": { @@ -129,7 +129,7 @@ async def setup_banking_controls(): }, }, "action": { - "decision": "warn" + "decision": "observe" } } @@ -211,7 +211,7 @@ async def setup_banking_controls(): control_configs = [ ("deny-sanctioned-countries", sanctioned_countries_control), ("deny-high-fraud-risk", fraud_risk_control), - ("warn-new-recipient", new_recipient_control), + ("observe-new-recipient", new_recipient_control), ("steer-large-transfer-verification", large_transfer_control), ("steer-manager-approval-required", manager_approval_control), ] @@ -269,6 +269,8 @@ async def setup_banking_controls(): print(" DENY:") print(" • Sanctioned countries (OFAC compliance)") print(" • High fraud risk (score > 0.8)") + print(" OBSERVE:") + print(" • New recipient audit trail (non-blocking)") print(" STEER:") print(" • Large transfer 2FA verification (>$10k)") print(" • Manager approval for over-limit transfers") diff --git a/models/src/agent_control_models/__init__.py b/models/src/agent_control_models/__init__.py index 8ef92d97..45a455d8 100644 --- a/models/src/agent_control_models/__init__.py +++ b/models/src/agent_control_models/__init__.py @@ -7,6 +7,12 @@ except PackageNotFoundError: __version__ = "0.0.0.dev" +from .actions import ( + ActionDecision, + expand_action_filter, + normalize_action, + normalize_action_list, +) from .agent import ( BUILTIN_STEP_TYPES, STEP_TYPE_LLM, @@ -91,6 +97,7 @@ "STEP_TYPE_TOOL", "STEP_TYPE_LLM", "BUILTIN_STEP_TYPES", + "ActionDecision", # Policy "Policy", # Evaluation @@ -107,6 +114,9 @@ "EvaluatorSpec", "EvaluatorResult", "SteeringContext", + "normalize_action", + "normalize_action_list", + "expand_action_filter", # Error models "ProblemDetail", "ErrorCode", diff --git a/models/src/agent_control_models/actions.py b/models/src/agent_control_models/actions.py new file mode 100644 index 00000000..a890c6f9 --- /dev/null +++ b/models/src/agent_control_models/actions.py @@ -0,0 +1,53 @@ +"""Shared control-action types and normalization helpers.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Literal, cast + +type ActionDecision = Literal["deny", "steer", "observe"] + +_OBSERVE_ACTION_ALIASES = frozenset({"allow", "observe", "warn", "log"}) +_ACTION_QUERY_EXPANSION: dict[ActionDecision, tuple[str, ...]] = { + "deny": ("deny",), + "steer": ("steer",), + "observe": ("observe", "allow", "warn", "log"), +} + + +def normalize_action(action: str) -> ActionDecision: + """Normalize a public or legacy action name to the canonical action.""" + if action in _OBSERVE_ACTION_ALIASES: + return "observe" + if action in ("deny", "steer"): + return cast(ActionDecision, action) + raise ValueError( + "Invalid action. Expected one of: deny, steer, observe " + "(legacy aliases allow/warn/log are also accepted temporarily)." + ) + + +def normalize_action_list(actions: Sequence[str]) -> list[ActionDecision]: + """Normalize a list of actions while preserving order and removing duplicates.""" + normalized: list[ActionDecision] = [] + seen: set[ActionDecision] = set() + for action in actions: + canonical = normalize_action(action) + if canonical in seen: + continue + seen.add(canonical) + normalized.append(canonical) + return normalized + + +def expand_action_filter(actions: Sequence[ActionDecision]) -> list[str]: + """Expand canonical action filters to include legacy stored event values.""" + expanded: list[str] = [] + seen: set[str] = set() + for action in actions: + for candidate in _ACTION_QUERY_EXPANSION[action]: + if candidate in seen: + continue + seen.add(candidate) + expanded.append(candidate) + return expanded diff --git a/models/src/agent_control_models/controls.py b/models/src/agent_control_models/controls.py index a043622b..0ed11b1b 100644 --- a/models/src/agent_control_models/controls.py +++ b/models/src/agent_control_models/controls.py @@ -10,6 +10,7 @@ import re2 from pydantic import ConfigDict, Field, ValidationInfo, field_validator, model_validator +from .actions import ActionDecision, normalize_action from .base import BaseModel @@ -265,7 +266,7 @@ class SteeringContext(BaseModel): class ControlAction(BaseModel): """What to do when control matches.""" - decision: Literal["allow", "deny", "steer", "warn", "log"] = Field( + decision: ActionDecision = Field( ..., description="Action to take when control is triggered" ) steering_context: SteeringContext | None = Field( @@ -277,6 +278,11 @@ class ControlAction(BaseModel): ) ) + @field_validator("decision", mode="before") + @classmethod + def normalize_decision(cls, value: str) -> ActionDecision: + return normalize_action(value) + MAX_CONDITION_DEPTH = 6 @@ -649,7 +655,7 @@ class ControlMatch(BaseModel): ) control_id: int = Field(..., description="Database ID of the control") control_name: str = Field(..., description="Name of the control") - action: Literal["allow", "deny", "steer", "warn", "log"] = Field( + action: ActionDecision = Field( ..., description="Action configured for this control" ) result: EvaluatorResult = Field( @@ -659,3 +665,8 @@ class ControlMatch(BaseModel): None, description="Steering context for steer actions if configured" ) + + @field_validator("action", mode="before") + @classmethod + def normalize_action_value(cls, value: str) -> ActionDecision: + return normalize_action(value) diff --git a/models/src/agent_control_models/observability.py b/models/src/agent_control_models/observability.py index 5735bd07..a78e4b1e 100644 --- a/models/src/agent_control_models/observability.py +++ b/models/src/agent_control_models/observability.py @@ -14,6 +14,7 @@ from pydantic import Field, field_validator +from .actions import ActionDecision, normalize_action, normalize_action_list from .agent import AGENT_NAME_MIN_LENGTH, AGENT_NAME_PATTERN, normalize_agent_name from .base import BaseModel @@ -42,7 +43,7 @@ class ControlExecutionEvent(BaseModel): control_name: Name of the control (denormalized for queries) check_stage: "pre" (before execution) or "post" (after execution) applies_to: "llm_call" or "tool_call" - action: The action taken (allow, deny, warn, log) + action: The action taken (deny, steer, observe) matched: Whether the control evaluator matched confidence: Confidence score from the evaluator (0.0-1.0) timestamp: When the control was executed (UTC) @@ -90,7 +91,7 @@ class ControlExecutionEvent(BaseModel): ) # Result - action: Literal["allow", "deny", "steer", "warn", "log"] = Field( + action: ActionDecision = Field( ..., description="Action taken by the control" ) matched: bool = Field( @@ -160,6 +161,11 @@ def validate_span_id(cls, v: str) -> str: def validate_and_normalize_agent_name(cls, value: str) -> str: return normalize_agent_name(str(value)) + @field_validator("action", mode="before") + @classmethod + def normalize_event_action(cls, value: str) -> ActionDecision: + return normalize_action(value) + model_config = { "json_schema_extra": { "examples": [ @@ -265,7 +271,7 @@ class EventQueryRequest(BaseModel): control_execution_id: Filter by specific event ID agent_name: Filter by agent identifier control_ids: Filter by control IDs - actions: Filter by actions (allow, deny, steer, warn, log) + actions: Filter by actions (deny, steer, observe) matched: Filter by matched status check_stages: Filter by check stages (pre, post) applies_to: Filter by call type (llm_call, tool_call) @@ -293,7 +299,7 @@ class EventQueryRequest(BaseModel): control_ids: list[int] | None = Field( default=None, description="Filter by control IDs" ) - actions: list[Literal["allow", "deny", "steer", "warn", "log"]] | None = Field( + actions: list[ActionDecision] | None = Field( default=None, description="Filter by actions" ) matched: bool | None = Field(default=None, description="Filter by matched status") @@ -318,7 +324,7 @@ class EventQueryRequest(BaseModel): {"trace_id": "4bf92f3577b34da6a3ce929d0e0e4736"}, { "agent_name": "my-agent", - "actions": ["deny", "warn"], + "actions": ["deny", "observe"], "start_time": "2025-01-09T00:00:00Z", "limit": 50, }, @@ -335,6 +341,15 @@ def validate_and_normalize_agent_name( return None return normalize_agent_name(str(value)) + @field_validator("actions", mode="before") + @classmethod + def normalize_actions_filter( + cls, value: list[str] | None + ) -> list[ActionDecision] | None: + if value is None: + return None + return normalize_action_list(value) + class EventQueryResponse(BaseModel): """ @@ -368,14 +383,15 @@ class ControlStats(BaseModel): execution_count: Total number of executions match_count: Number of times the control matched non_match_count: Number of times the control did not match - allow_count: Number of allow actions deny_count: Number of deny actions steer_count: Number of steer actions - warn_count: Number of warn actions - log_count: Number of log actions + observe_count: Number of observe actions error_count: Number of errors during evaluation avg_confidence: Average confidence score avg_duration_ms: Average execution duration in milliseconds + + Invariant: + deny_count + steer_count + observe_count == match_count """ control_id: int = Field(..., description="Control ID") @@ -383,11 +399,9 @@ class ControlStats(BaseModel): execution_count: int = Field(..., ge=0, description="Total executions") match_count: int = Field(..., ge=0, description="Total matches") non_match_count: int = Field(..., ge=0, description="Total non-matches") - allow_count: int = Field(..., ge=0, description="Allow actions") deny_count: int = Field(..., ge=0, description="Deny actions") steer_count: int = Field(..., ge=0, description="Steer actions") - warn_count: int = Field(..., ge=0, description="Warn actions") - log_count: int = Field(..., ge=0, description="Log actions") + observe_count: int = Field(..., ge=0, description="Observe actions") error_count: int = Field(..., ge=0, description="Evaluation errors") avg_confidence: float = Field(..., ge=0.0, le=1.0, description="Average confidence") avg_duration_ms: float | None = Field( @@ -460,7 +474,7 @@ def ensure_timezone_aware(cls, v: datetime) -> datetime: error_count: int = Field(..., ge=0, description="Errors in bucket") action_counts: dict[str, int] = Field( default_factory=dict, - description="Action breakdown: {allow, deny, steer, warn, log}", + description="Action breakdown: {deny, steer, observe}", ) avg_confidence: float | None = Field( default=None, ge=0.0, le=1.0, description="Average confidence score" @@ -476,7 +490,7 @@ class StatsTotals(BaseModel): Invariant: execution_count = match_count + non_match_count + error_count - Matches have actions (allow, deny, steer, warn, log) tracked in action_counts. + Matches have actions (deny, steer, observe) tracked in action_counts. sum(action_counts.values()) == match_count Attributes: @@ -494,7 +508,7 @@ class StatsTotals(BaseModel): error_count: int = Field(default=0, ge=0, description="Total errors") action_counts: dict[str, int] = Field( default_factory=dict, - description="Action breakdown for matches: {allow, deny, steer, warn, log}", + description="Action breakdown for matches: {deny, steer, observe}", ) timeseries: list[TimeseriesBucket] | None = Field( default=None, diff --git a/models/tests/test_actions.py b/models/tests/test_actions.py new file mode 100644 index 00000000..c9a8c0cf --- /dev/null +++ b/models/tests/test_actions.py @@ -0,0 +1,60 @@ +"""Tests for shared control-action compatibility behavior.""" + +from __future__ import annotations + +import pytest +from agent_control_models import ( + ControlAction, + ControlExecutionEvent, + ControlMatch, + EventQueryRequest, + EvaluatorResult, + expand_action_filter, +) +from pydantic import ValidationError + + +def test_event_query_actions_normalize_and_expand_for_legacy_observability() -> None: + # Given: a query that mixes canonical and legacy advisory action names + query = EventQueryRequest( + actions=["warn", "observe", "deny", "log", "deny", "steer", "allow", "steer"] + ) + + # When: expanding the normalized public action filter for stored event rows + expanded = expand_action_filter(query.actions or []) + + # Then: the public filter is canonicalized, deduped, and expanded for legacy rows + assert query.actions == ["observe", "deny", "steer"] + assert expanded == ["observe", "allow", "warn", "log", "deny", "steer"] + + +def test_invalid_action_is_rejected_across_public_model_boundaries() -> None: + # Given: the same invalid action at each public model boundary + invalid_action = "block" + invalid_builders = [ + lambda: ControlAction.model_validate({"decision": invalid_action}), + lambda: ControlMatch( + control_id=123, + control_name="pii-check", + action=invalid_action, + result=EvaluatorResult(matched=True, confidence=0.9), + ), + lambda: ControlExecutionEvent( + trace_id="trace-123", + span_id="span-123", + agent_name="test-agent", + control_id=123, + control_name="pii-check", + check_stage="pre", + applies_to="llm_call", + action=invalid_action, + matched=True, + confidence=0.9, + ), + lambda: EventQueryRequest(actions=[invalid_action]), + ] + + for build_invalid_model in invalid_builders: + # When / Then: validation fails before the invalid action can enter the system + with pytest.raises(ValidationError, match="Invalid action"): + build_invalid_model() diff --git a/sdks/python/src/agent_control/control_decorators.py b/sdks/python/src/agent_control/control_decorators.py index d1edb06e..f886793d 100644 --- a/sdks/python/src/agent_control/control_decorators.py +++ b/sdks/python/src/agent_control/control_decorators.py @@ -24,7 +24,7 @@ async def chat(message: str) -> str: # - stage: "pre" or "post" # - selector.path: "input" or "output" # - evaluator: regex, list, Luna2 evaluator, etc. - # - action: deny, warn, or log + # - action: deny, steer, or observe """ import asyncio @@ -35,7 +35,7 @@ async def chat(message: str) -> str: from dataclasses import dataclass, field from typing import Any, TypeVar -from agent_control_models import Step +from agent_control_models import Step, normalize_action from agent_control import AgentControlClient from agent_control.evaluation import check_evaluation_with_local @@ -125,7 +125,7 @@ def process_result(self, result: dict[str, Any], check_stage: str) -> None: # Log each control evaluation _log_control_evaluations(result, self.trace_id, self.span_id, check_stage) - # Handle deny/steer/warn/log actions (may raise ControlViolationError, ControlSteerError) + # Handle deny/steer/observe actions (may raise ControlViolationError, ControlSteerError) _handle_evaluation_result(result) # Update stats in place @@ -144,7 +144,7 @@ def _update_stats(self, result: dict[str, Any]) -> None: for match in result.get("matches") or []: self.total_executions += 1 self.total_matches += 1 - action = match.get("action", "allow") + action = normalize_action(str(match.get("action", "observe"))) self.actions[action] = self.actions.get(action, 0) + 1 # Process non-matches (controls evaluated but didn't match) @@ -521,8 +521,8 @@ def _handle_evaluation_result(result: dict[str, Any]) -> None: f"Errors: {'; '.join(error_messages)}" ) - # CRITICAL: Handle actions in priority order: deny > steer > warn > log - # Check blocking actions first (deny/steer), then always log warn/log + # CRITICAL: Handle actions in priority order: deny > steer > observe + # Check blocking actions first (deny/steer), then log observe matches. if not is_safe: # Pass 1: Check for deny actions (highest priority - blocks immediately) @@ -577,13 +577,13 @@ def _handle_evaluation_result(result: dict[str, Any]) -> None: steering_context=steering_context ) - # Log warn and log actions (non-blocking, always processed) + # Log observe actions (non-blocking, always processed) for match in matches: if not isinstance(match, dict): continue - action = match.get("action", "deny") - if action not in ("warn", "log"): + action = normalize_action(str(match.get("action", "observe"))) + if action != "observe": continue result_data = match.get("result") or {} @@ -594,10 +594,7 @@ def _handle_evaluation_result(result: dict[str, Any]) -> None: ) control_name = match.get("control_name", "unknown") - if action == "warn": - logger.warning(f"⚠️ Control [{control_name}]: {message}") - else: # action == "log" - logger.info(f"ℹ️ Control [{control_name}]: {message}") + logger.info(f"Control observe [{control_name}]: {message}") def _log_control_evaluations( @@ -633,7 +630,7 @@ def _log_single_control( span_id=span_id, control_name=control_data.get("control_name", "unknown"), matched=matched, - action=control_data.get("action", "allow"), + action=normalize_action(str(control_data.get("action", "observe"))), confidence=control_data.get("result", {}).get("confidence", 0.0), duration_ms=control_data.get("result", {}).get("execution_duration_ms"), control_execution_id=control_data.get("control_execution_id"), diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index d76af177..eb8ff349 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -354,7 +354,7 @@ async def check_evaluation_with_local( ControlMatch( control_id=control_id, control_name=control_name, - action="log", + action="observe", result=EvaluatorResult( matched=False, confidence=0.0, diff --git a/sdks/python/src/agent_control/integrations/strands/plugin.py b/sdks/python/src/agent_control/integrations/strands/plugin.py index 5b3b9c79..1aa503cd 100644 --- a/sdks/python/src/agent_control/integrations/strands/plugin.py +++ b/sdks/python/src/agent_control/integrations/strands/plugin.py @@ -250,7 +250,7 @@ async def check_before_tool(self, event: BeforeToolCallEvent) -> None: async def check_after_tool(self, event: AfterToolCallEvent) -> None: tool_name, tool_output = self._extract_tool_data(event) - tool_input = event.tool_use.get("input", {}) if event.tool_use else {} + tool_input: Any = event.tool_use.get("input", {}) if event.tool_use else {} context = self._extract_context(event) await self._evaluate_and_enforce( step_name=tool_name, @@ -357,6 +357,7 @@ def _extract_tool_data( else: tool_name = event.tool_use.get("name", "unknown-tool") + tool_data: Any if isinstance(event, BeforeToolCallEvent): tool_data = event.tool_use.get("input", {}) else: diff --git a/sdks/python/src/agent_control/observability.py b/sdks/python/src/agent_control/observability.py index 26a7d553..96d26813 100644 --- a/sdks/python/src/agent_control/observability.py +++ b/sdks/python/src/agent_control/observability.py @@ -928,7 +928,7 @@ def log_span_end( matches: Number of controls that matched non_matches: Number of controls that didn't match errors: Number of errors during evaluation - actions: Dict of action counts (e.g., {"deny": 2, "warn": 1}) + actions: Dict of action counts (e.g., {"deny": 2, "observe": 1}) """ if not _should_log("span_end"): return @@ -965,7 +965,7 @@ def log_control_evaluation( span_id: Span ID (full 16-char hex) control_name: Name of the control that was evaluated matched: Whether the control matched - action: Action taken (allow, deny, warn, log) + action: Action taken (deny, steer, observe) confidence: Confidence score (0.0-1.0) duration_ms: Evaluation duration in milliseconds control_execution_id: Unique ID for this control execution diff --git a/sdks/python/tests/test_control_decorators.py b/sdks/python/tests/test_control_decorators.py index d13ec46d..e7972348 100644 --- a/sdks/python/tests/test_control_decorators.py +++ b/sdks/python/tests/test_control_decorators.py @@ -52,16 +52,16 @@ def mock_unsafe_response(): @pytest.fixture -def mock_warn_response(): - """Response when evaluation triggers warning.""" +def mock_observe_response(): + """Response when evaluation triggers a non-blocking observe action.""" return { "is_safe": False, "confidence": 0.7, "matches": [ { "control_id": 1, - "control_name": "warn-control", - "action": "warn", + "control_name": "observe-control", + "action": "observe", "result": { "matched": True, "message": "Potential issue detected" @@ -132,10 +132,10 @@ async def chat(message: str) -> str: assert "toxicity" in exc_info.value.message.lower() @pytest.mark.asyncio - async def test_warns_without_blocking(self, mock_agent, mock_warn_response, caplog): - """Test that warn action logs but allows execution.""" + async def test_observes_without_blocking(self, mock_agent, mock_observe_response, caplog): + """Test that observe action logs but allows execution.""" with patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), \ - patch("agent_control.control_decorators._evaluate", return_value=mock_warn_response): + patch("agent_control.control_decorators._evaluate", return_value=mock_observe_response): @control() async def chat(message: str) -> str: @@ -144,6 +144,39 @@ async def chat(message: str) -> str: result = await chat("Questionable message") assert result == "Response to: Questionable message" + @pytest.mark.asyncio + async def test_legacy_warn_action_observes_without_blocking(self, mock_agent): + """Legacy advisory actions from older servers should be treated as observe.""" + legacy_observe_response = { + "is_safe": False, + "confidence": 0.7, + "matches": [ + { + "control_id": 1, + "control_name": "legacy-observe-control", + "action": "warn", + "result": { + "matched": True, + "message": "Potential issue detected" + } + } + ] + } + with patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), \ + patch("agent_control.control_decorators._evaluate", return_value=legacy_observe_response), \ + patch("agent_control.control_decorators.logger") as mock_logger: + + @control() + async def chat(message: str) -> str: + return f"Response to: {message}" + + result = await chat("Questionable legacy message") + + assert result == "Response to: Questionable legacy message" + mock_logger.info.assert_any_call( + "Control observe [legacy-observe-control]: Potential issue detected" + ) + @pytest.mark.asyncio async def test_no_agent_runs_without_protection(self): """Test that decorator passes through if no agent initialized.""" diff --git a/sdks/python/tests/test_integration_agents.py b/sdks/python/tests/test_integration_agents.py index 04ee5f86..271294fa 100644 --- a/sdks/python/tests/test_integration_agents.py +++ b/sdks/python/tests/test_integration_agents.py @@ -221,7 +221,7 @@ async def test_convenience_agent_association_functions( "config": {"pattern": ".*"}, }, }, - "action": {"decision": "allow"}, + "action": {"decision": "observe"}, "tags": ["test"], } diff --git a/sdks/python/tests/test_integrations_core.py b/sdks/python/tests/test_integrations_core.py index 35c4c567..03146ea7 100644 --- a/sdks/python/tests/test_integrations_core.py +++ b/sdks/python/tests/test_integrations_core.py @@ -137,7 +137,7 @@ async def test_evaluate_and_enforce_raises_steer(): async def test_evaluate_and_enforce_fallback_violation_includes_control_id(): result = MagicMock(spec=EvaluationResult) result.is_safe = False - result.matches = [_match(action="warn", control_id=42, message="unsafe")] + result.matches = [_match(action="observe", control_id=42, message="unsafe")] result.errors = [] result.reason = None diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 2a012bab..cb610084 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -601,8 +601,8 @@ async def test_tool_step_local_evaluation(self, agent_name, tool_payload): async def test_mixed_controls_merged_results(self, agent_name, llm_payload): """Results from local and server should be merged.""" controls = [ - # Local control (action=log, will match but not deny) - make_control_dict(1, "local_log", execution="sdk", pattern=r"test", action="log"), + # Local control (action=observe, will match but not deny) + make_control_dict(1, "local_log", execution="sdk", pattern=r"test", action="observe"), # Server control make_control_dict(2, "server_ctrl", execution="server"), ] @@ -611,7 +611,7 @@ async def test_mixed_controls_merged_results(self, agent_name, llm_payload): server_match = { "control_id": 2, "control_name": "server_ctrl", - "action": "log", + "action": "observe", "result": {"matched": True, "confidence": 1.0}, } client = MagicMock(spec=AgentControlClient) @@ -665,7 +665,7 @@ async def test_tool_step_mixed_local_and_server_controls(self, agent_name, tool_ "server_tool_ctrl", execution="server", pattern=r"test", - action="log", + action="observe", step_type="tool", ), ] @@ -674,7 +674,7 @@ async def test_tool_step_mixed_local_and_server_controls(self, agent_name, tool_ server_match = { "control_id": 2, "control_name": "server_tool_ctrl", - "action": "log", + "action": "observe", "result": {"matched": True, "confidence": 1.0}, } client = MagicMock(spec=AgentControlClient) diff --git a/sdks/python/tests/test_observability.py b/sdks/python/tests/test_observability.py index 961e83b6..04115c2b 100644 --- a/sdks/python/tests/test_observability.py +++ b/sdks/python/tests/test_observability.py @@ -34,7 +34,7 @@ def create_mock_event(): "control_name": "test-control", "check_stage": "pre", "applies_to": "llm_call", - "action": "allow", + "action": "observe", "matched": False, "confidence": 0.95, "timestamp": datetime.now(timezone.utc).isoformat(), @@ -818,7 +818,7 @@ def test_log_span_end(self, caplog): matches=1, non_matches=2, errors=0, - actions={"allow": 1}, + actions={"observe": 1}, ) # Check that logging occurred diff --git a/sdks/python/tests/test_observability_updates.py b/sdks/python/tests/test_observability_updates.py index bb11a5ae..cb792987 100644 --- a/sdks/python/tests/test_observability_updates.py +++ b/sdks/python/tests/test_observability_updates.py @@ -58,7 +58,7 @@ def _make_response(self, **kwargs): defaults.update(kwargs) return EvaluationResponse(**defaults) - def _make_match(self, control_id, control_name="ctrl", action="allow", matched=True): + def _make_match(self, control_id, control_name="ctrl", action="observe", matched=True): from agent_control_models import ControlMatch, EvaluatorResult return ControlMatch( control_id=control_id, @@ -273,10 +273,10 @@ def test_composite_control_emits_representative_leaf_metadata(self): }, ] }, - action={"decision": "allow"}, + action={"decision": "observe"}, ), ) - non_match = self._make_match(1, "composite-ctrl", action="allow", matched=False) + non_match = self._make_match(1, "composite-ctrl", action="observe", matched=False) response = self._make_response(non_matches=[non_match]) request = self._make_request() @@ -352,7 +352,7 @@ async def test_emits_events_when_trace_context_provided(self): ControlMatch( control_id=1, control_name="test-ctrl", - action="allow", + action="observe", result=EvaluatorResult(matched=False, confidence=0.1), ) ], @@ -369,7 +369,7 @@ async def test_emits_events_when_trace_context_provided(self): "evaluator": {"name": "regex", "config": {"pattern": "test"}}, "selector": {"path": "input"}, }, - "action": {"decision": "allow"}, + "action": {"decision": "observe"}, "execution": "sdk", }, }] @@ -423,7 +423,7 @@ async def test_emits_events_without_trace_context(self): "evaluator": {"name": "regex", "config": {"pattern": "test"}}, "selector": {"path": "input"}, }, - "action": {"decision": "allow"}, + "action": {"decision": "observe"}, "execution": "sdk", }, }] @@ -582,7 +582,7 @@ async def test_non_matches_populated_in_stats(self): { "control_id": 1, "control_name": "ctrl-1", - "action": "allow", + "action": "observe", "result": {"matched": False, "confidence": 0.1}, }, { @@ -610,3 +610,58 @@ async def test_non_matches_populated_in_stats(self): assert ctx.total_non_matches == 2 assert ctx.total_matches == 0 assert ctx.total_errors == 0 + + def test_legacy_advisory_matches_collapse_into_observe_stats(self): + """Legacy advisory action names should not leak into local action counters.""" + from agent_control.control_decorators import ControlContext + + result = { + "is_safe": True, + "confidence": 1.0, + "matches": [ + { + "control_id": 1, + "control_name": "ctrl-allow", + "action": "allow", + "result": {"matched": True, "confidence": 0.3}, + }, + { + "control_id": 2, + "control_name": "ctrl-warn", + "action": "warn", + "result": {"matched": True, "confidence": 0.4}, + }, + { + "control_id": 3, + "control_name": "ctrl-log", + "action": "log", + "result": {"matched": True, "confidence": 0.5}, + }, + { + "control_id": 4, + "control_name": "ctrl-observe", + "action": "observe", + "result": {"matched": True, "confidence": 0.6}, + }, + ], + "errors": None, + "non_matches": None, + } + + ctx = ControlContext( + agent_name="test-agent", + server_url="http://localhost:8000", + func=lambda: None, + args=(), + kwargs={}, + trace_id="trace123", + span_id="span456", + start_time=0, + ) + + ctx._update_stats(result) + assert ctx.total_executions == 4 + assert ctx.total_matches == 4 + assert ctx.total_non_matches == 0 + assert ctx.total_errors == 0 + assert ctx.actions == {"observe": 4} diff --git a/sdks/python/tests/test_strands_plugin.py b/sdks/python/tests/test_strands_plugin.py index e9fb5474..3f52848e 100644 --- a/sdks/python/tests/test_strands_plugin.py +++ b/sdks/python/tests/test_strands_plugin.py @@ -113,9 +113,9 @@ def test_action_error_with_no_blocking_match(): """Test _action_error returns None when no blocking matches.""" from agent_control.integrations.strands.plugin import _action_error - # Create a mock match with allow action + # Create a mock match with observe action mock_match = MagicMock() - mock_match.action = "allow" + mock_match.action = "observe" # Create a mock result result = MagicMock(spec=EvaluationResult) @@ -286,7 +286,7 @@ async def test_evaluate_and_enforce_unsafe_result(agent_control_hook): # Mock an unsafe result mock_match = MagicMock() - mock_match.action = "log" + mock_match.action = "observe" mock_match.control_name = "test_control" mock_match.result.message = "Unsafe" @@ -530,7 +530,7 @@ async def test_evaluate_and_enforce_unsafe_no_reason_with_match(agent_control_ho # Mock an unsafe result with no reason but with a match that has a message mock_match = MagicMock() - mock_match.action = "log" + mock_match.action = "observe" mock_match.control_name = "test_control" mock_match.result.message = "Match message" diff --git a/sdks/python/tests/test_strands_steering.py b/sdks/python/tests/test_strands_steering.py index e5011277..0553de9d 100644 --- a/sdks/python/tests/test_strands_steering.py +++ b/sdks/python/tests/test_strands_steering.py @@ -210,12 +210,12 @@ async def test_steer_after_model_no_matches(steering_handler, mock_strands_steer @pytest.mark.asyncio -async def test_steer_after_model_allow_match(steering_handler, mock_strands_steering_modules): - """Test steer_after_model returns Proceed for allow action.""" +async def test_steer_after_model_observe_match(steering_handler, mock_strands_steering_modules): + """Test steer_after_model returns Proceed for observe action.""" with patch("agent_control.integrations.strands.steering.agent_control.evaluate_controls") as mock_evaluate: # noqa: E501 - # Mock an allow match + # Mock an observe match mock_match = MagicMock() - mock_match.action = "allow" + mock_match.action = "observe" mock_match.control_name = "test_control" mock_result = MagicMock(spec=EvaluationResult) diff --git a/sdks/typescript/src/errors.ts b/sdks/typescript/src/errors.ts index 42e5fee7..566a3e03 100644 --- a/sdks/typescript/src/errors.ts +++ b/sdks/typescript/src/errors.ts @@ -1,4 +1,4 @@ -export type ControlAction = "allow" | "deny"; +export type ControlAction = "deny" | "steer" | "observe"; export interface EvaluationResult { isSafe: boolean; diff --git a/sdks/typescript/src/generated/funcs/observability-query-events.ts b/sdks/typescript/src/generated/funcs/observability-query-events.ts index f61b92f0..92b93bae 100644 --- a/sdks/typescript/src/generated/funcs/observability-query-events.ts +++ b/sdks/typescript/src/generated/funcs/observability-query-events.ts @@ -38,7 +38,7 @@ import { Result } from "../types/fp.js"; * - control_execution_id: Get a specific event * - agent_name: Filter by agent * - control_ids: Filter by controls - * - actions: Filter by actions (allow, deny, warn, log) + * - actions: Filter by actions (deny, steer, observe) * - matched: Filter by matched status * - check_stages: Filter by check stage (pre, post) * - applies_to: Filter by call type (llm_call, tool_call) diff --git a/sdks/typescript/src/generated/models/action-decision.ts b/sdks/typescript/src/generated/models/action-decision.ts new file mode 100644 index 00000000..6be26dba --- /dev/null +++ b/sdks/typescript/src/generated/models/action-decision.ts @@ -0,0 +1,25 @@ +/* + * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + */ + +import * as z from "zod/v4-mini"; +import * as openEnums from "../types/enums.js"; +import { OpenEnum } from "../types/enums.js"; + +export const ActionDecision = { + Deny: "deny", + Steer: "steer", + Observe: "observe", +} as const; +export type ActionDecision = OpenEnum; + +/** @internal */ +export const ActionDecision$inboundSchema: z.ZodMiniType< + ActionDecision, + unknown +> = openEnums.inboundSchema(ActionDecision); +/** @internal */ +export const ActionDecision$outboundSchema: z.ZodMiniType< + string, + ActionDecision +> = openEnums.outboundSchema(ActionDecision); diff --git a/sdks/typescript/src/generated/models/control-action.ts b/sdks/typescript/src/generated/models/control-action.ts index 33126901..ca5af786 100644 --- a/sdks/typescript/src/generated/models/control-action.ts +++ b/sdks/typescript/src/generated/models/control-action.ts @@ -5,9 +5,12 @@ import * as z from "zod/v4-mini"; import { remap as remap$ } from "../lib/primitives.js"; import { safeParse } from "../lib/schemas.js"; -import * as openEnums from "../types/enums.js"; -import { OpenEnum } from "../types/enums.js"; import { Result as SafeParseResult } from "../types/fp.js"; +import { + ActionDecision, + ActionDecision$inboundSchema, + ActionDecision$outboundSchema, +} from "./action-decision.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; import { SteeringContext, @@ -16,49 +19,24 @@ import { SteeringContext$outboundSchema, } from "./steering-context.js"; -/** - * Action to take when control is triggered - */ -export const Decision = { - Allow: "allow", - Deny: "deny", - Steer: "steer", - Warn: "warn", - Log: "log", -} as const; -/** - * Action to take when control is triggered - */ -export type Decision = OpenEnum; - /** * What to do when control matches. */ export type ControlAction = { - /** - * Action to take when control is triggered - */ - decision: Decision; + decision: ActionDecision; /** * Steering context object for steer actions. Strongly recommended when decision='steer' to provide correction suggestions. If not provided, the evaluator result message will be used as fallback. */ steeringContext?: SteeringContext | null | undefined; }; -/** @internal */ -export const Decision$inboundSchema: z.ZodMiniType = - openEnums.inboundSchema(Decision); -/** @internal */ -export const Decision$outboundSchema: z.ZodMiniType = - openEnums.outboundSchema(Decision); - /** @internal */ export const ControlAction$inboundSchema: z.ZodMiniType< ControlAction, unknown > = z.pipe( z.object({ - decision: Decision$inboundSchema, + decision: ActionDecision$inboundSchema, steering_context: z.optional(z.nullable(SteeringContext$inboundSchema)), }), z.transform((v) => { @@ -79,7 +57,7 @@ export const ControlAction$outboundSchema: z.ZodMiniType< ControlAction > = z.pipe( z.object({ - decision: Decision$outboundSchema, + decision: ActionDecision$outboundSchema, steeringContext: z.optional(z.nullable(SteeringContext$outboundSchema)), }), z.transform((v) => { diff --git a/sdks/typescript/src/generated/models/control-execution-event.ts b/sdks/typescript/src/generated/models/control-execution-event.ts index e49b3005..b9f4dd68 100644 --- a/sdks/typescript/src/generated/models/control-execution-event.ts +++ b/sdks/typescript/src/generated/models/control-execution-event.ts @@ -9,25 +9,13 @@ import * as openEnums from "../types/enums.js"; import { OpenEnum } from "../types/enums.js"; import { Result as SafeParseResult } from "../types/fp.js"; import * as types from "../types/primitives.js"; +import { + ActionDecision, + ActionDecision$inboundSchema, + ActionDecision$outboundSchema, +} from "./action-decision.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; -/** - * Action taken by the control - */ -export const ControlExecutionEventAction = { - Allow: "allow", - Deny: "deny", - Steer: "steer", - Warn: "warn", - Log: "log", -} as const; -/** - * Action taken by the control - */ -export type ControlExecutionEventAction = OpenEnum< - typeof ControlExecutionEventAction ->; - /** * Type of call: 'llm_call' or 'tool_call' */ @@ -75,7 +63,7 @@ export type CheckStage = OpenEnum; * control_name: Name of the control (denormalized for queries) * check_stage: "pre" (before execution) or "post" (after execution) * applies_to: "llm_call" or "tool_call" - * action: The action taken (allow, deny, warn, log) + * action: The action taken (deny, steer, observe) * matched: Whether the control evaluator matched * confidence: Confidence score from the evaluator (0.0-1.0) * timestamp: When the control was executed (UTC) @@ -86,10 +74,7 @@ export type CheckStage = OpenEnum; * metadata: Additional metadata for extensibility */ export type ControlExecutionEvent = { - /** - * Action taken by the control - */ - action: ControlExecutionEventAction; + action: ActionDecision; /** * Identifier of the agent */ @@ -156,17 +141,6 @@ export type ControlExecutionEvent = { traceId: string; }; -/** @internal */ -export const ControlExecutionEventAction$inboundSchema: z.ZodMiniType< - ControlExecutionEventAction, - unknown -> = openEnums.inboundSchema(ControlExecutionEventAction); -/** @internal */ -export const ControlExecutionEventAction$outboundSchema: z.ZodMiniType< - string, - ControlExecutionEventAction -> = openEnums.outboundSchema(ControlExecutionEventAction); - /** @internal */ export const ControlExecutionEventAppliesTo$inboundSchema: z.ZodMiniType< ControlExecutionEventAppliesTo, @@ -191,7 +165,7 @@ export const ControlExecutionEvent$inboundSchema: z.ZodMiniType< unknown > = z.pipe( z.object({ - action: ControlExecutionEventAction$inboundSchema, + action: ActionDecision$inboundSchema, agent_name: types.string(), applies_to: ControlExecutionEventAppliesTo$inboundSchema, check_stage: CheckStage$inboundSchema, @@ -253,7 +227,7 @@ export const ControlExecutionEvent$outboundSchema: z.ZodMiniType< ControlExecutionEvent > = z.pipe( z.object({ - action: ControlExecutionEventAction$outboundSchema, + action: ActionDecision$outboundSchema, agentName: z.string(), appliesTo: ControlExecutionEventAppliesTo$outboundSchema, checkStage: CheckStage$outboundSchema, diff --git a/sdks/typescript/src/generated/models/control-match.ts b/sdks/typescript/src/generated/models/control-match.ts index 2a2a9198..6a7caa1f 100644 --- a/sdks/typescript/src/generated/models/control-match.ts +++ b/sdks/typescript/src/generated/models/control-match.ts @@ -5,10 +5,12 @@ import * as z from "zod/v4-mini"; import { remap as remap$ } from "../lib/primitives.js"; import { safeParse } from "../lib/schemas.js"; -import * as openEnums from "../types/enums.js"; -import { OpenEnum } from "../types/enums.js"; import { Result as SafeParseResult } from "../types/fp.js"; import * as types from "../types/primitives.js"; +import { + ActionDecision, + ActionDecision$inboundSchema, +} from "./action-decision.js"; import { SDKValidationError } from "./errors/sdk-validation-error.js"; import { EvaluatorResult, @@ -19,29 +21,11 @@ import { SteeringContext$inboundSchema, } from "./steering-context.js"; -/** - * Action configured for this control - */ -export const Action = { - Allow: "allow", - Deny: "deny", - Steer: "steer", - Warn: "warn", - Log: "log", -} as const; -/** - * Action configured for this control - */ -export type Action = OpenEnum; - /** * Represents a control evaluation result (match, non-match, or error). */ export type ControlMatch = { - /** - * Action configured for this control - */ - action: Action; + action: ActionDecision; /** * Unique ID for this control execution (generated by engine) */ @@ -77,15 +61,11 @@ export type ControlMatch = { steeringContext?: SteeringContext | null | undefined; }; -/** @internal */ -export const Action$inboundSchema: z.ZodMiniType = openEnums - .inboundSchema(Action); - /** @internal */ export const ControlMatch$inboundSchema: z.ZodMiniType = z.pipe( z.object({ - action: Action$inboundSchema, + action: ActionDecision$inboundSchema, control_execution_id: types.optional(types.string()), control_id: types.number(), control_name: types.string(), diff --git a/sdks/typescript/src/generated/models/control-stats-response.ts b/sdks/typescript/src/generated/models/control-stats-response.ts index e9727a51..285796ac 100644 --- a/sdks/typescript/src/generated/models/control-stats-response.ts +++ b/sdks/typescript/src/generated/models/control-stats-response.ts @@ -44,7 +44,7 @@ export type ControlStatsResponse = { * * Invariant: execution_count = match_count + non_match_count + error_count * - * Matches have actions (allow, deny, steer, warn, log) tracked in action_counts. + * Matches have actions (deny, steer, observe) tracked in action_counts. * sum(action_counts.values()) == match_count * * Attributes: diff --git a/sdks/typescript/src/generated/models/control-stats.ts b/sdks/typescript/src/generated/models/control-stats.ts index 689be9d1..7a9f0d32 100644 --- a/sdks/typescript/src/generated/models/control-stats.ts +++ b/sdks/typescript/src/generated/models/control-stats.ts @@ -20,20 +20,17 @@ import { SDKValidationError } from "./errors/sdk-validation-error.js"; * execution_count: Total number of executions * match_count: Number of times the control matched * non_match_count: Number of times the control did not match - * allow_count: Number of allow actions * deny_count: Number of deny actions * steer_count: Number of steer actions - * warn_count: Number of warn actions - * log_count: Number of log actions + * observe_count: Number of observe actions * error_count: Number of errors during evaluation * avg_confidence: Average confidence score * avg_duration_ms: Average execution duration in milliseconds + * + * Invariant: + * deny_count + steer_count + observe_count == match_count */ export type ControlStats = { - /** - * Allow actions - */ - allowCount: number; /** * Average confidence */ @@ -62,10 +59,6 @@ export type ControlStats = { * Total executions */ executionCount: number; - /** - * Log actions - */ - logCount: number; /** * Total matches */ @@ -75,20 +68,19 @@ export type ControlStats = { */ nonMatchCount: number; /** - * Steer actions + * Observe actions */ - steerCount: number; + observeCount: number; /** - * Warn actions + * Steer actions */ - warnCount: number; + steerCount: number; }; /** @internal */ export const ControlStats$inboundSchema: z.ZodMiniType = z.pipe( z.object({ - allow_count: types.number(), avg_confidence: types.number(), avg_duration_ms: z.optional(z.nullable(types.number())), control_id: types.number(), @@ -96,15 +88,13 @@ export const ControlStats$inboundSchema: z.ZodMiniType = deny_count: types.number(), error_count: types.number(), execution_count: types.number(), - log_count: types.number(), match_count: types.number(), non_match_count: types.number(), + observe_count: types.number(), steer_count: types.number(), - warn_count: types.number(), }), z.transform((v) => { return remap$(v, { - "allow_count": "allowCount", "avg_confidence": "avgConfidence", "avg_duration_ms": "avgDurationMs", "control_id": "controlId", @@ -112,11 +102,10 @@ export const ControlStats$inboundSchema: z.ZodMiniType = "deny_count": "denyCount", "error_count": "errorCount", "execution_count": "executionCount", - "log_count": "logCount", "match_count": "matchCount", "non_match_count": "nonMatchCount", + "observe_count": "observeCount", "steer_count": "steerCount", - "warn_count": "warnCount", }); }), ); diff --git a/sdks/typescript/src/generated/models/event-query-request.ts b/sdks/typescript/src/generated/models/event-query-request.ts index 89c3de92..3484a7db 100644 --- a/sdks/typescript/src/generated/models/event-query-request.ts +++ b/sdks/typescript/src/generated/models/event-query-request.ts @@ -5,15 +5,10 @@ import * as z from "zod/v4-mini"; import { remap as remap$ } from "../lib/primitives.js"; import { ClosedEnum } from "../types/enums.js"; - -export const Actions = { - Allow: "allow", - Deny: "deny", - Steer: "steer", - Warn: "warn", - Log: "log", -} as const; -export type Actions = ClosedEnum; +import { + ActionDecision, + ActionDecision$outboundSchema, +} from "./action-decision.js"; export const AppliesTo = { LlmCall: "llm_call", @@ -40,7 +35,7 @@ export type CheckStages = ClosedEnum; * control_execution_id: Filter by specific event ID * agent_name: Filter by agent identifier * control_ids: Filter by control IDs - * actions: Filter by actions (allow, deny, steer, warn, log) + * actions: Filter by actions (deny, steer, observe) * matched: Filter by matched status * check_stages: Filter by check stages (pre, post) * applies_to: Filter by call type (llm_call, tool_call) @@ -53,7 +48,7 @@ export type EventQueryRequest = { /** * Filter by actions */ - actions?: Array | null | undefined; + actions?: Array | null | undefined; /** * Filter by agent identifier */ @@ -104,11 +99,6 @@ export type EventQueryRequest = { traceId?: string | null | undefined; }; -/** @internal */ -export const Actions$outboundSchema: z.ZodMiniEnum = z.enum( - Actions, -); - /** @internal */ export const AppliesTo$outboundSchema: z.ZodMiniEnum = z.enum( AppliesTo, @@ -141,7 +131,7 @@ export const EventQueryRequest$outboundSchema: z.ZodMiniType< EventQueryRequest > = z.pipe( z.object({ - actions: z.optional(z.nullable(z.array(Actions$outboundSchema))), + actions: z.optional(z.nullable(z.array(ActionDecision$outboundSchema))), agentName: z.optional(z.nullable(z.string())), appliesTo: z.optional(z.nullable(z.array(AppliesTo$outboundSchema))), checkStages: z.optional(z.nullable(z.array(CheckStages$outboundSchema))), diff --git a/sdks/typescript/src/generated/models/index.ts b/sdks/typescript/src/generated/models/index.ts index 02085da1..71614244 100644 --- a/sdks/typescript/src/generated/models/index.ts +++ b/sdks/typescript/src/generated/models/index.ts @@ -2,6 +2,7 @@ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. */ +export * from "./action-decision.js"; export * from "./agent-controls-response.js"; export * from "./agent-ref.js"; export * from "./agent-summary.js"; diff --git a/sdks/typescript/src/generated/models/stats-response.ts b/sdks/typescript/src/generated/models/stats-response.ts index eb6a68cc..003c0364 100644 --- a/sdks/typescript/src/generated/models/stats-response.ts +++ b/sdks/typescript/src/generated/models/stats-response.ts @@ -44,7 +44,7 @@ export type StatsResponse = { * * Invariant: execution_count = match_count + non_match_count + error_count * - * Matches have actions (allow, deny, steer, warn, log) tracked in action_counts. + * Matches have actions (deny, steer, observe) tracked in action_counts. * sum(action_counts.values()) == match_count * * Attributes: diff --git a/sdks/typescript/src/generated/models/stats-totals.ts b/sdks/typescript/src/generated/models/stats-totals.ts index e1269c50..11ab1098 100644 --- a/sdks/typescript/src/generated/models/stats-totals.ts +++ b/sdks/typescript/src/generated/models/stats-totals.ts @@ -20,7 +20,7 @@ import { * * Invariant: execution_count = match_count + non_match_count + error_count * - * Matches have actions (allow, deny, steer, warn, log) tracked in action_counts. + * Matches have actions (deny, steer, observe) tracked in action_counts. * sum(action_counts.values()) == match_count * * Attributes: @@ -33,7 +33,7 @@ import { */ export type StatsTotals = { /** - * Action breakdown for matches: {allow, deny, steer, warn, log} + * Action breakdown for matches: {deny, steer, observe} */ actionCounts?: { [k: string]: number } | undefined; /** diff --git a/sdks/typescript/src/generated/models/timeseries-bucket.ts b/sdks/typescript/src/generated/models/timeseries-bucket.ts index 969394b5..4dbfacc3 100644 --- a/sdks/typescript/src/generated/models/timeseries-bucket.ts +++ b/sdks/typescript/src/generated/models/timeseries-bucket.ts @@ -28,7 +28,7 @@ import { SDKValidationError } from "./errors/sdk-validation-error.js"; */ export type TimeseriesBucket = { /** - * Action breakdown: {allow, deny, steer, warn, log} + * Action breakdown: {deny, steer, observe} */ actionCounts?: { [k: string]: number } | undefined; /** diff --git a/sdks/typescript/src/generated/sdk/observability.ts b/sdks/typescript/src/generated/sdk/observability.ts index 1697b9a8..f11f88ce 100644 --- a/sdks/typescript/src/generated/sdk/observability.ts +++ b/sdks/typescript/src/generated/sdk/observability.ts @@ -51,7 +51,7 @@ export class Observability extends ClientSDK { * - control_execution_id: Get a specific event * - agent_name: Filter by agent * - control_ids: Filter by controls - * - actions: Filter by actions (allow, deny, warn, log) + * - actions: Filter by actions (deny, steer, observe) * - matched: Filter by matched status * - check_stages: Filter by check stage (pre, post) * - applies_to: Filter by call type (llm_call, tool_call) diff --git a/server/alembic/versions/5f2b5f4e1a90_collapse_advisory_actions_to_observe.py b/server/alembic/versions/5f2b5f4e1a90_collapse_advisory_actions_to_observe.py new file mode 100644 index 00000000..172fb342 --- /dev/null +++ b/server/alembic/versions/5f2b5f4e1a90_collapse_advisory_actions_to_observe.py @@ -0,0 +1,39 @@ +"""collapse advisory actions to observe + +Revision ID: 5f2b5f4e1a90 +Revises: 8b23d645f86d +Create Date: 2026-04-01 17:15:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "5f2b5f4e1a90" +down_revision = "8b23d645f86d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + sa.text( + """ + UPDATE controls + SET data = jsonb_set(data, '{action,decision}', '"observe"'::jsonb, false) + WHERE jsonb_typeof(data) = 'object' + AND jsonb_typeof(data->'action') = 'object' + AND data->'action'->>'decision' IN ('allow', 'warn', 'log') + """ + ) + ) + + +def downgrade() -> None: + raise NotImplementedError( + "This migration is irreversible and does not reconstruct legacy advisory action names." + ) diff --git a/server/src/agent_control_server/endpoints/observability.py b/server/src/agent_control_server/endpoints/observability.py index ea127fc3..5de90c0a 100644 --- a/server/src/agent_control_server/endpoints/observability.py +++ b/server/src/agent_control_server/endpoints/observability.py @@ -135,7 +135,7 @@ async def query_events( - control_execution_id: Get a specific event - agent_name: Filter by agent - control_ids: Filter by controls - - actions: Filter by actions (allow, deny, warn, log) + - actions: Filter by actions (deny, steer, observe) - matched: Filter by matched status - check_stages: Filter by check stage (pre, post) - applies_to: Filter by call type (llm_call, tool_call) diff --git a/server/src/agent_control_server/observability/store/base.py b/server/src/agent_control_server/observability/store/base.py index c73de99c..2c78f983 100644 --- a/server/src/agent_control_server/observability/store/base.py +++ b/server/src/agent_control_server/observability/store/base.py @@ -37,7 +37,7 @@ class StatsResult(BaseModel): Invariant: total_executions = total_matches + total_non_matches + total_errors - Matches have actions (allow, deny, warn, log) tracked in action_counts. + Matches have actions (deny, steer, observe) tracked in action_counts. sum(action_counts.values()) == total_matches Attributes: @@ -57,7 +57,7 @@ class StatsResult(BaseModel): total_errors: int = Field(default=0, ge=0, description="Total errors") action_counts: dict[str, int] = Field( default_factory=dict, - description="Action breakdown for matches: {allow, deny, warn, log}", + description="Action breakdown for matches: {deny, steer, observe}", ) timeseries: list[TimeseriesBucket] | None = Field( default=None, diff --git a/server/src/agent_control_server/observability/store/postgres.py b/server/src/agent_control_server/observability/store/postgres.py index 985ca7b5..aff1931e 100644 --- a/server/src/agent_control_server/observability/store/postgres.py +++ b/server/src/agent_control_server/observability/store/postgres.py @@ -13,6 +13,7 @@ import logging from datetime import UTC, datetime, timedelta +from agent_control_models.actions import ActionDecision, expand_action_filter from agent_control_models.observability import ( ControlExecutionEvent, ControlStats, @@ -46,21 +47,23 @@ SQL_ERROR_COUNT = """SUM(CASE WHEN data->>'error_message' IS NOT NULL THEN 1 ELSE 0 END)""" -# Action count expressions (only count when matched) -SQL_ALLOW_COUNT = """SUM(CASE WHEN (data->>'matched')::boolean - AND data->>'action' = 'allow' THEN 1 ELSE 0 END)""" +# Historical event rows intentionally preserve legacy advisory action names +# (`allow`, `warn`, `log`). Query-time reads normalize those stored values into the +# canonical `observe` bucket instead of rewriting past event payloads in-place. +_OBSERVE_ACTION_FILTER: tuple[ActionDecision, ...] = ("observe",) +_OBSERVE_ACTION_SQL_VALUES = ", ".join( + f"'{action}'" for action in expand_action_filter(_OBSERVE_ACTION_FILTER) +) +# Action count expressions (only count when matched) SQL_DENY_COUNT = """SUM(CASE WHEN (data->>'matched')::boolean AND data->>'action' = 'deny' THEN 1 ELSE 0 END)""" SQL_STEER_COUNT = """SUM(CASE WHEN (data->>'matched')::boolean AND data->>'action' = 'steer' THEN 1 ELSE 0 END)""" -SQL_WARN_COUNT = """SUM(CASE WHEN (data->>'matched')::boolean - AND data->>'action' = 'warn' THEN 1 ELSE 0 END)""" - -SQL_LOG_COUNT = """SUM(CASE WHEN (data->>'matched')::boolean - AND data->>'action' = 'log' THEN 1 ELSE 0 END)""" +SQL_OBSERVE_COUNT = f"""SUM(CASE WHEN (data->>'matched')::boolean + AND data->>'action' IN ({_OBSERVE_ACTION_SQL_VALUES}) THEN 1 ELSE 0 END)""" # Average expressions SQL_AVG_CONFIDENCE = "AVG((data->>'confidence')::float)" @@ -74,11 +77,9 @@ {SQL_MATCH_COUNT} as match_count, {SQL_NON_MATCH_COUNT} as non_match_count, {SQL_ERROR_COUNT} as error_count, - {SQL_ALLOW_COUNT} as allow_count, {SQL_DENY_COUNT} as deny_count, {SQL_STEER_COUNT} as steer_count, - {SQL_WARN_COUNT} as warn_count, - {SQL_LOG_COUNT} as log_count, + {SQL_OBSERVE_COUNT} as observe_count, {SQL_AVG_CONFIDENCE} as avg_confidence, {SQL_AVG_DURATION} as avg_duration_ms""" @@ -249,11 +250,9 @@ async def query_stats( COALESCE(bs.match_count, 0) as match_count, COALESCE(bs.non_match_count, 0) as non_match_count, COALESCE(bs.error_count, 0) as error_count, - COALESCE(bs.allow_count, 0) as allow_count, COALESCE(bs.deny_count, 0) as deny_count, COALESCE(bs.steer_count, 0) as steer_count, - COALESCE(bs.warn_count, 0) as warn_count, - COALESCE(bs.log_count, 0) as log_count, + COALESCE(bs.observe_count, 0) as observe_count, bs.avg_confidence, bs.avg_duration_ms FROM all_buckets ab @@ -296,7 +295,7 @@ async def query_stats( total_matches = 0 total_non_matches = 0 total_errors = 0 - action_counts: dict[str, int] = {"allow": 0, "deny": 0, "steer": 0, "warn": 0, "log": 0} + action_counts: dict[str, int] = {"deny": 0, "steer": 0, "observe": 0} for row in rows: if row.query_type == "control": @@ -307,11 +306,9 @@ async def query_stats( match_count=row.match_count, non_match_count=row.non_match_count, error_count=row.error_count, - allow_count=row.allow_count, deny_count=row.deny_count, steer_count=row.steer_count, - warn_count=row.warn_count, - log_count=row.log_count, + observe_count=row.observe_count, avg_confidence=row.avg_confidence or 0.0, avg_duration_ms=row.avg_duration_ms, ) @@ -321,27 +318,21 @@ async def query_stats( total_matches += row.match_count total_non_matches += row.non_match_count total_errors += row.error_count - action_counts["allow"] += row.allow_count action_counts["deny"] += row.deny_count action_counts["steer"] += row.steer_count - action_counts["warn"] += row.warn_count - action_counts["log"] += row.log_count + action_counts["observe"] += row.observe_count elif row.query_type == "timeseries": if timeseries is None: timeseries = [] bucket_action_counts: dict[str, int] = {} - if row.allow_count > 0: - bucket_action_counts["allow"] = row.allow_count if row.deny_count > 0: bucket_action_counts["deny"] = row.deny_count if row.steer_count > 0: bucket_action_counts["steer"] = row.steer_count - if row.warn_count > 0: - bucket_action_counts["warn"] = row.warn_count - if row.log_count > 0: - bucket_action_counts["log"] = row.log_count + if row.observe_count > 0: + bucket_action_counts["observe"] = row.observe_count timeseries.append(TimeseriesBucket( timestamp=row.bucket, @@ -440,7 +431,7 @@ async def query_events(self, query: EventQueryRequest) -> EventQueryResponse: if query.actions: where_clauses.append("data->>'action' = ANY(:actions)") - params["actions"] = query.actions + params["actions"] = expand_action_filter(query.actions) if query.matched is not None: where_clauses.append("(data->>'matched')::boolean = :matched") diff --git a/server/tests/test_control_action_alembic_migration.py b/server/tests/test_control_action_alembic_migration.py new file mode 100644 index 00000000..623ee559 --- /dev/null +++ b/server/tests/test_control_action_alembic_migration.py @@ -0,0 +1,149 @@ +"""Alembic coverage for collapsing advisory actions to observe.""" + +from __future__ import annotations + +import json +import uuid +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine, make_url + +from agent_control_server.config import db_config + +from .utils import VALID_CONTROL_PAYLOAD + +SERVER_DIR = Path(__file__).resolve().parents[1] +PRE_MIGRATION_REVISION = "8b23d645f86d" +MIGRATION_REVISION = "5f2b5f4e1a90" +_BASE_DB_URL = make_url(db_config.get_url()) + +pytestmark = pytest.mark.skipif( + _BASE_DB_URL.get_backend_name() != "postgresql", + reason="Control action Alembic migration tests require PostgreSQL.", +) + + +def _control_payload(action: str) -> dict[str, Any]: + payload = deepcopy(VALID_CONTROL_PAYLOAD) + payload["action"]["decision"] = action + return payload + + +def _insert_control(engine: Engine, *, name: str, data: dict[str, Any]) -> int: + with engine.begin() as conn: + return int( + conn.execute( + text( + """ + INSERT INTO controls (name, data) + VALUES (:name, CAST(:data AS JSONB)) + RETURNING id + """ + ), + {"name": name, "data": json.dumps(data)}, + ).scalar_one() + ) + + +def _fetch_control_data(engine: Engine, control_id: int) -> Any: + with engine.begin() as conn: + return conn.execute( + text("SELECT data FROM controls WHERE id = :id"), + {"id": control_id}, + ).scalar_one() + + +@pytest.fixture +def temp_db_url() -> str: + temp_db_name = f"agent_control_action_{uuid.uuid4().hex[:12]}" + admin_url = _BASE_DB_URL.set(database="postgres").render_as_string(hide_password=False) + target_url = _BASE_DB_URL.set(database=temp_db_name).render_as_string(hide_password=False) + + admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT") + with admin_engine.connect() as conn: + conn.execute(text(f'CREATE DATABASE "{temp_db_name}"')) + admin_engine.dispose() + + try: + yield target_url + finally: + cleanup_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT") + with cleanup_engine.connect() as conn: + conn.execute( + text( + """ + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = :db_name AND pid <> pg_backend_pid() + """ + ), + {"db_name": temp_db_name}, + ) + conn.execute(text(f'DROP DATABASE IF EXISTS "{temp_db_name}"')) + cleanup_engine.dispose() + + +@pytest.fixture +def alembic_config(temp_db_url: str) -> Config: + cfg = Config(str(SERVER_DIR / "alembic.ini")) + cfg.set_main_option("script_location", str(SERVER_DIR / "alembic")) + cfg.set_main_option("sqlalchemy.url", temp_db_url) + return cfg + + +@pytest.fixture +def temp_engine(temp_db_url: str) -> Engine: + engine = create_engine(temp_db_url, future=True) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture +def upgrade_to(alembic_config: Config): + def _upgrade(revision: str, *, sql: bool = False) -> None: + command.upgrade(alembic_config, revision, sql=sql) + + return _upgrade + + +def test_upgrade_collapses_legacy_advisory_actions( + upgrade_to, + temp_engine: Engine, +) -> None: + upgrade_to(PRE_MIGRATION_REVISION) + + allow_id = _insert_control(temp_engine, name="allow-legacy", data=_control_payload("allow")) + warn_id = _insert_control(temp_engine, name="warn-legacy", data=_control_payload("warn")) + log_id = _insert_control(temp_engine, name="log-legacy", data=_control_payload("log")) + + upgrade_to(MIGRATION_REVISION) + + for control_id in (allow_id, warn_id, log_id): + assert _fetch_control_data(temp_engine, control_id)["action"]["decision"] == "observe" + + +def test_upgrade_preserves_canonical_actions( + upgrade_to, + temp_engine: Engine, +) -> None: + upgrade_to(PRE_MIGRATION_REVISION) + + deny_id = _insert_control(temp_engine, name="deny", data=_control_payload("deny")) + steer_id = _insert_control(temp_engine, name="steer", data=_control_payload("steer")) + observe_id = _insert_control( + temp_engine, name="observe", data=_control_payload("observe") + ) + + upgrade_to(MIGRATION_REVISION) + + assert _fetch_control_data(temp_engine, deny_id)["action"]["decision"] == "deny" + assert _fetch_control_data(temp_engine, steer_id)["action"]["decision"] == "steer" + assert _fetch_control_data(temp_engine, observe_id)["action"]["decision"] == "observe" diff --git a/server/tests/test_evaluation_e2e.py b/server/tests/test_evaluation_e2e.py index 3ba5cde6..fbe9a483 100644 --- a/server/tests/test_evaluation_e2e.py +++ b/server/tests/test_evaluation_e2e.py @@ -208,15 +208,15 @@ def test_evaluation_tool_step_nested(client: TestClient): def test_evaluation_deny_precedence(client: TestClient): """Test that Deny takes precedence over other controls.""" - # Given: A policy with two controls: one Warn, one Deny + # Given: A policy with two controls: one Observe, one Deny control_warn = { - "description": "Warn on keyword", + "description": "Observe on keyword", "enabled": True, "execution": "server", "scope": {"step_types": ["llm"], "stages": ["pre"]}, "selector": {"path": "input"}, "evaluator": {"name": "regex", "config": {"pattern": "keyword"}}, - "action": {"decision": "warn"} + "action": {"decision": "observe"} } # Use helper to setup agent with first control agent_name, warn_control_name = create_and_assign_policy(client, control_warn, agent_name="PrecedenceAgent") @@ -263,7 +263,7 @@ def test_evaluation_deny_precedence(client: TestClient): assert len(data["matches"]) == 2 actions = {m["action"] for m in data["matches"]} assert "deny" in actions - assert "warn" in actions + assert "observe" in actions def test_evaluation_stage_filtering(client: TestClient): diff --git a/server/tests/test_observability_direct_ingest.py b/server/tests/test_observability_direct_ingest.py index f0d3b964..1e83f816 100644 --- a/server/tests/test_observability_direct_ingest.py +++ b/server/tests/test_observability_direct_ingest.py @@ -50,7 +50,7 @@ async def test_direct_ingestor_drops_on_store_error() -> None: control_name="c", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=True, confidence=0.9, ) @@ -78,7 +78,7 @@ async def test_direct_ingestor_logs_when_enabled(caplog: pytest.LogCaptureFixtur control_name="c", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=True, confidence=0.9, ) diff --git a/server/tests/test_observability_endpoints.py b/server/tests/test_observability_endpoints.py index 528b3846..476cf00c 100644 --- a/server/tests/test_observability_endpoints.py +++ b/server/tests/test_observability_endpoints.py @@ -1,10 +1,12 @@ """Tests for observability API endpoints.""" +import json from datetime import datetime, timedelta, timezone from uuid import UUID, uuid4 import pytest from fastapi.testclient import TestClient +from sqlalchemy import text from agent_control_models import ( BatchEventsRequest, @@ -18,7 +20,7 @@ def create_test_event( control_id: int = 1, agent_name: str | UUID | None = None, - action: str = "allow", + action: str = "observe", matched: bool = False, timestamp: datetime | None = None, execution_duration_ms: float | None = None, @@ -108,7 +110,7 @@ def test_query_request_with_filters(self): trace_id="a" * 32, agent_name=f"agent-{uuid4().hex[:12]}", control_ids=[1, 2, 3], - actions=["allow", "deny"], + actions=["observe", "deny"], matched=True, check_stages=["pre", "post"], applies_to=["llm_call"], @@ -117,6 +119,7 @@ def test_query_request_with_filters(self): ) assert request.trace_id == "a" * 32 assert len(request.control_ids) == 3 + assert request.actions == ["observe", "deny"] assert request.limit == 50 @@ -273,7 +276,7 @@ async def test_stats_with_timeseries(self, client: TestClient, setup_observabili create_test_event( agent_name=agent_name, matched=True, - action="allow", + action="observe", timestamp=now - timedelta(minutes=30), execution_duration_ms=10.0, ), @@ -395,7 +398,7 @@ async def test_timeseries_aggregates_events_per_bucket( create_test_event( agent_name=agent_name, matched=True, - action="allow", + action="observe", timestamp=base_time + timedelta(seconds=30), execution_duration_ms=10.0, ), @@ -438,13 +441,15 @@ async def test_timeseries_aggregates_events_per_bucket( total_exec = sum(b["execution_count"] for b in buckets_with_events) total_match = sum(b["match_count"] for b in buckets_with_events) total_non_match = sum(b["non_match_count"] for b in buckets_with_events) - total_allow = sum(b["action_counts"].get("allow", 0) for b in buckets_with_events) + total_observe = sum( + b["action_counts"].get("observe", 0) for b in buckets_with_events + ) total_deny = sum(b["action_counts"].get("deny", 0) for b in buckets_with_events) assert total_exec == 3 assert total_match == 2 assert total_non_match == 1 - assert total_allow == 1 + assert total_observe == 1 assert total_deny == 1 @pytest.mark.asyncio @@ -507,9 +512,9 @@ async def test_control_stats_basic(self, client: TestClient, setup_observability # Create events for multiple controls events = [ - create_test_event(control_id=1, agent_name=agent_name, matched=True, action="allow"), + create_test_event(control_id=1, agent_name=agent_name, matched=True, action="observe"), create_test_event(control_id=1, agent_name=agent_name, matched=True, action="deny"), - create_test_event(control_id=2, agent_name=agent_name, matched=True, action="warn"), + create_test_event(control_id=2, agent_name=agent_name, matched=True, action="observe"), ] await store.store(events) @@ -532,7 +537,7 @@ async def test_control_stats_basic(self, client: TestClient, setup_observability # Verify only control 1's stats assert data["stats"]["execution_count"] == 2 assert data["stats"]["match_count"] == 2 - assert data["stats"]["action_counts"]["allow"] == 1 + assert data["stats"]["action_counts"]["observe"] == 1 assert data["stats"]["action_counts"]["deny"] == 1 @pytest.mark.asyncio @@ -548,7 +553,7 @@ async def test_control_stats_with_timeseries(self, client: TestClient, setup_obs control_id=1, agent_name=agent_name, matched=True, - action="allow", + action="observe", timestamp=now - timedelta(minutes=30), ), create_test_event( @@ -563,7 +568,7 @@ async def test_control_stats_with_timeseries(self, client: TestClient, setup_obs control_id=2, agent_name=agent_name, matched=True, - action="warn", + action="observe", timestamp=now - timedelta(minutes=20), ), ] @@ -617,6 +622,51 @@ async def test_control_stats_no_data(self, client: TestClient, setup_observabili assert data["stats"]["execution_count"] == 0 assert data["stats"]["match_count"] == 0 + @pytest.mark.asyncio + async def test_control_stats_normalize_historical_legacy_advisory_rows( + self, client: TestClient, setup_observability + ): + """Historical advisory rows are normalized to observe in control stats responses.""" + store = setup_observability + agent_name = f"agent-{uuid4().hex[:12]}" + event = create_test_event( + control_id=11, + agent_name=agent_name, + matched=True, + action="observe", + ) + legacy_payload = event.model_dump(mode="json") + legacy_payload["action"] = "log" + + async with store.session_maker() as session: + await session.execute( + text(""" + INSERT INTO control_execution_events ( + control_execution_id, timestamp, agent_name, data + ) VALUES ( + :control_execution_id, :timestamp, :agent_name, CAST(:data AS JSONB) + ) + """), + { + "control_execution_id": event.control_execution_id, + "timestamp": event.timestamp, + "agent_name": event.agent_name, + "data": json.dumps(legacy_payload), + }, + ) + await session.commit() + + response = client.get( + "/api/v1/observability/stats/controls/11", + params={"agent_name": str(agent_name), "time_range": "1h"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["stats"]["execution_count"] == 1 + assert data["stats"]["match_count"] == 1 + assert data["stats"]["action_counts"] == {"observe": 1} + class TestObservabilityQueries: """Tests for observability query endpoints.""" @@ -624,11 +674,11 @@ class TestObservabilityQueries: def test_query_events_filters_and_pagination(self, client: TestClient, setup_observability): """Test POST /events/query with filters and pagination.""" # Given: multiple events ingested into the store - event1 = create_test_event(control_id=1, action="allow", matched=True) + event1 = create_test_event(control_id=1, action="observe", matched=True) event2 = create_test_event(control_id=2, action="deny", matched=False).model_copy( update={"trace_id": "c" * 32} ) - event3 = create_test_event(control_id=1, action="allow", matched=True).model_copy( + event3 = create_test_event(control_id=1, action="observe", matched=True).model_copy( update={"span_id": "d" * 16} ) @@ -667,14 +717,14 @@ def test_get_stats_aggregates_events(self, client: TestClient, setup_observabili """Test GET /stats aggregates events for an agent.""" # Given: events for a specific agent and one other agent agent_name = f"agent-{uuid4().hex[:12]}" - event1 = create_test_event(control_id=1, action="allow", matched=True).model_copy( + event1 = create_test_event(control_id=1, action="observe", matched=True).model_copy( update={"agent_name": agent_name} ) event2 = create_test_event(control_id=2, action="deny", matched=True).model_copy( update={"agent_name": agent_name, "trace_id": "c" * 32} ) # Event from different agent (should not be counted) - event3 = create_test_event(control_id=1, action="warn", matched=True).model_copy( + event3 = create_test_event(control_id=1, action="observe", matched=True).model_copy( update={"trace_id": "d" * 32} ) @@ -699,6 +749,48 @@ def test_get_stats_aggregates_events(self, client: TestClient, setup_observabili # And per-control breakdown exists assert len(body["controls"]) == 2 + def test_ingest_query_and_stats_accept_legacy_advisory_actions( + self, client: TestClient, setup_observability + ): + """Legacy advisory actions remain accepted at the API boundary for one release cycle.""" + agent_name = f"agent-{uuid4().hex[:12]}" + legacy_event = create_test_event( + control_id=7, + agent_name=agent_name, + action="observe", + matched=True, + ).model_dump(mode="json") + legacy_event["action"] = "warn" + + ingest_resp = client.post( + "/api/v1/observability/events", + json={"events": [legacy_event]}, + ) + assert ingest_resp.status_code == 202 + + query_resp = client.post( + "/api/v1/observability/events/query", + json=EventQueryRequest( + agent_name=agent_name, + actions=["observe"], + limit=10, + offset=0, + ).model_dump(mode="json"), + ) + assert query_resp.status_code == 200 + query_body = query_resp.json() + assert query_body["total"] == 1 + assert query_body["events"][0]["control_id"] == 7 + assert query_body["events"][0]["action"] == "observe" + + stats_resp = client.get( + "/api/v1/observability/stats", + params={"agent_name": str(agent_name), "time_range": "1h"}, + ) + assert stats_resp.status_code == 200 + stats_body = stats_resp.json() + assert stats_body["totals"]["action_counts"] == {"observe": 1} + class TestObservabilityIngestStatus: """Tests for ingestion status mapping.""" diff --git a/server/tests/test_observability_models.py b/server/tests/test_observability_models.py index 01d02be6..deb30fa1 100644 --- a/server/tests/test_observability_models.py +++ b/server/tests/test_observability_models.py @@ -53,7 +53,7 @@ def test_trace_id_validation_empty(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -70,7 +70,7 @@ def test_trace_id_accepts_various_formats(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -88,7 +88,7 @@ def test_span_id_validation_empty(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -105,7 +105,7 @@ def test_confidence_bounds(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=1.5, # > 1.0 ) @@ -121,14 +121,22 @@ def test_check_stage_values(self): control_name="test", check_stage="invalid", # invalid applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) def test_action_values(self): - """Test valid action values.""" - for action in ["allow", "deny", "steer", "warn", "log"]: + """Test canonical actions and legacy advisory normalization.""" + expected_actions = { + "allow": "observe", + "deny": "deny", + "steer": "steer", + "warn": "observe", + "log": "observe", + "observe": "observe", + } + for action, expected in expected_actions.items(): event = ControlExecutionEvent( trace_id="4bf92f3577b34da6a3ce929d0e0e4736", span_id="00f067aa0ba902b7", @@ -141,7 +149,7 @@ def test_action_values(self): matched=True, confidence=0.9, ) - assert event.action == action + assert event.action == expected def test_timestamp_default(self): """Test that timestamp defaults to now (UTC).""" @@ -153,7 +161,7 @@ def test_timestamp_default(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -171,7 +179,7 @@ def test_optional_fields(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, execution_duration_ms=15.3, @@ -196,7 +204,7 @@ def test_to_dict(self): control_name="test", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -220,7 +228,7 @@ def test_valid_batch(self): control_name=f"control-{i}", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -245,7 +253,7 @@ def test_max_batch_size(self): control_name=f"control-{i}", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=False, confidence=0.5, ) @@ -301,7 +309,7 @@ def test_filter_by_trace_id(self): def test_filter_by_actions(self): """Test filtering by actions.""" query = EventQueryRequest(actions=["deny", "warn"]) - assert query.actions == ["deny", "warn"] + assert query.actions == ["deny", "observe"] def test_limit_bounds(self): """Test limit bounds.""" @@ -322,11 +330,9 @@ def test_valid_stats(self): execution_count=1000, match_count=50, non_match_count=950, - allow_count=950, deny_count=45, steer_count=0, - warn_count=5, - log_count=0, + observe_count=5, error_count=0, avg_confidence=0.95, ) @@ -357,4 +363,3 @@ def test_time_range_values(self): assert request.time_range == time_range - diff --git a/server/tests/test_observability_store_postgres.py b/server/tests/test_observability_store_postgres.py index 9643ca8f..72cd4ab6 100644 --- a/server/tests/test_observability_store_postgres.py +++ b/server/tests/test_observability_store_postgres.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from datetime import UTC, datetime, timedelta from uuid import uuid4 @@ -65,7 +66,7 @@ async def test_postgres_event_store_query_events_and_stats() -> None: _event( agent_name=agent_name, control_id=1, - action="allow", + action="observe", matched=True, timestamp=now - timedelta(seconds=10), trace_id="a" * 32, @@ -81,7 +82,7 @@ async def test_postgres_event_store_query_events_and_stats() -> None: _event( agent_name=agent_name, control_id=1, - action="allow", + action="observe", matched=True, timestamp=now, trace_id="a" * 32, @@ -112,7 +113,7 @@ async def test_postgres_event_store_query_events_and_stats() -> None: assert stats.total_matches == 2 assert stats.total_non_matches == 1 assert stats.total_errors == 0 - assert stats.action_counts == {"allow": 2} + assert stats.action_counts == {"observe": 2} # When: querying stats with a control filter filtered_stats = await store.query_stats(agent_name, timedelta(hours=1), control_id=1) @@ -160,7 +161,7 @@ async def test_postgres_event_store_query_events_all_filters() -> None: _event( agent_name=agent_name, control_id=1, - action="allow", + action="observe", matched=True, timestamp=now - timedelta(seconds=1), trace_id=target_trace_id, @@ -194,7 +195,7 @@ async def test_postgres_event_store_query_events_all_filters() -> None: trace_id=target_trace_id, span_id=target_span_id, control_ids=[1], - actions=["allow"], + actions=["observe"], matched=True, check_stages=["pre"], applies_to=["llm_call"], @@ -209,8 +210,61 @@ async def test_postgres_event_store_query_events_all_filters() -> None: @pytest.mark.asyncio -async def test_postgres_event_store_timeseries_includes_steer_and_warn_counts() -> None: - # Given: a Postgres-backed store with steer and warn events +async def test_postgres_event_store_normalizes_legacy_advisory_rows() -> None: + # Given: a historical event row stored with a legacy advisory action name + session_maker = async_sessionmaker( + bind=async_engine, + class_=AsyncSession, + expire_on_commit=False, + ) + store = PostgresEventStore(session_maker) + + agent_name = f"agent-{uuid4().hex[:12]}" + now = datetime.now(UTC) + event = _event( + agent_name=agent_name, + control_id=7, + action="observe", + matched=True, + timestamp=now, + trace_id="d" * 32, + ) + legacy_payload = event.model_dump(mode="json") + legacy_payload["action"] = "warn" + + async with session_maker() as session: + await session.execute( + text(""" + INSERT INTO control_execution_events ( + control_execution_id, timestamp, agent_name, data + ) VALUES ( + :control_execution_id, :timestamp, :agent_name, CAST(:data AS JSONB) + ) + """), + { + "control_execution_id": event.control_execution_id, + "timestamp": event.timestamp, + "agent_name": event.agent_name, + "data": json.dumps(legacy_payload), + }, + ) + await session.commit() + + # When: querying with the canonical observe filter + resp = await store.query_events( + EventQueryRequest(agent_name=agent_name, actions=["observe"], limit=10, offset=0) + ) + stats = await store.query_stats(agent_name, timedelta(hours=1)) + + # Then: the legacy row is returned and normalized to observe + assert resp.total == 1 + assert resp.events[0].action == "observe" + assert stats.action_counts == {"observe": 1} + + +@pytest.mark.asyncio +async def test_postgres_event_store_timeseries_includes_steer_and_observe_counts() -> None: + # Given: a Postgres-backed store with steer and advisory events session_maker = async_sessionmaker( bind=async_engine, class_=AsyncSession, @@ -233,7 +287,7 @@ async def test_postgres_event_store_timeseries_includes_steer_and_warn_counts() _event( agent_name=agent_name, control_id=2, - action="warn", + action="observe", matched=True, timestamp=now - timedelta(seconds=5), trace_id="b" * 32, @@ -241,7 +295,7 @@ async def test_postgres_event_store_timeseries_includes_steer_and_warn_counts() _event( agent_name=agent_name, control_id=3, - action="allow", + action="observe", matched=True, timestamp=now, trace_id="c" * 32, @@ -259,12 +313,11 @@ async def test_postgres_event_store_timeseries_includes_steer_and_warn_counts() bucket_size=timedelta(minutes=1), ) - # Then: action counts include steer and warn + # Then: action counts include steer and observe assert stats.action_counts["steer"] == 1 - assert stats.action_counts["warn"] == 1 - assert stats.action_counts["allow"] == 1 + assert stats.action_counts["observe"] == 2 - # Then: timeseries buckets include steer and warn in their action_counts + # Then: timeseries buckets include steer and observe in their action_counts assert stats.timeseries is not None all_bucket_actions = [ action @@ -272,7 +325,7 @@ async def test_postgres_event_store_timeseries_includes_steer_and_warn_counts() for action in bucket.action_counts.keys() ] assert "steer" in all_bucket_actions - assert "warn" in all_bucket_actions + assert "observe" in all_bucket_actions @pytest.mark.asyncio @@ -286,7 +339,7 @@ async def test_postgres_event_store_parses_string_json_rows() -> None: control_name="control-1", check_stage="pre", applies_to="llm_call", - action="allow", + action="observe", matched=True, confidence=0.9, timestamp=datetime.now(UTC), diff --git a/server/unit_tests/test_endpoint_helpers.py b/server/unit_tests/test_endpoint_helpers.py index 9c878841..0b0b214f 100644 --- a/server/unit_tests/test_endpoint_helpers.py +++ b/server/unit_tests/test_endpoint_helpers.py @@ -75,7 +75,7 @@ async def test_emit_observability_events_uses_representative_leaf_for_composites }, ] }, - action={"decision": "allow"}, + action={"decision": "observe"}, ), ) response = EvaluationResponse( @@ -85,7 +85,7 @@ async def test_emit_observability_events_uses_representative_leaf_for_composites ControlMatch( control_id=1, control_name="composite-ctrl", - action="allow", + action="observe", result=EvaluatorResult( matched=False, confidence=0.9, diff --git a/ui/src/core/api/generated/api-types.ts b/ui/src/core/api/generated/api-types.ts index 907d2b3c..08bce80c 100644 --- a/ui/src/core/api/generated/api-types.ts +++ b/ui/src/core/api/generated/api-types.ts @@ -725,7 +725,7 @@ export interface paths { * - control_execution_id: Get a specific event * - agent_name: Filter by agent * - control_ids: Filter by controls - * - actions: Filter by actions (allow, deny, warn, log) + * - actions: Filter by actions (deny, steer, observe) * - matched: Filter by matched status * - check_stages: Filter by check stage (pre, post) * - applies_to: Filter by call type (llm_call, tool_call) @@ -980,6 +980,8 @@ export interface paths { export type webhooks = Record; export interface components { schemas: { + /** @enum {string} */ + ActionDecision: 'deny' | 'steer' | 'observe'; /** * Agent * @description Agent metadata for registration and tracking. @@ -1344,12 +1346,8 @@ export interface components { * @description What to do when control matches. */ ControlAction: { - /** - * Decision - * @description Action to take when control is triggered - * @enum {string} - */ - decision: 'allow' | 'deny' | 'steer' | 'warn' | 'log'; + /** @description Action to take when control is triggered */ + decision: components['schemas']['ActionDecision']; /** @description Steering context object for steer actions. Strongly recommended when decision='steer' to provide correction suggestions. If not provided, the evaluator result message will be used as fallback. */ steering_context?: components['schemas']['SteeringContext'] | null; }; @@ -1509,7 +1507,7 @@ export interface components { * control_name: Name of the control (denormalized for queries) * check_stage: "pre" (before execution) or "post" (after execution) * applies_to: "llm_call" or "tool_call" - * action: The action taken (allow, deny, warn, log) + * action: The action taken (deny, steer, observe) * matched: Whether the control evaluator matched * confidence: Confidence score from the evaluator (0.0-1.0) * timestamp: When the control was executed (UTC) @@ -1537,12 +1535,8 @@ export interface components { * } */ ControlExecutionEvent: { - /** - * Action - * @description Action taken by the control - * @enum {string} - */ - action: 'allow' | 'deny' | 'steer' | 'warn' | 'log'; + /** @description Action taken by the control */ + action: components['schemas']['ActionDecision']; /** * Agent Name * @description Identifier of the agent @@ -1634,12 +1628,8 @@ export interface components { * @description Represents a control evaluation result (match, non-match, or error). */ ControlMatch: { - /** - * Action - * @description Action configured for this control - * @enum {string} - */ - action: 'allow' | 'deny' | 'steer' | 'warn' | 'log'; + /** @description Action configured for this control */ + action: components['schemas']['ActionDecision']; /** * Control Execution Id * @description Unique ID for this control execution (generated by engine) @@ -1754,21 +1744,14 @@ export interface components { * execution_count: Total number of executions * match_count: Number of times the control matched * non_match_count: Number of times the control did not match - * allow_count: Number of allow actions * deny_count: Number of deny actions * steer_count: Number of steer actions - * warn_count: Number of warn actions - * log_count: Number of log actions + * observe_count: Number of observe actions * error_count: Number of errors during evaluation * avg_confidence: Average confidence score * avg_duration_ms: Average execution duration in milliseconds */ ControlStats: { - /** - * Allow Count - * @description Allow actions - */ - allow_count: number; /** * Avg Confidence * @description Average confidence @@ -1804,11 +1787,6 @@ export interface components { * @description Total executions */ execution_count: number; - /** - * Log Count - * @description Log actions - */ - log_count: number; /** * Match Count * @description Total matches @@ -1819,16 +1797,16 @@ export interface components { * @description Total non-matches */ non_match_count: number; + /** + * Observe Count + * @description Observe actions + */ + observe_count: number; /** * Steer Count * @description Steer actions */ steer_count: number; - /** - * Warn Count - * @description Warn actions - */ - warn_count: number; }; /** * ControlStatsResponse @@ -2294,7 +2272,7 @@ export interface components { * control_execution_id: Filter by specific event ID * agent_name: Filter by agent identifier * control_ids: Filter by control IDs - * actions: Filter by actions (allow, deny, steer, warn, log) + * actions: Filter by actions (deny, steer, observe) * matched: Filter by matched status * check_stages: Filter by check stages (pre, post) * applies_to: Filter by call type (llm_call, tool_call) @@ -2308,7 +2286,7 @@ export interface components { * @example { * "actions": [ * "deny", - * "warn" + * "observe" * ], * "agent_name": "my-agent", * "limit": 50, @@ -2320,7 +2298,7 @@ export interface components { * Actions * @description Filter by actions */ - actions?: ('allow' | 'deny' | 'steer' | 'warn' | 'log')[] | null; + actions?: components['schemas']['ActionDecision'][] | null; /** * Agent Name * @description Filter by agent identifier @@ -2913,7 +2891,7 @@ export interface components { * * Invariant: execution_count = match_count + non_match_count + error_count * - * Matches have actions (allow, deny, steer, warn, log) tracked in action_counts. + * Matches have actions (deny, steer, observe) tracked in action_counts. * sum(action_counts.values()) == match_count * * Attributes: @@ -2927,7 +2905,7 @@ export interface components { StatsTotals: { /** * Action Counts - * @description Action breakdown for matches: {allow, deny, steer, warn, log} + * @description Action breakdown for matches: {deny, steer, observe} */ action_counts?: { [key: string]: number; @@ -3117,7 +3095,7 @@ export interface components { TimeseriesBucket: { /** * Action Counts - * @description Action breakdown: {allow, deny, steer, warn, log} + * @description Action breakdown: {deny, steer, observe} */ action_counts?: { [key: string]: number; diff --git a/ui/src/core/page-components/agent-detail/modals/edit-control/control-definition-form.tsx b/ui/src/core/page-components/agent-detail/modals/edit-control/control-definition-form.tsx index 61e266d0..77aeb98d 100644 --- a/ui/src/core/page-components/agent-detail/modals/edit-control/control-definition-form.tsx +++ b/ui/src/core/page-components/agent-detail/modals/edit-control/control-definition-form.tsx @@ -95,16 +95,14 @@ export const ControlDefinitionForm = ({ label={ } labelProps={labelPropsInline} data={[ - { value: 'allow', label: 'Allow' }, { value: 'deny', label: 'Deny' }, { value: 'steer', label: 'Steer' }, - { value: 'warn', label: 'Warn' }, - { value: 'log', label: 'Log' }, + { value: 'observe', label: 'Observe' }, ]} size="sm" {...form.getInputProps('action_decision')} diff --git a/ui/src/core/page-components/agent-detail/monitor/summary-card.tsx b/ui/src/core/page-components/agent-detail/monitor/summary-card.tsx index 0acdaa13..31bb8312 100644 --- a/ui/src/core/page-components/agent-detail/monitor/summary-card.tsx +++ b/ui/src/core/page-components/agent-detail/monitor/summary-card.tsx @@ -317,17 +317,6 @@ export function SummaryCard({ size={130} thickness={14} sections={[ - { - value: - summary.actionCounts?.allow !== undefined - ? (summary.actionCounts.allow / - summary.totalMatches) * - 100 * - ringAnimationProgress - : 0, - color: 'var(--mantine-color-green-4)', - tooltip: `Allow: ${summary.actionCounts?.allow || 0}`, - }, { value: summary.actionCounts?.deny !== undefined @@ -352,25 +341,14 @@ export function SummaryCard({ }, { value: - summary.actionCounts?.warn !== undefined - ? (summary.actionCounts.warn / - summary.totalMatches) * - 100 * - ringAnimationProgress - : 0, - color: 'var(--mantine-color-yellow-4)', - tooltip: `Warn: ${summary.actionCounts?.warn || 0}`, - }, - { - value: - summary.actionCounts?.log !== undefined - ? (summary.actionCounts.log / + summary.actionCounts?.observe !== undefined + ? (summary.actionCounts.observe / summary.totalMatches) * 100 * ringAnimationProgress : 0, color: 'var(--mantine-color-blue-4)', - tooltip: `Log: ${summary.actionCounts?.log || 0}`, + tooltip: `Observe: ${summary.actionCounts?.observe || 0}`, }, ]} label={ @@ -388,11 +366,9 @@ export function SummaryCard({ {/* Legend - horizontal layout */} {[ - { key: 'allow', label: 'Allow', color: 'green' }, { key: 'deny', label: 'Deny', color: 'red' }, { key: 'steer', label: 'Steer', color: 'cyan' }, - { key: 'warn', label: 'Warn', color: 'yellow' }, - { key: 'log', label: 'Log', color: 'blue' }, + { key: 'observe', label: 'Observe', color: 'blue' }, ].map(({ key, label, color }) => { const count = summary.actionCounts?.[key] ?? 0; if (count === 0) return null; diff --git a/ui/tests/agent-stats.spec.ts b/ui/tests/agent-stats.spec.ts index d5d69c3f..4ef071bf 100644 --- a/ui/tests/agent-stats.spec.ts +++ b/ui/tests/agent-stats.spec.ts @@ -69,10 +69,11 @@ test.describe('Agent Monitor Tab', () => { await expect(mockedPage.getByText('Actions Distribution')).toBeVisible(); // Check action types are displayed (use exact match to avoid matching badges) - await expect(mockedPage.getByText('Allow', { exact: true })).toBeVisible(); + await expect( + mockedPage.getByText('Observe', { exact: true }) + ).toBeVisible(); await expect(mockedPage.getByText('Deny', { exact: true })).toBeVisible(); - await expect(mockedPage.getByText('Warn', { exact: true })).toBeVisible(); - await expect(mockedPage.getByText('Log', { exact: true })).toBeVisible(); + await expect(mockedPage.getByText('Steer', { exact: true })).toHaveCount(0); }); test('should display per-control statistics table', async ({ @@ -189,11 +190,9 @@ test.describe('Agent Monitor Tab - Refetch Flow', () => { execution_count: 100, match_count: 10, non_match_count: 90, - allow_count: 5, + observe_count: 5, deny_count: 5, steer_count: 0, - warn_count: 0, - log_count: 0, error_count: 0, avg_confidence: 0.85, avg_duration_ms: 40, @@ -216,11 +215,9 @@ test.describe('Agent Monitor Tab - Refetch Flow', () => { execution_count: 250, match_count: 35, non_match_count: 215, - allow_count: 15, + observe_count: 15, deny_count: 20, steer_count: 0, - warn_count: 0, - log_count: 0, error_count: 1, avg_confidence: 0.91, avg_duration_ms: 38, diff --git a/ui/tests/fixtures.ts b/ui/tests/fixtures.ts index 9f566844..a46fdd19 100644 --- a/ui/tests/fixtures.ts +++ b/ui/tests/fixtures.ts @@ -138,7 +138,7 @@ const controlsList: Control[] = [ config: { values: [], logic: 'any', match_on: 'match' }, }, }, - action: { decision: 'allow' }, + action: { decision: 'observe' }, tags: [], }, }, @@ -282,11 +282,9 @@ const statsResponse: StatsResponse = { non_match_count: 390, error_count: 2, action_counts: { - allow: 10, + observe: 15, deny: 25, steer: 0, - warn: 3, - log: 2, }, }, controls: [ @@ -296,11 +294,9 @@ const statsResponse: StatsResponse = { execution_count: 150, match_count: 25, non_match_count: 125, - allow_count: 5, + observe_count: 10, deny_count: 15, steer_count: 0, - warn_count: 3, - log_count: 2, error_count: 0, avg_confidence: 0.92, avg_duration_ms: 45, @@ -311,11 +307,9 @@ const statsResponse: StatsResponse = { execution_count: 80, match_count: 10, non_match_count: 70, - allow_count: 0, + observe_count: 0, deny_count: 10, steer_count: 0, - warn_count: 0, - log_count: 0, error_count: 2, avg_confidence: 0.88, avg_duration_ms: 32, @@ -326,11 +320,9 @@ const statsResponse: StatsResponse = { execution_count: 200, match_count: 5, non_match_count: 195, - allow_count: 5, + observe_count: 5, deny_count: 0, steer_count: 0, - warn_count: 0, - log_count: 0, error_count: 0, avg_confidence: 0.95, avg_duration_ms: 12,