feat(evaluators): add AgentControl support for Orbit preset SLM scorer invocation (#259)

## Summary

- Adds AgentControl support for Orbit preset SLM scorer invocation and
is paired with rungalileo/orbit#1720.
- Requires `scorer_id`; `scorer_label` is optional display metadata and
`scorer_version_id` is a deprecated compatibility identifier.
- Calls `/api/v1/scorers/invoke` with a scoped internal HS256 JWT
(`scorers.invoke`).
- Dual-writes legacy `inputs` and a structured `record` during the
rollout, including selector-aligned input/output plus Step context,
tools, and ground truth.
- Sends an Orbit execution timeout below the AgentControl HTTP deadline
(8 seconds for the default 10-second deadline).

## Scope

- Orbit owns feature-flag routing between RuntimeScorer and the legacy
adapter.
- Nine flat preset SLM scorers are validated for direct runtime
execution.
- Existing Step context propagation through models, engine,
integrations, and SDKs is retained.
- Broader preset SLM runtime coverage is deferred to Jira phases 2 and
3.
- Customer-owned Luna scorers are not supported in this release.
- No backend selector, Protect payload construction, Wizard
configuration, trusted tenant identity, or Galileo record-tree
construction is added.

## Risk and Rollout

- Risk level: medium.
- Deploy Orbit #1720 and keep its legacy adapter available for the
tandem rollout.
- Roll back AgentControl to the legacy `inputs.query`/`inputs.response`
contract if needed.
- Retain dual-write until older supported Orbit versions are retired.

## Testing

- [x] Galileo evaluator tests (116 passed)
- [x] Models tests (71 passed)
- [x] Engine tests (117 passed)
- [x] Affected Python SDK tests (103 passed)
- [x] Built-in evaluator tests (310 passed)
- [x] TypeScript SDK tests, lint, and typecheck
- [x] Repository lint, mypy, and `git diff --check`
- [ ] Full Python SDK target requires local PostgreSQL on
`localhost:5432`

## Follow-up

Orbit #1720 must update all six AgentControl git references to commit
`8a2b5d52717090ed7c65b627035f73d89c46f5ff` and regenerate requirements
before the tandem merge.

Author: namrataghadi-galileo

engine/pyproject.toml

@@ -4,8 +4,8 @@ version = "8.6.0"
 description = "Control execution engine for Agent Control"
 requires-python = ">=3.12"
 dependencies = [
-    "agent-control-models>=3.0.0",
-    "agent-control-evaluators>=3.0.0",
+    "agent-control-models>=8.6.0",
+    "agent-control-evaluators>=8.6.0",
     "google-re2>=1.1",
 ]
 authors = [

engine/src/agent_control_engine/core.py

@@ -320,7 +320,7 @@ async def _evaluate_leaf(
                     timeout = DEFAULT_EVALUATOR_TIMEOUT
 
                 result = await asyncio.wait_for(
-                    evaluator.evaluate(data),
+                    evaluator.evaluate_with_context(data, request.step),
                     timeout=timeout,
                 )
         except TimeoutError:

engine/tests/test_core.py

@@ -39,13 +39,15 @@ class SimpleConfig(BaseModel):
 # Shared state for coordination between test evaluators
 _execution_log: list[str] = []
 _blocker_event: asyncio.Event | None = None
+_context_calls: list[tuple[Any, Step]] = []
 
 
 def reset_test_state() -> None:
     """Reset shared test state."""
-    global _execution_log, _blocker_event
+    global _execution_log, _blocker_event, _context_calls
     _execution_log = []
     _blocker_event = asyncio.Event()
+    _context_calls = []
 
 
 class AllowEvaluator(Evaluator[SimpleConfig]):
@@ -163,6 +165,24 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         return result
 
 
+class ContextEvaluator(Evaluator[SimpleConfig]):
+    """Evaluator that records selector data and full request context."""
+
+    metadata = EvaluatorMetadata(
+        name="test-context",
+        version="1.0.0",
+        description="Records contextual calls",
+    )
+    config_model = SimpleConfig
+
+    async def evaluate(self, data: Any) -> EvaluatorResult:
+        raise AssertionError("engine should call evaluate_with_context")
+
+    async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult:
+        _context_calls.append((data, step))
+        return EvaluatorResult(matched=False, confidence=1.0, message="context received")
+
+
 @dataclass
 class MockControlWithIdentity:
     """Mock control for testing."""
@@ -185,6 +205,7 @@ def setup_test_evaluators():
         BlockerEvaluator,
         SlowEvaluator,
         MetadataEvaluator,
+        ContextEvaluator,
     ]:
         try:
             register_evaluator(evaluator_cls)
@@ -253,6 +274,69 @@ def make_control(
     )
 
 
+@pytest.mark.asyncio
+async def test_context_evaluator_receives_selected_data_and_complete_step() -> None:
+    # Given: a selector targeting output and a step with structured context
+    engine = ControlEngine(
+        [make_control(1, "context", "test-context", action="observe", path="output")]
+    )
+    step = Step(
+        type="llm",
+        name="test-step",
+        input="question",
+        output="answer",
+        context={"conversation_id": "c-1"},
+        tools=[{"name": "search", "description": "Search", "input_schema": {}}],
+        ground_truth="expected answer",
+    )
+
+    # When: evaluating the request through the engine
+    await engine.process(
+        EvaluationRequest(
+            agent_name="00000000-0000-0000-0000-000000000001",
+            step=step,
+            stage="pre",
+        )
+    )
+
+    # Then: selector behavior is unchanged and the complete Step is separate
+    assert _context_calls == [("answer", step)]
+
+
+@pytest.mark.asyncio
+async def test_cached_context_evaluator_handles_concurrent_steps_without_retaining_state() -> None:
+    # Given: one cached evaluator configuration and two independent requests
+    engine = ControlEngine(
+        [make_control(1, "context", "test-context", action="observe", path="input")]
+    )
+    first = Step(type="llm", name="test-step", input="first", ground_truth="one")
+    second = Step(type="llm", name="test-step", input="second", ground_truth="two")
+
+    # When: the requests are evaluated concurrently
+    await asyncio.gather(
+        engine.process(
+            EvaluationRequest(
+                agent_name="00000000-0000-0000-0000-000000000001",
+                step=first,
+                stage="pre",
+            )
+        ),
+        engine.process(
+            EvaluationRequest(
+                agent_name="00000000-0000-0000-0000-000000000001",
+                step=second,
+                stage="pre",
+            )
+        ),
+    )
+
+    # Then: each selected value remains paired with its own full Step
+    assert {(data, step.ground_truth) for data, step in _context_calls} == {
+        ("first", "one"),
+        ("second", "two"),
+    }
+
+
 # =============================================================================
 # Test: Parallel Execution
 # =============================================================================

evaluators/builtin/pyproject.toml

@@ -7,15 +7,15 @@ requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
 authors = [{ name = "Agent Control Team" }]
 dependencies = [
-    "agent-control-models>=7.5.0",
+    "agent-control-models>=8.6.0",
     "pydantic>=2.12.4",
     "google-re2>=1.1",
     "jsonschema>=4.0.0",
     "sqlglot[c]>=30.11.0,<30.12.0",
 ]
 
 [project.optional-dependencies]
-galileo = ["agent-control-evaluator-galileo>=7.5.0"]
+galileo = ["agent-control-evaluator-galileo>=8.6.0"]
 budget = ["agent-control-evaluator-budget>=7.5.0"]
 cisco = ["agent-control-evaluator-cisco>=7.5.0"]
 defenseclaw = ["agent-control-evaluator-defenseclaw>=8.2.0"]

evaluators/builtin/src/agent_control_evaluators/_base.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
 
-from agent_control_models import EvaluatorResult
+from agent_control_models import EvaluatorResult, Step
 from agent_control_models.base import BaseModel
 
 if TYPE_CHECKING:
@@ -161,6 +161,23 @@ async def evaluate(self, data: Any) -> EvaluatorResult:
         """
         pass
 
+    async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult:
+        """Evaluate selected data with access to the complete runtime step.
+
+        The default implementation preserves compatibility with evaluators that
+        implement only :meth:`evaluate`. Evaluators must treat ``step`` as
+        immutable request-scoped context because evaluator instances are cached
+        and may be invoked concurrently.
+
+        Args:
+            data: Data extracted by the configured selector.
+            step: Complete runtime step for the current request.
+
+        Returns:
+            EvaluatorResult produced by this evaluator.
+        """
+        return await self.evaluate(data)
+
     def get_timeout_seconds(self) -> float:
         """Get timeout in seconds from config or metadata default."""
         timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms)

evaluators/builtin/tests/test_base.py

@@ -3,11 +3,12 @@
 Architecture: Evaluators take config at __init__, evaluate() only takes data.
 """
 
-import pytest
 from typing import Any
 
+import pytest
+
 from agent_control_evaluators import Evaluator, EvaluatorConfig, EvaluatorMetadata
-from agent_control_models import EvaluatorResult
+from agent_control_models import EvaluatorResult, Step
 
 
 class MockConfig(EvaluatorConfig):
@@ -106,6 +107,20 @@ async def test_mock_evaluator_evaluate_no_match(self):
 
         assert result.matched is False
 
+    @pytest.mark.asyncio
+    async def test_contextual_evaluation_delegates_to_existing_evaluate(self):
+        """Existing evaluators receive selected data through the default hook."""
+        # Given: an evaluator that implements only evaluate(data)
+        evaluator = MockEvaluator.from_dict({"should_match": True})
+        step = Step(type="llm", name="answer", input="full input")
+
+        # When: the engine-facing contextual hook is called
+        result = await evaluator.evaluate_with_context("selected data", step)
+
+        # Then: the legacy evaluate implementation handles the selected data
+        assert result.matched is True
+        assert result.metadata == {"data": "selected data"}
+
     def test_evaluator_config_stored(self):
         """Test that evaluator stores config."""
         evaluator = MockEvaluator.from_dict({"should_match": True})

evaluators/contrib/galileo/README.md

@@ -7,13 +7,24 @@ Integration package for Galileo Luna evaluator.
 The `galileo.luna2` evaluator ID has been removed. Existing controls that use
 `galileo.luna2` should migrate to `galileo.luna` and update their evaluator
 configuration to use the direct Luna scorer fields. `scorer_id` is required;
-`scorer_label` and `scorer_version_id` are optional. The evaluator calls the
+`scorer_label` and `scorer_version_id` are optional. `scorer_version_id` is a
+deprecated optional compatibility identifier; Orbit currently invokes the
+scorer's current default version. The evaluator calls the
 URL configured by `GALILEO_LUNA_INVOKE_URL`; the target must support the Luna
 scorer invoke request/response contract and internal Galileo secret auth. Also
 set `threshold` and `operator` as needed. If you still need the legacy Luna2
 evaluator, pin
 `agent-control-evaluator-galileo <8`.
 
+The optional evaluator `config` mirrors Orbit's allowlisted scorer-invoke
+configuration. Supported keys are `threshold`, `score_threshold`, and
+`request_timeout_seconds`; unsupported keys are rejected locally before an HTTP
+request is made.
+
+Agent Control always sends an Orbit execution timeout shorter than its HTTP
+deadline. When `request_timeout_seconds` is omitted, it defaults to 80% of the
+configured HTTP timeout (8 seconds for the default 10-second deadline).
+
 ## Install
 
 Canonical install path:

evaluators/contrib/galileo/pyproject.toml

@@ -7,8 +7,8 @@ requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
 authors = [{ name = "Agent Control Team" }]
 dependencies = [
-    "agent-control-evaluators>=7.5.0",
-    "agent-control-models>=7.5.0",
+    "agent-control-evaluators>=8.6.0",
+    "agent-control-models>=8.6.0",
     "httpx>=0.24.0",
     "pydantic>=2.12.4",
 ]

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py

@@ -25,13 +25,17 @@
     LunaEvaluator,
     LunaEvaluatorConfig,
     LunaOperator,
+    ScorerInvokeConfig,
+    ScorerInvokeRecord,
     ScorerInvokeRequest,
     ScorerInvokeResponse,
 )
 
 __all__ = [
     "GalileoLunaClient",
     "ScorerInvokeRequest",
+    "ScorerInvokeConfig",
+    "ScorerInvokeRecord",
     "ScorerInvokeResponse",
     "LunaEvaluator",
     "LunaEvaluatorConfig",

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py

@@ -3,15 +3,22 @@
 from agent_control_evaluator_galileo.luna.client import (
     GalileoLunaClient,
     ScorerInvokeInputs,
+    ScorerInvokeRecord,
     ScorerInvokeRequest,
     ScorerInvokeResponse,
 )
-from agent_control_evaluator_galileo.luna.config import LunaEvaluatorConfig, LunaOperator
+from agent_control_evaluator_galileo.luna.config import (
+    LunaEvaluatorConfig,
+    LunaOperator,
+    ScorerInvokeConfig,
+)
 from agent_control_evaluator_galileo.luna.evaluator import LUNA_AVAILABLE, LunaEvaluator
 
 __all__ = [
     "GalileoLunaClient",
     "ScorerInvokeInputs",
+    "ScorerInvokeConfig",
+    "ScorerInvokeRecord",
     "ScorerInvokeRequest",
     "ScorerInvokeResponse",
     "LunaEvaluatorConfig",