Merge branch 'main' into feature/SAO-17535-add-LLMAJ-evaluator2

Author: wrisa

CHANGELOG.md

@@ -2,6 +2,30 @@
 
 <!-- version list -->
 
+## v8.12.0 (2026-10-09)
+
+### Features
+
+- **engine**: Capture per-control execution duration [SAO-18023]
+  ([#283](https://github.com/agentcontrol/agent-control/pull/283),
+  [`d1b9d21`](https://github.com/agentcontrol/agent-control/commit/d1b9d21f900129a33661f0f1449f989166da27ce))
+
+- **evaluators**: 17516 request payload contract
+  ([#271](https://github.com/agentcontrol/agent-control/pull/271),
+  [`1f07d7c`](https://github.com/agentcontrol/agent-control/commit/1f07d7cee069c5515c384d2fa794dfbaa8fcdce0))
+
+- **examples**: Add TypeSafe Jev evaluator example
+  ([#279](https://github.com/agentcontrol/agent-control/pull/279),
+  [`095edf0`](https://github.com/agentcontrol/agent-control/commit/095edf0321e7c2ed46f16cffcfb66bfcbd79f2ed))
+
+- **sdk**: Add Step.children ([#282](https://github.com/agentcontrol/agent-control/pull/282),
+  [`86fbbfe`](https://github.com/agentcontrol/agent-control/commit/86fbbfe42bf95a332c100fa29e3d7056ea7f9ad4))
+
+- **server**: Report sanitized diagnostics on upstream 4xx rejections [SAO-17523]
+  ([#270](https://github.com/agentcontrol/agent-control/pull/270),
+  [`db9c667`](https://github.com/agentcontrol/agent-control/commit/db9c6676f058d91b08f761626cdad5113696df92))
+
+
 ## v8.11.0 (2026-10-02)
 
 ### Features

engine/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-engine"
-version = "8.11.0"
+version = "8.12.0"
 description = "Control execution engine for Agent Control"
 requires-python = ">=3.12"
 dependencies = [

evaluators/builtin/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-evaluators"
-version = "8.11.0"
+version = "8.12.0"
 description = "Builtin evaluators for agent-control"
 readme = "README.md"
 requires-python = ">=3.12"

evaluators/contrib/budget/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-evaluator-budget"
-version = "8.11.0"
+version = "8.12.0"
 description = "Budget evaluator for agent-control -- cumulative LLM cost and token tracking"
 readme = "README.md"
 requires-python = ">=3.12"

evaluators/contrib/cisco/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-evaluator-cisco"
-version = "8.11.0"
+version = "8.12.0"
 description = "Cisco AI Defense evaluator for agent-control"
 readme = "README.md"
 requires-python = ">=3.12"

evaluators/contrib/defenseclaw/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-evaluator-defenseclaw"
-version = "8.11.0"
+version = "8.12.0"
 description = "DefenseClaw evaluators for agent-control"
 readme = "README.md"
 requires-python = ">=3.12"

evaluators/contrib/galileo/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-control-evaluator-galileo"
-version = "8.11.0"
+version = "8.12.0"
 description = "Galileo evaluators (Luna, LLM-as-judge) for agent-control"
 readme = "README.md"
 requires-python = ">=3.12"

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py

@@ -19,7 +19,7 @@
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
 
 from ..records import UnsupportedStepTypeError, record_from_step
-from .config import ScorerInvokeConfig
+from .config import ScorerInvokeConfig, scorer_invoke_runtime_enabled
 
 logger = logging.getLogger(__name__)
 
@@ -367,6 +367,8 @@ class GalileoLunaClient:
     Environment Variables:
         GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth.
         GALILEO_LUNA_INVOKE_URL: Luna scorer invoke URL or service root (required).
+        GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME: Set to ``enabled`` to send
+            structured runtime data.
         GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify Luna invoke TLS.
         AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback.
         GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry.
@@ -414,6 +416,7 @@ def __init__(
             )
 
         self.api_secret = resolved_api_secret
+        self.runtime_enabled = scorer_invoke_runtime_enabled()
         self.luna_invoke_url = _normalize_luna_invoke_url(resolved_luna_invoke_url)
         self.luna_invoke_ca_file = (
             luna_invoke_ca_file
@@ -559,7 +562,7 @@ async def invoke(
                 selected_data=selected_data,
                 payload_field=selected_data_payload_field,
             )
-            if step is not None and scorer_version_id is not None
+            if self.runtime_enabled and step is not None and scorer_version_id is not None
             else None
         )
         request_body = ScorerInvokeRequest(
@@ -573,7 +576,7 @@ async def invoke(
                 tools=step.tools if step is not None else None,
             ),
             record=record,
-            execution_context=execution_context,
+            execution_context=execution_context if self.runtime_enabled else None,
             config=invoke_config,
         ).to_dict()
 

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import Literal
 
 from agent_control_evaluators import EvaluatorConfig
@@ -12,6 +13,12 @@
 LunaPayloadField = Literal["input", "output"]
 
 _NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"})
+SCORER_INVOKE_RUNTIME_FLAG_ENV = "GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME"
+
+
+def scorer_invoke_runtime_enabled() -> bool:
+    """Return whether structured runtime data is enabled for Luna requests."""
+    return os.getenv(SCORER_INVOKE_RUNTIME_FLAG_ENV) == "enabled"
 
 
 class ScorerInvokeConfig(BaseModel):

evaluators/contrib/galileo/tests/test_luna_evaluator.py

@@ -17,6 +17,11 @@
 LUNA_ENV = {
     "GALILEO_API_SECRET_KEY": "test-secret",
     "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
+    "GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME": "enabled",
+}
+LEGACY_LUNA_ENV = {
+    "GALILEO_API_SECRET_KEY": "test-secret",
+    "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
 }
 SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4"
 SCORER_VERSION_ID = "07fb9c96-9752-4cf5-a253-1a396100e9d2"
@@ -885,6 +890,60 @@ def handler(request: httpx.Request) -> httpx.Response:
             "config": {"request_timeout_seconds": 8.0},
         }
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("flag_value", [None, "disabled", "Enabled"])
+    async def test_client_uses_legacy_request_without_runtime_flag(
+        self, flag_value: str | None
+    ) -> None:
+        from agent_control_evaluator_galileo.luna import GalileoLunaClient
+        from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext
+
+        captured: dict[str, object] = {}
+
+        def handler(request: httpx.Request) -> httpx.Response:
+            captured["body"] = json.loads(request.content.decode())
+            return httpx.Response(200, json={"score": 0.8, "status": "success"})
+
+        step = Step(
+            type="llm",
+            name="answer",
+            input="step input",
+            output="step output",
+            ground_truth={"expected": "answer"},
+            tools=[{"name": "search", "input_schema": {}}],
+        )
+        env = LEGACY_LUNA_ENV.copy()
+        if flag_value is not None:
+            env["GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME"] = flag_value
+        with patch.dict(os.environ, env, clear=True):
+            client = GalileoLunaClient()
+        client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+
+        try:
+            await client.invoke(
+                scorer_id=SCORER_ID,
+                scorer_version_id=SCORER_VERSION_ID,
+                input="selected input",
+                output="selected output",
+                step=step,
+                execution_context=GalileoExecutionContext(organization_id="org-1"),
+            )
+        finally:
+            await client.close()
+
+        body = captured["body"]
+        assert isinstance(body, dict)
+        assert body["scorer_id"] == SCORER_ID
+        assert body["scorer_version_id"] == SCORER_VERSION_ID
+        assert body["inputs"] == {
+            "query": "selected input",
+            "response": "selected output",
+            "ground_truth": {"expected": "answer"},
+            "tools": [{"name": "search", "input_schema": {}}],
+        }
+        assert "record" not in body
+        assert "execution_context" not in body
+
     @pytest.mark.asyncio
     async def test_client_omits_record_when_step_has_no_pinned_version(self) -> None:
         from agent_control_evaluator_galileo.luna import GalileoLunaClient