Skip to content

Commit 875873d

Browse files
committed
Merge branch 'main' into feature/SAO-16858-add-evaluator-for-LLM
2 parents 0d2596f + 15fc749 commit 875873d

15 files changed

Lines changed: 145 additions & 47 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
<!-- version list -->
44

5+
## v8.11.0 (2026-10-02)
6+
7+
### Features
8+
9+
- **evaluators**: Send authenticated execution context to runners api
10+
([#278](https://github.com/agentcontrol/agent-control/pull/278),
11+
[`60814a8`](https://github.com/agentcontrol/agent-control/commit/60814a8d839cb5c50262e87ff9196e41b6e7d69a))
12+
13+
514
## v8.10.0 (2026-10-01)
615

716
### Features

‎engine/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-engine"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "Control execution engine for Agent Control"
55
requires-python = ">=3.12"
66
dependencies = [

‎evaluators/builtin/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-evaluators"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "Builtin evaluators for agent-control"
55
readme = "README.md"
66
requires-python = ">=3.12"

‎evaluators/contrib/budget/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-evaluator-budget"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "Budget evaluator for agent-control -- cumulative LLM cost and token tracking"
55
readme = "README.md"
66
requires-python = ">=3.12"

‎evaluators/contrib/cisco/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-evaluator-cisco"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "Cisco AI Defense evaluator for agent-control"
55
readme = "README.md"
66
requires-python = ">=3.12"

‎evaluators/contrib/defenseclaw/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-evaluator-defenseclaw"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "DefenseClaw evaluators for agent-control"
55
readme = "README.md"
66
requires-python = ">=3.12"

‎evaluators/contrib/galileo/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "agent-control-evaluator-galileo"
3-
version = "8.10.0"
3+
version = "8.11.0"
44
description = "Galileo evaluators (Luna, LLM-as-judge) for agent-control"
55
readme = "README.md"
66
requires-python = ">=3.12"

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -229,12 +229,12 @@ class ScorerInvokeRecord(BaseModel):
229229

230230

231231
class GalileoExecutionContext(BaseModel):
232-
"""Authenticated identity fields required by Galileo scorer invocation."""
232+
"""Authenticated organization and optional identity fields for Galileo scorer invocation."""
233233

234234
organization_id: str = Field(min_length=1)
235-
user_id: str = Field(min_length=1)
236-
project_id: str = Field(min_length=1)
237-
run_id: str = Field(min_length=1)
235+
user_id: str | None = Field(default=None, min_length=1)
236+
project_id: str | None = Field(default=None, min_length=1)
237+
run_id: str | None = Field(default=None, min_length=1)
238238

239239

240240
class ScorerInvokeRequest(BaseModel):
@@ -266,7 +266,10 @@ def ensure_required_values(self) -> ScorerInvokeRequest:
266266

267267
def to_dict(self) -> JSONObject:
268268
"""Convert to the Luna scorer invoke request shape."""
269-
return self.model_dump(mode="json", exclude_none=True)
269+
request = self.model_dump(mode="json", exclude_none=True)
270+
if self.execution_context is not None:
271+
request["execution_context"] = self.execution_context.model_dump(mode="json")
272+
return request
270273

271274

272275
def _orbit_record_from_step(

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py‎

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -236,38 +236,26 @@ def _execution_context_from_extensions(
236236
metadata = extensions.get("metadata")
237237
metadata_obj = metadata if isinstance(metadata, dict) else {}
238238
organization_id = extensions.get("namespace_key")
239-
# Orbit's runtime-auth contract defines caller_id as the authenticated
240-
# user ID for this flow. Keep that contract mapping inside Galileo.
239+
# The runtime envelope's caller_id is supplied by the authenticated
240+
# principal and is the caller identity Galileo associates with this run.
241241
user_id = extensions.get("caller_id")
242242
project_id = metadata_obj.get("project_id")
243243
target_type = extensions.get("target_type")
244244
target_id = extensions.get("target_id")
245-
run_id = target_id if target_type == "log_stream" else None
246-
247-
missing = [
248-
field
249-
for field, value in (
250-
("organization_id", organization_id),
251-
("user_id", user_id),
252-
("project_id", project_id),
253-
("run_id", run_id),
254-
)
255-
if not isinstance(value, str) or not value
256-
]
257-
if missing:
258-
raise ValueError(
259-
"Authenticated execution metadata is missing required fields: "
260-
+ ", ".join(missing)
261-
)
245+
run_id = (
246+
target_id
247+
if target_type in ("log_stream", "agent_stream")
248+
and isinstance(target_id, str)
249+
and target_id
250+
else None
251+
)
252+
if not isinstance(organization_id, str) or not organization_id:
253+
raise ValueError("Authenticated execution metadata is missing organization_id")
262254

263-
assert isinstance(organization_id, str)
264-
assert isinstance(user_id, str)
265-
assert isinstance(project_id, str)
266-
assert isinstance(run_id, str)
267255
return GalileoExecutionContext(
268256
organization_id=organization_id,
269-
user_id=user_id,
270-
project_id=project_id,
257+
user_id=user_id if isinstance(user_id, str) and user_id else None,
258+
project_id=project_id if isinstance(project_id, str) and project_id else None,
271259
run_id=run_id,
272260
)
273261

‎evaluators/contrib/galileo/tests/test_luna_evaluator.py‎

Lines changed: 104 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,23 @@ def test_scorer_invoke_request_serialization_keeps_legacy_record_shape(self) ->
246246
"config": {},
247247
}
248248

249+
def test_execution_context_serializes_missing_optional_claims_as_null(self) -> None:
250+
from agent_control_evaluator_galileo.luna import ScorerInvokeInputs, ScorerInvokeRequest
251+
from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext
252+
253+
request = ScorerInvokeRequest(
254+
scorer_id="scorer-123",
255+
inputs=ScorerInvokeInputs(query="hello"),
256+
execution_context=GalileoExecutionContext(organization_id="org-1"),
257+
)
258+
259+
assert request.to_dict()["execution_context"] == {
260+
"organization_id": "org-1",
261+
"user_id": None,
262+
"project_id": None,
263+
"run_id": None,
264+
}
265+
249266
def test_scorer_invoke_request_rejects_unknown_config(self) -> None:
250267
from agent_control_evaluator_galileo.luna import (
251268
ScorerInvokeInputs,
@@ -1064,7 +1081,8 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None:
10641081

10651082
@patch.dict(os.environ, LUNA_ENV)
10661083
@pytest.mark.asyncio
1067-
async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
1084+
@pytest.mark.parametrize("target_type", ["log_stream", "agent_stream"])
1085+
async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: str) -> None:
10681086
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
10691087
from agent_control_evaluator_galileo.luna.client import (
10701088
GalileoExecutionContext,
@@ -1077,7 +1095,7 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
10771095
extensions = {
10781096
"namespace_key": "org-1",
10791097
"caller_id": "verified-user-2",
1080-
"target_type": "log_stream",
1098+
"target_type": target_type,
10811099
"target_id": "run-4",
10821100
"metadata": {"project_id": "project-3"},
10831101
}
@@ -1106,25 +1124,105 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
11061124

11071125
@patch.dict(os.environ, LUNA_ENV)
11081126
@pytest.mark.asyncio
1109-
async def test_evaluator_requires_authenticated_caller_id(self) -> None:
1110-
from agent_control_evaluator_galileo.luna import LunaEvaluator
1111-
from agent_control_evaluator_galileo.luna.client import GalileoLunaClient
1127+
async def test_evaluator_allows_missing_caller_id(self) -> None:
1128+
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
1129+
from agent_control_evaluator_galileo.luna.client import (
1130+
GalileoExecutionContext,
1131+
GalileoLunaClient,
1132+
)
11121133

11131134
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
11141135
extensions = {
11151136
"namespace_key": "org-1",
11161137
"target_type": "log_stream",
11171138
"target_id": "run-4",
1139+
"metadata": {},
1140+
}
1141+
1142+
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1143+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
1144+
result = await evaluator.evaluate_with_extensions(
1145+
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
1146+
)
1147+
1148+
assert result.error is None
1149+
mock_invoke.assert_awaited_once_with(
1150+
scorer_id="scorer-123",
1151+
step=Step(type="llm", name="answer", input="prompt"),
1152+
execution_context=GalileoExecutionContext(
1153+
organization_id="org-1",
1154+
user_id=None,
1155+
project_id=None,
1156+
run_id="run-4",
1157+
),
1158+
input="selected input",
1159+
output=None,
1160+
config=None,
1161+
timeout=10.0,
1162+
)
1163+
1164+
@patch.dict(os.environ, LUNA_ENV)
1165+
@pytest.mark.asyncio
1166+
async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None:
1167+
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
1168+
from agent_control_evaluator_galileo.luna.client import (
1169+
GalileoExecutionContext,
1170+
GalileoLunaClient,
1171+
)
1172+
1173+
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
1174+
extensions = {
1175+
"namespace_key": "org-1",
1176+
"caller_id": "verified-user-2",
1177+
"target_type": "environment",
1178+
"target_id": "prod",
11181179
"metadata": {"project_id": "project-3"},
11191180
}
11201181

11211182
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1183+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
11221184
result = await evaluator.evaluate_with_extensions(
11231185
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
11241186
)
11251187

1188+
assert result.error is None
1189+
mock_invoke.assert_awaited_once_with(
1190+
scorer_id="scorer-123",
1191+
step=Step(type="llm", name="answer", input="prompt"),
1192+
execution_context=GalileoExecutionContext(
1193+
organization_id="org-1",
1194+
user_id="verified-user-2",
1195+
project_id="project-3",
1196+
run_id=None,
1197+
),
1198+
input="selected input",
1199+
output=None,
1200+
config=None,
1201+
timeout=10.0,
1202+
)
1203+
1204+
@patch.dict(os.environ, LUNA_ENV)
1205+
@pytest.mark.asyncio
1206+
async def test_evaluator_rejects_missing_organization_id(self) -> None:
1207+
from agent_control_evaluator_galileo.luna import LunaEvaluator
1208+
from agent_control_evaluator_galileo.luna.client import GalileoLunaClient
1209+
1210+
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
1211+
extensions = {
1212+
"target_type": "log_stream",
1213+
"target_id": "run-4",
1214+
"metadata": {"project_id": "project-3"},
1215+
}
1216+
1217+
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1218+
result = await evaluator.evaluate_with_extensions(
1219+
"selected input",
1220+
Step(type="llm", name="answer", input="prompt"),
1221+
extensions,
1222+
)
1223+
11261224
assert result.error is not None
1127-
assert "user_id" in result.error
1225+
assert "organization_id" in result.error
11281226
mock_invoke.assert_not_called()
11291227

11301228
@patch.dict(os.environ, LUNA_ENV)

0 commit comments

Comments
 (0)