Skip to content

Commit 60814a8

Browse files
namrataghadi-galileonamrataghadi-galileo
authored
feat(evaluators): send authenticated execution context to runners api (#278)
## Summary - Add Galileo `execution_context` to the existing `POST /api/v1/scorers/invoke` request. - Map the authenticated Agent Control context to Galileo fields: `namespace_key` → `organization_id`, `caller_id` → `user_id`, trusted extension `project_id` → `project_id`, and supported target ID → `run_id`. - Emit optional context values as `null`; preserve legacy requests when no runtime extensions are present. ## Scope - User-facing/API changes: Galileo scorer invocation includes the additive `execution_context` object. - Internal changes: Context translation stays inside the Galileo evaluator package. `run_id` is set only for `log_stream` and `agent_stream` targets. - Out of scope: Generic Agent Control models/auth, scorer-grant flows, record serialization/factory, and record normalization. ## Risk and Rollout - Risk level: low - Rollback plan: Revert the Galileo package changes; callers without runtime context continue using the legacy request shape. ## Testing - [x] Added or updated automated tests - [x] Ran Galileo package tests: 172 passed - [x] Ran Galileo Ruff: all checks passed - [ ] Ran `make check` (package Makefile targets could not resolve `hatchling` because the configured package index returned HTTP 401; equivalent package-local test and Ruff commands passed from the existing virtualenv) - [ ] Manually verified behavior ## Checklist - [ ] Linked issue/spec (if applicable) - [ ] Updated docs/examples for user-facing changes - [ ] Included any required follow-up tasks
1 parent 097338b commit 60814a8

3 files changed

Lines changed: 125 additions & 36 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -229,12 +229,12 @@ class ScorerInvokeRecord(BaseModel):
229229

230230

231231
class GalileoExecutionContext(BaseModel):
232-
"""Authenticated identity fields required by Galileo scorer invocation."""
232+
"""Authenticated organization and optional identity fields for Galileo scorer invocation."""
233233

234234
organization_id: str = Field(min_length=1)
235-
user_id: str = Field(min_length=1)
236-
project_id: str = Field(min_length=1)
237-
run_id: str = Field(min_length=1)
235+
user_id: str | None = Field(default=None, min_length=1)
236+
project_id: str | None = Field(default=None, min_length=1)
237+
run_id: str | None = Field(default=None, min_length=1)
238238

239239

240240
class ScorerInvokeRequest(BaseModel):
@@ -266,7 +266,10 @@ def ensure_required_values(self) -> ScorerInvokeRequest:
266266

267267
def to_dict(self) -> JSONObject:
268268
"""Convert to the Luna scorer invoke request shape."""
269-
return self.model_dump(mode="json", exclude_none=True)
269+
request = self.model_dump(mode="json", exclude_none=True)
270+
if self.execution_context is not None:
271+
request["execution_context"] = self.execution_context.model_dump(mode="json")
272+
return request
270273

271274

272275
def _orbit_record_from_step(

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py‎

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -236,38 +236,26 @@ def _execution_context_from_extensions(
236236
metadata = extensions.get("metadata")
237237
metadata_obj = metadata if isinstance(metadata, dict) else {}
238238
organization_id = extensions.get("namespace_key")
239-
# Orbit's runtime-auth contract defines caller_id as the authenticated
240-
# user ID for this flow. Keep that contract mapping inside Galileo.
239+
# The runtime envelope's caller_id is supplied by the authenticated
240+
# principal and is the caller identity Galileo associates with this run.
241241
user_id = extensions.get("caller_id")
242242
project_id = metadata_obj.get("project_id")
243243
target_type = extensions.get("target_type")
244244
target_id = extensions.get("target_id")
245-
run_id = target_id if target_type == "log_stream" else None
246-
247-
missing = [
248-
field
249-
for field, value in (
250-
("organization_id", organization_id),
251-
("user_id", user_id),
252-
("project_id", project_id),
253-
("run_id", run_id),
254-
)
255-
if not isinstance(value, str) or not value
256-
]
257-
if missing:
258-
raise ValueError(
259-
"Authenticated execution metadata is missing required fields: "
260-
+ ", ".join(missing)
261-
)
245+
run_id = (
246+
target_id
247+
if target_type in ("log_stream", "agent_stream")
248+
and isinstance(target_id, str)
249+
and target_id
250+
else None
251+
)
252+
if not isinstance(organization_id, str) or not organization_id:
253+
raise ValueError("Authenticated execution metadata is missing organization_id")
262254

263-
assert isinstance(organization_id, str)
264-
assert isinstance(user_id, str)
265-
assert isinstance(project_id, str)
266-
assert isinstance(run_id, str)
267255
return GalileoExecutionContext(
268256
organization_id=organization_id,
269-
user_id=user_id,
270-
project_id=project_id,
257+
user_id=user_id if isinstance(user_id, str) and user_id else None,
258+
project_id=project_id if isinstance(project_id, str) and project_id else None,
271259
run_id=run_id,
272260
)
273261

‎evaluators/contrib/galileo/tests/test_luna_evaluator.py‎

Lines changed: 104 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,23 @@ def test_scorer_invoke_request_serialization_keeps_legacy_record_shape(self) ->
246246
"config": {},
247247
}
248248

249+
def test_execution_context_serializes_missing_optional_claims_as_null(self) -> None:
250+
from agent_control_evaluator_galileo.luna import ScorerInvokeInputs, ScorerInvokeRequest
251+
from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext
252+
253+
request = ScorerInvokeRequest(
254+
scorer_id="scorer-123",
255+
inputs=ScorerInvokeInputs(query="hello"),
256+
execution_context=GalileoExecutionContext(organization_id="org-1"),
257+
)
258+
259+
assert request.to_dict()["execution_context"] == {
260+
"organization_id": "org-1",
261+
"user_id": None,
262+
"project_id": None,
263+
"run_id": None,
264+
}
265+
249266
def test_scorer_invoke_request_rejects_unknown_config(self) -> None:
250267
from agent_control_evaluator_galileo.luna import (
251268
ScorerInvokeInputs,
@@ -1064,7 +1081,8 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None:
10641081

10651082
@patch.dict(os.environ, LUNA_ENV)
10661083
@pytest.mark.asyncio
1067-
async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
1084+
@pytest.mark.parametrize("target_type", ["log_stream", "agent_stream"])
1085+
async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: str) -> None:
10681086
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
10691087
from agent_control_evaluator_galileo.luna.client import (
10701088
GalileoExecutionContext,
@@ -1077,7 +1095,7 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
10771095
extensions = {
10781096
"namespace_key": "org-1",
10791097
"caller_id": "verified-user-2",
1080-
"target_type": "log_stream",
1098+
"target_type": target_type,
10811099
"target_id": "run-4",
10821100
"metadata": {"project_id": "project-3"},
10831101
}
@@ -1106,25 +1124,105 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None:
11061124

11071125
@patch.dict(os.environ, LUNA_ENV)
11081126
@pytest.mark.asyncio
1109-
async def test_evaluator_requires_authenticated_caller_id(self) -> None:
1110-
from agent_control_evaluator_galileo.luna import LunaEvaluator
1111-
from agent_control_evaluator_galileo.luna.client import GalileoLunaClient
1127+
async def test_evaluator_allows_missing_caller_id(self) -> None:
1128+
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
1129+
from agent_control_evaluator_galileo.luna.client import (
1130+
GalileoExecutionContext,
1131+
GalileoLunaClient,
1132+
)
11121133

11131134
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
11141135
extensions = {
11151136
"namespace_key": "org-1",
11161137
"target_type": "log_stream",
11171138
"target_id": "run-4",
1139+
"metadata": {},
1140+
}
1141+
1142+
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1143+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
1144+
result = await evaluator.evaluate_with_extensions(
1145+
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
1146+
)
1147+
1148+
assert result.error is None
1149+
mock_invoke.assert_awaited_once_with(
1150+
scorer_id="scorer-123",
1151+
step=Step(type="llm", name="answer", input="prompt"),
1152+
execution_context=GalileoExecutionContext(
1153+
organization_id="org-1",
1154+
user_id=None,
1155+
project_id=None,
1156+
run_id="run-4",
1157+
),
1158+
input="selected input",
1159+
output=None,
1160+
config=None,
1161+
timeout=10.0,
1162+
)
1163+
1164+
@patch.dict(os.environ, LUNA_ENV)
1165+
@pytest.mark.asyncio
1166+
async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None:
1167+
from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse
1168+
from agent_control_evaluator_galileo.luna.client import (
1169+
GalileoExecutionContext,
1170+
GalileoLunaClient,
1171+
)
1172+
1173+
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
1174+
extensions = {
1175+
"namespace_key": "org-1",
1176+
"caller_id": "verified-user-2",
1177+
"target_type": "environment",
1178+
"target_id": "prod",
11181179
"metadata": {"project_id": "project-3"},
11191180
}
11201181

11211182
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1183+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
11221184
result = await evaluator.evaluate_with_extensions(
11231185
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
11241186
)
11251187

1188+
assert result.error is None
1189+
mock_invoke.assert_awaited_once_with(
1190+
scorer_id="scorer-123",
1191+
step=Step(type="llm", name="answer", input="prompt"),
1192+
execution_context=GalileoExecutionContext(
1193+
organization_id="org-1",
1194+
user_id="verified-user-2",
1195+
project_id="project-3",
1196+
run_id=None,
1197+
),
1198+
input="selected input",
1199+
output=None,
1200+
config=None,
1201+
timeout=10.0,
1202+
)
1203+
1204+
@patch.dict(os.environ, LUNA_ENV)
1205+
@pytest.mark.asyncio
1206+
async def test_evaluator_rejects_missing_organization_id(self) -> None:
1207+
from agent_control_evaluator_galileo.luna import LunaEvaluator
1208+
from agent_control_evaluator_galileo.luna.client import GalileoLunaClient
1209+
1210+
evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"})
1211+
extensions = {
1212+
"target_type": "log_stream",
1213+
"target_id": "run-4",
1214+
"metadata": {"project_id": "project-3"},
1215+
}
1216+
1217+
with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke:
1218+
result = await evaluator.evaluate_with_extensions(
1219+
"selected input",
1220+
Step(type="llm", name="answer", input="prompt"),
1221+
extensions,
1222+
)
1223+
11261224
assert result.error is not None
1127-
assert "user_id" in result.error
1225+
assert "organization_id" in result.error
11281226
mock_invoke.assert_not_called()
11291227

11301228
@patch.dict(os.environ, LUNA_ENV)

0 commit comments

Comments
 (0)