Skip to content

Commit 167a8f7

Browse files
wrisawrisa
committed
added execution context
1 parent 875873d commit 167a8f7

3 files changed

Lines changed: 212 additions & 30 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -229,12 +229,12 @@ class ScorerInvokeRecord(BaseModel):
229229

230230

231231
class GalileoExecutionContext(BaseModel):
232-
"""Authenticated identity fields required by Galileo scorer invocation."""
232+
"""Authenticated organization and optional identity fields for Galileo scorer invocation."""
233233

234234
organization_id: str = Field(min_length=1)
235-
user_id: str = Field(min_length=1)
236-
project_id: str = Field(min_length=1)
237-
run_id: str = Field(min_length=1)
235+
user_id: str | None = Field(default=None, min_length=1)
236+
project_id: str | None = Field(default=None, min_length=1)
237+
run_id: str | None = Field(default=None, min_length=1)
238238

239239

240240
class ScorerInvokeRequest(BaseModel):
@@ -267,7 +267,10 @@ def ensure_required_values(self) -> ScorerInvokeRequest:
267267

268268
def to_dict(self) -> JSONObject:
269269
"""Convert to the LLM scorer invoke request shape."""
270-
return self.model_dump(mode="json", exclude_none=True)
270+
request = self.model_dump(mode="json", exclude_none=True)
271+
if self.execution_context is not None:
272+
request["execution_context"] = self.execution_context.model_dump(mode="json")
273+
return request
271274

272275

273276
def _orbit_record_from_step(

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py‎

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -236,38 +236,26 @@ def _execution_context_from_extensions(
236236
metadata = extensions.get("metadata")
237237
metadata_obj = metadata if isinstance(metadata, dict) else {}
238238
organization_id = extensions.get("namespace_key")
239-
# Orbit's runtime-auth contract defines caller_id as the authenticated
240-
# user ID for this flow. Keep that contract mapping inside Galileo.
239+
# The runtime envelope's caller_id is supplied by the authenticated
240+
# principal and is the caller identity Galileo associates with this run.
241241
user_id = extensions.get("caller_id")
242242
project_id = metadata_obj.get("project_id")
243243
target_type = extensions.get("target_type")
244244
target_id = extensions.get("target_id")
245-
run_id = target_id if target_type == "log_stream" else None
246-
247-
missing = [
248-
field
249-
for field, value in (
250-
("organization_id", organization_id),
251-
("user_id", user_id),
252-
("project_id", project_id),
253-
("run_id", run_id),
254-
)
255-
if not isinstance(value, str) or not value
256-
]
257-
if missing:
258-
raise ValueError(
259-
"Authenticated execution metadata is missing required fields: "
260-
+ ", ".join(missing)
261-
)
245+
run_id = (
246+
target_id
247+
if target_type in ("log_stream", "agent_stream")
248+
and isinstance(target_id, str)
249+
and target_id
250+
else None
251+
)
252+
if not isinstance(organization_id, str) or not organization_id:
253+
raise ValueError("Authenticated execution metadata is missing organization_id")
262254

263-
assert isinstance(organization_id, str)
264-
assert isinstance(user_id, str)
265-
assert isinstance(project_id, str)
266-
assert isinstance(run_id, str)
267255
return GalileoExecutionContext(
268256
organization_id=organization_id,
269-
user_id=user_id,
270-
project_id=project_id,
257+
user_id=user_id if isinstance(user_id, str) and user_id else None,
258+
project_id=project_id if isinstance(project_id, str) and project_id else None,
271259
run_id=run_id,
272260
)
273261

‎evaluators/contrib/galileo/tests/test_llm_evaluator.py‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
"""Tests for the direct Galileo LLM-as-judge evaluator and client."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
from unittest.mock import AsyncMock, patch
7+
8+
import pytest
9+
from agent_control_models import Step
10+
11+
LLM_ENV = {
12+
"GALILEO_API_SECRET_KEY": "test-secret",
13+
"GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
14+
}
15+
16+
17+
class TestGalileoLLMClient:
18+
"""Tests for ``GalileoLLMClient`` and ``ScorerInvokeRequest``."""
19+
20+
def test_execution_context_serializes_missing_optional_claims_as_null(self) -> None:
21+
from agent_control_evaluator_galileo.llm.client import (
22+
GalileoExecutionContext,
23+
ScorerInvokeInputs,
24+
ScorerInvokeRequest,
25+
)
26+
27+
request = ScorerInvokeRequest(
28+
scorer_id="scorer-123",
29+
inputs=ScorerInvokeInputs(query="hello"),
30+
execution_context=GalileoExecutionContext(organization_id="org-1"),
31+
)
32+
33+
assert request.to_dict()["execution_context"] == {
34+
"organization_id": "org-1",
35+
"user_id": None,
36+
"project_id": None,
37+
"run_id": None,
38+
}
39+
40+
41+
class TestLlmEvaluator:
42+
"""Tests for ``LlmEvaluator`` execution context handling."""
43+
44+
@patch.dict(os.environ, LLM_ENV)
45+
@pytest.mark.asyncio
46+
@pytest.mark.parametrize("target_type", ["log_stream", "agent_stream"])
47+
async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: str) -> None:
48+
from agent_control_evaluator_galileo.llm import LlmEvaluator
49+
from agent_control_evaluator_galileo.llm.client import (
50+
GalileoExecutionContext,
51+
GalileoLLMClient,
52+
ScorerInvokeResponse,
53+
)
54+
55+
evaluator = LlmEvaluator.from_dict(
56+
{"scorer_id": "scorer-123", "threshold": 0.5, "operator": "gte"}
57+
)
58+
extensions = {
59+
"namespace_key": "org-1",
60+
"caller_id": "verified-user-2",
61+
"target_type": target_type,
62+
"target_id": "run-4",
63+
"metadata": {"project_id": "project-3"},
64+
}
65+
66+
with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
67+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
68+
result = await evaluator.evaluate_with_extensions(
69+
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
70+
)
71+
72+
assert result.matched is True
73+
mock_invoke.assert_awaited_once_with(
74+
scorer_id="scorer-123",
75+
step=Step(type="llm", name="answer", input="prompt"),
76+
execution_context=GalileoExecutionContext(
77+
organization_id="org-1",
78+
user_id="verified-user-2",
79+
project_id="project-3",
80+
run_id="run-4",
81+
),
82+
input="selected input",
83+
output=None,
84+
config=None,
85+
timeout=10.0,
86+
)
87+
88+
@patch.dict(os.environ, LLM_ENV)
89+
@pytest.mark.asyncio
90+
async def test_evaluator_allows_missing_caller_id(self) -> None:
91+
from agent_control_evaluator_galileo.llm import LlmEvaluator
92+
from agent_control_evaluator_galileo.llm.client import (
93+
GalileoExecutionContext,
94+
GalileoLLMClient,
95+
ScorerInvokeResponse,
96+
)
97+
98+
evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"})
99+
extensions = {
100+
"namespace_key": "org-1",
101+
"target_type": "log_stream",
102+
"target_id": "run-4",
103+
"metadata": {},
104+
}
105+
106+
with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
107+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
108+
result = await evaluator.evaluate_with_extensions(
109+
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
110+
)
111+
112+
assert result.error is None
113+
mock_invoke.assert_awaited_once_with(
114+
scorer_id="scorer-123",
115+
step=Step(type="llm", name="answer", input="prompt"),
116+
execution_context=GalileoExecutionContext(
117+
organization_id="org-1",
118+
user_id=None,
119+
project_id=None,
120+
run_id="run-4",
121+
),
122+
input="selected input",
123+
output=None,
124+
config=None,
125+
timeout=10.0,
126+
)
127+
128+
@patch.dict(os.environ, LLM_ENV)
129+
@pytest.mark.asyncio
130+
async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None:
131+
from agent_control_evaluator_galileo.llm import LlmEvaluator
132+
from agent_control_evaluator_galileo.llm.client import (
133+
GalileoExecutionContext,
134+
GalileoLLMClient,
135+
ScorerInvokeResponse,
136+
)
137+
138+
evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"})
139+
extensions = {
140+
"namespace_key": "org-1",
141+
"caller_id": "verified-user-2",
142+
"target_type": "environment",
143+
"target_id": "prod",
144+
"metadata": {"project_id": "project-3"},
145+
}
146+
147+
with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
148+
mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
149+
result = await evaluator.evaluate_with_extensions(
150+
"selected input", Step(type="llm", name="answer", input="prompt"), extensions
151+
)
152+
153+
assert result.error is None
154+
mock_invoke.assert_awaited_once_with(
155+
scorer_id="scorer-123",
156+
step=Step(type="llm", name="answer", input="prompt"),
157+
execution_context=GalileoExecutionContext(
158+
organization_id="org-1",
159+
user_id="verified-user-2",
160+
project_id="project-3",
161+
run_id=None,
162+
),
163+
input="selected input",
164+
output=None,
165+
config=None,
166+
timeout=10.0,
167+
)
168+
169+
@patch.dict(os.environ, LLM_ENV)
170+
@pytest.mark.asyncio
171+
async def test_evaluator_rejects_missing_organization_id(self) -> None:
172+
from agent_control_evaluator_galileo.llm import LlmEvaluator
173+
from agent_control_evaluator_galileo.llm.client import GalileoLLMClient
174+
175+
evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"})
176+
extensions = {
177+
"target_type": "log_stream",
178+
"target_id": "run-4",
179+
"metadata": {"project_id": "project-3"},
180+
}
181+
182+
with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
183+
result = await evaluator.evaluate_with_extensions(
184+
"selected input",
185+
Step(type="llm", name="answer", input="prompt"),
186+
extensions,
187+
)
188+
189+
assert result.error is not None
190+
assert "organization_id" in result.error
191+
mock_invoke.assert_not_called()

0 commit comments

Comments
 (0)