Skip to content

Commit 215fdd7

Browse files
fix the records creation for traces and sessions
1 parent 66c3697 commit 215fdd7

2 files changed

Lines changed: 63 additions & 8 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,17 @@ async def invoke(
548548
invoke_config,
549549
http_timeout_seconds=timeout,
550550
)
551+
record = (
552+
_scorer_invoke_record_from_step(
553+
step,
554+
selected_input=input,
555+
selected_output=output,
556+
selected_data=selected_data,
557+
payload_field=selected_data_payload_field,
558+
)
559+
if step is not None and scorer_version_id is not None
560+
else None
561+
)
551562
request_body = ScorerInvokeRequest(
552563
scorer_id=scorer_id,
553564
scorer_version_id=scorer_version_id,
@@ -558,13 +569,7 @@ async def invoke(
558569
ground_truth=step.ground_truth if step is not None else None,
559570
tools=step.tools if step is not None else None,
560571
),
561-
record=_scorer_invoke_record_from_step(
562-
step,
563-
selected_input=input,
564-
selected_output=output,
565-
selected_data=selected_data,
566-
payload_field=selected_data_payload_field,
567-
),
572+
record=record,
568573
execution_context=execution_context,
569574
config=invoke_config,
570575
).to_dict()

‎evaluators/contrib/galileo/tests/test_luna_evaluator.py‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,49 @@ def handler(request: httpx.Request) -> httpx.Response:
885885
"config": {"request_timeout_seconds": 8.0},
886886
}
887887

888+
@pytest.mark.asyncio
889+
async def test_client_omits_record_when_step_has_no_pinned_version(self) -> None:
890+
from agent_control_evaluator_galileo.luna import GalileoLunaClient
891+
892+
captured: dict[str, object] = {}
893+
894+
def handler(request: httpx.Request) -> httpx.Response:
895+
captured["body"] = json.loads(request.content.decode())
896+
return httpx.Response(200, json={"score": 0.5, "status": "success"})
897+
898+
step = Step(
899+
type="llm",
900+
name="answer",
901+
input="complete prompt",
902+
output="complete response",
903+
tools=[{"name": "search", "input_schema": {}}],
904+
ground_truth="expected answer",
905+
)
906+
with patch.dict(os.environ, LUNA_ENV, clear=True):
907+
client = GalileoLunaClient()
908+
client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
909+
910+
try:
911+
await client.invoke(
912+
scorer_id=SCORER_ID,
913+
input="selected prompt",
914+
output="selected response",
915+
step=step,
916+
)
917+
finally:
918+
await client.close()
919+
920+
assert captured["body"] == {
921+
"scorer_id": SCORER_ID,
922+
"inputs": {
923+
"query": "selected prompt",
924+
"response": "selected response",
925+
"ground_truth": "expected answer",
926+
"tools": [{"name": "search", "input_schema": {}}],
927+
},
928+
"config": {"request_timeout_seconds": 8.0},
929+
}
930+
888931
@pytest.mark.asyncio
889932
async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> None:
890933
from agent_control_evaluator_galileo.luna import GalileoLunaClient
@@ -946,6 +989,7 @@ def handler(request: httpx.Request) -> httpx.Response:
946989
try:
947990
await client.invoke(
948991
scorer_id="scorer-123",
992+
scorer_version_id=SCORER_VERSION_ID,
949993
input="selected input",
950994
output="selected output",
951995
step=step,
@@ -1000,6 +1044,7 @@ def handler(request: httpx.Request) -> httpx.Response:
10001044
try:
10011045
await client.invoke(
10021046
scorer_id="scorer-123",
1047+
scorer_version_id=SCORER_VERSION_ID,
10031048
input={"query": "selected"},
10041049
output={"answer": "selected"},
10051050
step=step,
@@ -1262,7 +1307,12 @@ async def test_client_rejects_hierarchy_fields_from_step_context(self, field: st
12621307

12631308
try:
12641309
with pytest.raises(RecordFactoryError, match="Step.context cannot define"):
1265-
await client.invoke(scorer_id="scorer-123", input="selected", step=step)
1310+
await client.invoke(
1311+
scorer_id="scorer-123",
1312+
scorer_version_id=SCORER_VERSION_ID,
1313+
input="selected",
1314+
step=step,
1315+
)
12661316
finally:
12671317
await client.close()
12681318

0 commit comments

Comments
 (0)