Skip to content

Commit 77a4763

Browse files
add messages as part of input for llm records
1 parent 215fdd7 commit 77a4763

3 files changed

Lines changed: 174 additions & 3 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import ast
56
import json
67
from collections.abc import Mapping, Sequence
78
from dataclasses import asdict, is_dataclass
@@ -180,6 +181,36 @@ def _message_sequence(value: Any) -> bool:
180181
)
181182

182183

184+
def _llm_message_sequence(value: Any) -> Sequence[Any] | None:
185+
"""Extract a role-bearing LLM message list from structured or string input."""
186+
if isinstance(value, str):
187+
try:
188+
parsed = json.loads(value)
189+
except (RecursionError, ValueError):
190+
try:
191+
parsed = ast.literal_eval(value)
192+
except (SyntaxError, ValueError, TypeError, RecursionError):
193+
return None
194+
if isinstance(parsed, Mapping):
195+
candidates = parsed.get("messages")
196+
if not isinstance(candidates, list):
197+
return None
198+
elif isinstance(parsed, list):
199+
candidates = parsed
200+
else:
201+
return None
202+
else:
203+
candidates = value.get("messages") if isinstance(value, Mapping) else value
204+
if not isinstance(candidates, Sequence) or isinstance(candidates, str | bytes | bytearray):
205+
return None
206+
if not all(
207+
isinstance(item, Message) or (isinstance(item, Mapping) and "role" in item)
208+
for item in candidates
209+
):
210+
return None
211+
return candidates
212+
213+
183214
def _flatten_message_sequence(value: Sequence[Any]) -> list[dict[str, Any]]:
184215
blocks: list[dict[str, Any]] = []
185216
for message in value:
@@ -246,11 +277,18 @@ def llm_input(cls, value: Any) -> Any:
246277
if value is None:
247278
return ""
248279
if isinstance(value, str | Message):
249-
return value
280+
messages = _llm_message_sequence(value)
281+
if messages is None:
282+
return value
283+
return [_json_compatible(item) for item in messages]
250284
if isinstance(value, Mapping):
285+
messages = _llm_message_sequence(value)
286+
if messages is not None:
287+
return [_json_compatible(item) for item in messages]
251288
return _json_compatible(value)
252289
if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
253-
if all(isinstance(item, Message | Mapping) for item in value):
290+
messages = _llm_message_sequence(value)
291+
if messages is not None:
254292
return [_json_compatible(item) for item in value]
255293
return cls.json_text(value)
256294

@@ -300,7 +338,11 @@ def tool_redacted_output(cls, value: Any) -> str | None:
300338

301339
@classmethod
302340
def retriever_input(cls, value: Any) -> str:
303-
return "" if value is None else cls.json_text(value)
341+
if value is None:
342+
return ""
343+
if isinstance(value, Mapping) and "query" in value:
344+
return cls.json_text(value["query"])
345+
return cls.json_text(value)
304346

305347
@staticmethod
306348
def retriever_output(value: Any) -> list[Document]:

‎evaluators/contrib/galileo/tests/test_luna_evaluator.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,49 @@ def handler(request: httpx.Request) -> httpx.Response:
928928
"config": {"request_timeout_seconds": 8.0},
929929
}
930930

931+
@patch.dict(os.environ, LUNA_ENV)
932+
@pytest.mark.asyncio
933+
async def test_evaluator_normalizes_record_without_changing_legacy_inputs(self) -> None:
934+
from agent_control_evaluator_galileo.luna import LunaEvaluator
935+
936+
captured: dict[str, object] = {}
937+
raw_messages = "{'messages': [{'role': 'user', 'content': 'wire policy'}]}"
938+
raw_response = "{'answer': 'approved'}"
939+
940+
def handler(request: httpx.Request) -> httpx.Response:
941+
captured["body"] = json.loads(request.content.decode())
942+
return httpx.Response(200, json={"score": 0.9, "status": "success"})
943+
944+
step = Step(
945+
type="llm",
946+
name="answer",
947+
input=raw_messages,
948+
output=raw_response,
949+
)
950+
evaluator = LunaEvaluator.from_dict(
951+
{"scorer_id": SCORER_ID, "scorer_version_id": SCORER_VERSION_ID}
952+
)
953+
evaluator._client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
954+
955+
try:
956+
result = await evaluator.evaluate_with_context(
957+
{"input": raw_messages, "output": raw_response},
958+
step,
959+
)
960+
finally:
961+
await evaluator._client.close()
962+
963+
assert result.error is None
964+
body = captured["body"]
965+
assert isinstance(body, dict)
966+
assert body["inputs"]["query"] == raw_messages
967+
assert body["inputs"]["response"] == raw_response
968+
assert body["scorer_version_id"] == SCORER_VERSION_ID
969+
record = body["record"]
970+
assert isinstance(record, dict)
971+
assert record["input"] == [{"role": "user", "content": "wire policy"}]
972+
assert record["output"] == {"content": raw_response, "role": "assistant"}
973+
931974
@pytest.mark.asyncio
932975
async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> None:
933976
from agent_control_evaluator_galileo.luna import GalileoLunaClient

‎evaluators/contrib/galileo/tests/test_records_factory.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,60 @@ def test_llm_messages_use_public_canonical_validation() -> None:
164164
assert record.dataset_output == json.dumps({"expected": "answer"})
165165

166166

167+
@pytest.mark.parametrize(
168+
("source", "expected_messages"),
169+
[
170+
("A plain prompt", None),
171+
(
172+
'[{"role": "user", "content": "question"}]',
173+
[{"role": "user", "content": "question"}],
174+
),
175+
(
176+
"{'messages': [{'role': 'user', 'content': 'question'}]}",
177+
[{"role": "user", "content": "question"}],
178+
),
179+
("{'messages': [", None),
180+
('{"content": "document without a role"}', None),
181+
('[{"content": "document without a role"}]', None),
182+
],
183+
)
184+
def test_llm_string_input_only_parses_role_bearing_message_shapes(
185+
source: str,
186+
expected_messages: list[dict[str, str]] | None,
187+
) -> None:
188+
record = record_from_step(Step(type="llm", name="answer", input=source))
189+
190+
assert isinstance(record, LlmSpan)
191+
if expected_messages is None:
192+
assert len(record.input) == 1
193+
assert record.input[0].content == source
194+
else:
195+
assert [
196+
{"role": message.role.value, "content": message.content}
197+
for message in record.input
198+
] == expected_messages
199+
200+
restored = LlmSpan.model_validate(record.model_dump(mode="json", exclude_none=True))
201+
assert [message.content for message in restored.input] == [
202+
message.content for message in record.input
203+
]
204+
205+
206+
def test_llm_mapping_with_messages_key_becomes_public_message_models() -> None:
207+
record = record_from_step(
208+
Step(
209+
type="llm",
210+
name="answer",
211+
input={"messages": [{"role": "user", "content": "question"}]},
212+
)
213+
)
214+
215+
assert isinstance(record, LlmSpan)
216+
assert [message.role.value for message in record.input] == ["user"]
217+
restored = LlmSpan.model_validate(record.model_dump(mode="json", exclude_none=True))
218+
assert [message.content for message in restored.input] == ["question"]
219+
220+
167221
@pytest.mark.parametrize(
168222
("output", "expected"),
169223
[
@@ -902,6 +956,38 @@ def test_retriever_dictionaries_become_canonical_documents() -> None:
902956
assert documents[0].metadata == {"source": "kb"}
903957

904958

959+
def test_retriever_input_mapping_extracts_query_and_round_trips_documents() -> None:
960+
record = record_from_step(
961+
Step(
962+
type="retriever",
963+
name="policy_search",
964+
input={"query": "wire policy", "partition": "banking"},
965+
output=[{"content": "Policy text", "metadata": {"source": "kb"}}],
966+
)
967+
)
968+
969+
assert isinstance(record, RetrieverSpan)
970+
assert record.input == "wire policy"
971+
assert len(record.output) == 1
972+
assert isinstance(record.output[0], Document)
973+
restored = RetrieverSpan.model_validate(record.model_dump(mode="json", exclude_none=True))
974+
assert restored.input == "wire policy"
975+
assert isinstance(restored.output[0], Document)
976+
assert restored.output[0].content == "Policy text"
977+
978+
979+
def test_retriever_input_mapping_without_query_uses_json_fallback() -> None:
980+
record = record_from_step(
981+
Step(type="retriever", name="search", input={"partition": "banking"})
982+
)
983+
984+
assert isinstance(record, RetrieverSpan)
985+
assert record.input == '{"partition": "banking"}'
986+
assert GalileoRecordNormalizer.retriever_input({"query": {"phrase": "wire"}}) == (
987+
'{"phrase": "wire"}'
988+
)
989+
990+
905991
def test_trace_and_session_use_children_and_selectors_only_change_the_root() -> None:
906992
trace = record_from_step(
907993
Step(

0 commit comments

Comments
 (0)