Skip to content

Commit 6c84192

Browse files
address comments
1 parent a0dc000 commit 6c84192

2 files changed

Lines changed: 61 additions & 18 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def default(self, value: Any) -> Any:
3333
)
3434
if isinstance(value, datetime):
3535
if value.tzinfo is None:
36-
value = value.replace(tzinfo=datetime.now().astimezone().tzinfo)
36+
value = value.astimezone()
3737
serialized = value.isoformat()
3838
if value.tzinfo is not None and value.tzinfo.tzname(None) == UTC.tzname(None):
3939
return serialized.replace("+00:00", "Z")
@@ -43,16 +43,14 @@ def default(self, value: Any) -> Any:
4343
if isinstance(value, UUID | Path):
4444
return str(value)
4545
if isinstance(value, Enum):
46-
return value.value
46+
return _sdk_json_value(value.value)
4747
if isinstance(value, bytes):
4848
try:
4949
return value.decode("utf-8")
5050
except UnicodeDecodeError:
5151
return "<not serializable bytes>"
5252
if is_dataclass(value) and not isinstance(value, type):
5353
return asdict(value)
54-
if isinstance(value, int) and not isinstance(value, bool):
55-
return value if -_MAX_SAFE_INTEGER <= value <= _MAX_SAFE_INTEGER else str(value)
5654
if isinstance(value, set | frozenset):
5755
return list(value)
5856
return f"<{type(value).__name__}>"
@@ -123,20 +121,20 @@ def _json_text(value: Any) -> str:
123121
"""Return the Galileo text representation of a structured value."""
124122
if isinstance(value, str):
125123
return value
126-
if value is None or isinstance(value, bool | int | float):
127-
return json.dumps(value)
128124
return json.dumps(_sdk_json_value(value))
129125

130126

131127
def _json_compatible(value: Any) -> Any:
132128
"""Convert models and containers into values accepted by Galileo Pydantic models."""
133129
if isinstance(value, BaseModel):
134-
return value.model_dump(mode="python", exclude_none=True)
130+
return _json_compatible(value.model_dump(mode="python", exclude_none=True))
135131
if isinstance(value, Mapping):
136132
return {key: _json_compatible(item) for key, item in value.items()}
137133
if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
138134
return [_json_compatible(item) for item in value]
139-
if isinstance(value, UUID | datetime | date | Enum):
135+
if isinstance(value, set | frozenset):
136+
return [_json_compatible(item) for item in value]
137+
if isinstance(value, UUID | datetime | date | Enum | Path | bytes):
140138
return _GalileoJSONEncoder().default(value)
141139
return value
142140

@@ -157,7 +155,7 @@ def _content_part(value: Any) -> dict[str, Any] | None:
157155
def _message_content(value: Any) -> Any:
158156
if isinstance(value, Message):
159157
return value.content
160-
if isinstance(value, Mapping) and ("role" in value or "content" in value):
158+
if isinstance(value, Mapping) and "role" in value:
161159
return value.get("content", "")
162160
return None
163161

@@ -177,7 +175,7 @@ def _normalize_content_items(value: Any) -> list[dict[str, Any]] | None:
177175
def _message_sequence(value: Any) -> bool:
178176
return isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray) and all(
179177
isinstance(item, Message)
180-
or (isinstance(item, Mapping) and ("role" in item or "content" in item))
178+
or (isinstance(item, Mapping) and "role" in item)
181179
for item in value
182180
)
183181

@@ -451,9 +449,7 @@ def session_output(cls, value: Any) -> Any:
451449
content_parts = _normalize_content_items(value)
452450
if content_parts is not None:
453451
return content_parts
454-
if all(isinstance(item, Mapping) for item in value):
455-
return [_json_compatible(item) for item in value]
456-
return [_json_compatible(item) for item in value]
452+
return cls.json_text(value)
457453
return cls.json_text(value)
458454

459455
@classmethod

‎evaluators/contrib/galileo/tests/test_records_factory.py‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,16 @@ def test_session_content_parts_are_preserved_by_input_and_output_normalizers() -
696696
]
697697

698698

699+
def test_session_output_serializes_unsupported_sequences_for_core_model() -> None:
700+
output = GalileoRecordNormalizer.session_output([1, 2])
701+
702+
assert isinstance(output, str)
703+
assert output == json.dumps([1, 2])
704+
assert json.loads(output) == [1, 2]
705+
session = Session(input=[], output=output, traces=[])
706+
assert session.output == output
707+
708+
699709
def test_record_normalizer_serializes_nested_models_uuids_and_timestamps() -> None:
700710
identifier = uuid4()
701711
timestamp = datetime(2025, 1, 2, 3, 4, 5, tzinfo=UTC)
@@ -735,6 +745,9 @@ class NestedModel(BaseModel):
735745
value: int
736746
optional: str | None = None
737747

748+
class NestedValuesModel(BaseModel):
749+
payload: object
750+
738751
@dataclass
739752
class NestedDataclass:
740753
identifier: object
@@ -756,21 +769,32 @@ def __init__(self) -> None:
756769
naive_timestamp = datetime(2025, 1, 2)
757770
offset_timestamp = datetime(2025, 1, 2, tzinfo=timezone(timedelta(hours=2)))
758771

772+
class StructuredState(Enum):
773+
VALUE = {"identifier": identifier, "timestamp": utc_timestamp}
774+
759775
assert encoder.default(NestedModel(value=3)) == {"value": 3}
760776
assert encoder.default(utc_timestamp) == "2025-01-02T00:00:00Z"
761-
assert isinstance(encoder.default(naive_timestamp), str)
777+
assert encoder.default(naive_timestamp) == naive_timestamp.astimezone().isoformat()
762778
assert encoder.default(offset_timestamp) == "2025-01-02T00:00:00+02:00"
763779
assert encoder.default(date(2025, 1, 2)) == "2025-01-02"
764780
assert encoder.default(identifier) == str(identifier)
765781
assert encoder.default(Path("folder/file.txt")) == "folder/file.txt"
766782
assert encoder.default(State.READY) == "ready"
783+
assert json.dumps(StructuredState.VALUE, cls=_GalileoJSONEncoder) == json.dumps(
784+
{"identifier": str(identifier), "timestamp": "2025-01-02T00:00:00Z"}
785+
)
767786
assert encoder.default(b"text") == "text"
768787
assert encoder.default(b"\xff") == "<not serializable bytes>"
769788
assert encoder.default(NestedDataclass(identifier)) == {"identifier": identifier}
770-
assert encoder.default(2**53) == str(2**53)
771789
assert encoder.default({"item"}) == ["item"]
772790
assert encoder.default(object()) == "<object>"
773791

792+
assert GalileoRecordNormalizer.json_text(42) == "42"
793+
assert GalileoRecordNormalizer.json_text(2**53) == json.dumps(str(2**53))
794+
assert GalileoRecordNormalizer.json_text({"large_integer": 2**53}) == json.dumps(
795+
{"large_integer": str(2**53)}
796+
)
797+
774798
recursive: list[object] = []
775799
recursive.append(recursive)
776800
normalized = _sdk_json_value(
@@ -805,29 +829,52 @@ def __init__(self) -> None:
805829
assert _json_compatible({"values": [State.READY, date(2025, 1, 2)]}) == {
806830
"values": ["ready", "2025-01-02"]
807831
}
832+
nested_values = _json_compatible(
833+
{
834+
"nested": {
835+
"set": {b"bytes"},
836+
"frozenset": frozenset({Path("nested/path")}),
837+
"invalid_bytes": {b"\xff"},
838+
},
839+
"model": NestedValuesModel(
840+
payload={"set": {b"model bytes"}, "path": Path("model/path")}
841+
),
842+
}
843+
)
844+
assert nested_values == {
845+
"nested": {
846+
"set": ["bytes"],
847+
"frozenset": ["nested/path"],
848+
"invalid_bytes": ["<not serializable bytes>"],
849+
},
850+
"model": {"payload": {"set": ["model bytes"], "path": "model/path"}},
851+
}
852+
assert json.loads(json.dumps(nested_values)) == nested_values
808853

809854

810855
def test_normalizer_preserves_message_content_and_reports_invalid_trace_inputs() -> None:
811856
from agent_control_evaluator_galileo.records.normalization import (
812857
_flatten_message_sequence,
858+
_message_content,
813859
_message_sequence,
814860
_normalize_content_items,
815861
)
816862

817863
messages = [
818864
{"role": "user", "content": ""},
819865
{"role": "assistant", "content": ["plain", {"unsupported": True}]},
820-
{"content": 7},
821-
{},
822866
]
823867
flattened = _flatten_message_sequence(messages)
824868

825869
assert [part["text"] for part in flattened] == [
826870
"plain",
827871
'{"unsupported": true}',
828-
"7",
829872
]
830873
assert _message_sequence([{"role": "user", "content": "question"}])
874+
assert not _message_sequence([{"content": "question"}])
875+
retriever_document = {"content": "retrieved text", "metadata": {"source": "kb"}}
876+
assert not _message_sequence([retriever_document])
877+
assert _message_content(retriever_document) is None
831878
assert not _message_sequence([{"other": "field"}])
832879
assert _normalize_content_items("not a sequence") is None
833880
assert _normalize_content_items([{"invalid": True}]) is None

0 commit comments

Comments
 (0)