From c17994fa218bd7f96199a8066316563ddd1cd65c Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Thu, 24 Sep 2026 13:46:47 -0700 Subject: [PATCH 01/17] feat(galileo): dual-write canonical Luna records --- .../luna/client.py | 60 +------ .../luna/config.py | 8 +- .../luna/evaluator.py | 22 ++- .../galileo/tests/test_luna_evaluator.py | 169 +++++++++++++++++- 4 files changed, 201 insertions(+), 58 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index d00d1457..f0554894 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -11,7 +11,7 @@ from hmac import new as hmac_new from json import dumps from time import time -from typing import Literal, cast, get_args +from typing import Literal, get_args from urllib.parse import urlsplit import httpx @@ -233,8 +233,8 @@ class ScorerInvokeRequest(BaseModel): Attributes: scorer_id: Required scorer identifier. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. + scorer_version_id: Optional for legacy requests and required when a + structured record is supplied. scorer_label: Optional display/metadata label. inputs: Selected scorer input values. record: Optional Orbit-compatible structured runtime record. @@ -245,13 +245,15 @@ class ScorerInvokeRequest(BaseModel): scorer_version_id: str | None = Field(default=None, min_length=1) scorer_label: str | None = Field(default=None, min_length=1) inputs: ScorerInvokeInputs - record: ScorerInvokeRecord | None = None + record: JSONObject | ScorerInvokeRecord | None = None config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) @model_validator(mode="after") def ensure_required_values(self) -> ScorerInvokeRequest: if not (_has_value(self.inputs.query) or _has_value(self.inputs.response)): raise ValueError("Either inputs.query or inputs.response must be set.") + if self.record is not None and self.scorer_version_id is None: + raise ValueError("scorer_version_id is required when a structured record is provided.") return self def to_dict(self) -> JSONObject: @@ -259,48 +261,6 @@ def to_dict(self) -> JSONObject: return self.model_dump(mode="json", exclude_none=True) -def _orbit_record_from_step( - step: Step | None, - *, - selected_input: JSONValue, - selected_output: JSONValue, -) -> ScorerInvokeRecord | None: - """Translate a generic Agent Control step into Orbit's record contract. - - Selector-selected values remain the primary scorer input. When a selector - supplies one side, that value is written to both the legacy and structured - representations so Orbit's conflict validation cannot observe two meanings. - The complete step supplies the unselected side and additional record context. - - Unknown Agent Control step types intentionally fall back to the legacy - ``inputs`` contract. This keeps the open-source Step model extensible without - sending an invalid discriminator to Orbit. - - Args: - step: Complete Agent Control step, when contextual evaluation is used. - selected_input: Selector-selected value sent as ``inputs.query``. - selected_output: Selector-selected value sent as ``inputs.response``. - - Returns: - An Orbit-compatible record, or ``None`` for absent/unsupported steps. - """ - if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: - return None - - # The membership check above narrows the runtime value to Orbit's known - # discriminator set, but static type checkers cannot infer that relationship. - record_type = cast(ScorerInvokeRecordType, step.type) - return ScorerInvokeRecord( - type=record_type, - name=step.name, - input=selected_input if selected_input is not None else step.input, - output=selected_output if selected_output is not None else step.output, - context=step.context, - tools=step.tools, - dataset_output=step.ground_truth, - ) - - class ScorerInvokeResponse(BaseModel): """Response from Luna scorer invocation. @@ -474,6 +434,7 @@ async def invoke( input: JSONValue = None, output: JSONValue = None, step: Step | None = None, + record: JSONObject | None = None, config: ScorerInvokeConfig | JSONObject | None = None, timeout: float = DEFAULT_TIMEOUT_SECS, headers: dict[str, str] | None = None, @@ -488,6 +449,7 @@ async def invoke( input: Optional user/system prompt text. output: Optional model response text. step: Optional complete runtime step used for structured dual-write. + record: Optional canonical Galileo record serialized by the evaluator. config: Optional Orbit-supported scorer invocation configuration. timeout: Request timeout in seconds. headers: Additional request headers. @@ -526,11 +488,7 @@ async def invoke( ground_truth=step.ground_truth if step is not None else None, tools=step.tools if step is not None else None, ), - record=_orbit_record_from_step( - step, - selected_input=input, - selected_output=output, - ), + record=record, config=invoke_config, ).to_dict() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py index 0bc0fa37..52c315ec 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py @@ -56,8 +56,8 @@ class LunaEvaluatorConfig(EvaluatorConfig): Attributes: scorer_id: Required scorer identifier for Luna scorer invocation. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. + scorer_version_id: Optional for legacy requests; required when the + contextual evaluator sends a structured record. scorer_label: Optional display/metadata label. threshold: Local threshold used by the evaluator for comparison. operator: Local comparison operator. Numeric operators use threshold as a number. @@ -75,8 +75,8 @@ class LunaEvaluatorConfig(EvaluatorConfig): default=None, min_length=1, description=( - "Deprecated optional compatibility identifier. Orbit currently invokes " - "the scorer's current default version." + "Optional for legacy requests and required when contextual evaluation " + "sends a structured record." ), ) scorer_label: str | None = Field( diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index e4147a9a..f8d3c15c 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -6,12 +6,13 @@ import logging import os from importlib.metadata import PackageNotFoundError, version -from typing import Any +from typing import Any, cast import httpx from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator -from agent_control_models import EvaluatorResult, JSONValue, Step +from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step +from ..records import build_galileo_record from .client import GalileoLunaClient, ScorerInvokeResponse from .config import LunaEvaluatorConfig, coerce_number @@ -225,8 +226,25 @@ async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: try: scorer_kwargs = self._scorer_kwargs() + record_payload: JSONObject | None = None if step is not None: scorer_kwargs["step"] = step + if step.type.strip().lower() in {"llm", "tool", "retriever", "trace", "session"}: + if self.config.scorer_version_id is None: + raise ValueError( + "scorer_version_id is required for structured Luna requests." + ) + record = build_galileo_record( + data, + step, + payload_field=self.config.payload_field, + ) + record_payload = cast( + JSONObject, + record.model_dump(mode="json", exclude_none=True), + ) + if record_payload is not None: + scorer_kwargs["record"] = record_payload response = await self._get_client().invoke( **scorer_kwargs, input=input_text if _has_text(input_text) else None, diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 4334ac5e..fa89f6c4 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -235,12 +235,14 @@ def test_scorer_invoke_request_serialization_keeps_legacy_record_shape(self) -> request = ScorerInvokeRequest( scorer_id="scorer-123", + scorer_version_id="version-123", inputs=ScorerInvokeInputs(query="hello"), record=ScorerInvokeRecord(type="tool", name="search", input={"q": "hello"}), ) assert request.to_dict() == { "scorer_id": "scorer-123", + "scorer_version_id": "version-123", "inputs": {"query": "hello", "response": ""}, "record": {"type": "tool", "name": "search", "input": {"q": "hello"}}, "config": {}, @@ -787,6 +789,15 @@ def handler(request: httpx.Request) -> httpx.Response: input="selected question", output="selected answer", step=step, + record={ + "type": "llm", + "name": "answer", + "input": "selected question", + "output": "selected answer", + "context": {"session": {"id": "s-1", "attributes": {"region": "west"}}}, + "tools": [{"name": "search", "description": "Search", "input_schema": {}}], + "dataset_output": {"text": "expected"}, + }, ) finally: await client.close() @@ -1022,7 +1033,12 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: # Given: selected scorer data and complete structured runtime context evaluator = LunaEvaluator.from_dict( - {"scorer_id": "scorer-123", "threshold": 0.5, "operator": "gte"} + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "threshold": 0.5, + "operator": "gte", + } ) step = Step( type="llm", @@ -1040,14 +1056,165 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: # Then: selector-selected data and the complete Step are both forwarded assert result.matched is True + record = mock_invoke.await_args.kwargs["record"] mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", + scorer_version_id="version-456", step=step, input="selected input", output=None, + record=record, config=None, timeout=10.0, ) + assert record["type"] == "llm" + assert record["input"][0]["content"] == "selected input" + assert record["output"]["content"] == "full output" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_trace_preserves_raw_record_and_nested_spans(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "threshold": 0.5, + } + ) + step = Step( + type="trace", + name="account_lookup_trace", + input={ + "input": "Find my account information.", + "spans": [ + { + "type": "tool", + "name": "lookup", + "input": {"account": "jane"}, + "output": "found", + }, + {"type": "llm", "name": "answer", "input": "question", "output": "answer"}, + ], + }, + output="Your account belongs to Jane Doe.", + ) + raw_data = step.input + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context(raw_data, step) + + kwargs = mock_invoke.await_args.kwargs + assert kwargs["input"] == "Find my account information." + record = kwargs["record"] + assert record["type"] == "trace" + assert record["name"] == "account_lookup_trace" + assert record["input"] == "Find my account information." + assert record["output"] == "Your account belongs to Jane Doe." + assert len(record["spans"]) == 2 + assert record["spans"][0]["name"] == "lookup" + assert json.loads(record["spans"][0]["input"]) == {"account": "jane"} + assert record["spans"][1]["name"] == "answer" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_session_preserves_nested_traces(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + {"scorer_id": "scorer-123", "scorer_version_id": "version-456"} + ) + step = Step( + type="session", + name="conversation", + input="question", + context={ + "traces": [ + { + "type": "trace", + "name": "request", + "input": "question", + "spans": [ + { + "type": "tool", + "name": "search", + "input": {"query": "question"}, + "output": "result", + } + ], + } + ] + }, + ) + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context(step.model_dump(mode="json"), step) + + record = mock_invoke.await_args.kwargs["record"] + assert record["type"] == "session" + assert record["traces"][0]["type"] == "trace" + assert record["traces"][0]["spans"][0]["name"] == "search" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("step_type", "expected_error"), + [ + ("trace", "Galileo trace context is missing"), + ("session", "Galileo session context is missing"), + ], + ) + async def test_contextual_missing_group_context_returns_factory_error( + self, step_type: str, expected_error: str + ) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + {"scorer_id": "scorer-123", "scorer_version_id": "version-456"} + ) + step = Step(type=step_type, name="missing-context", input="question", output="answer") + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + result = await evaluator.evaluate_with_context(step.model_dump(mode="json"), step) + + assert expected_error in (result.error or "") + mock_invoke.assert_not_called() + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_structured_request_requires_explicit_scorer_version(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + step = Step(type="llm", name="answer", input="question", output="answer") + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + result = await evaluator.evaluate_with_context("question", step) + + assert "scorer_version_id is required" in (result.error or "") + mock_invoke.assert_not_called() + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_unsupported_step_keeps_legacy_request(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + step = Step(type="custom", name="custom", input="full input", output="full output") + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context("selected input", step) + + assert "record" not in mock_invoke.await_args.kwargs + assert mock_invoke.await_args.kwargs["input"] == "selected input" @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio From 2cd9380ce94d3faef7eb82348dd06c9d70147e33 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Thu, 24 Sep 2026 15:55:31 -0700 Subject: [PATCH 02/17] refactor(galileo): normalize records with galileo-core --- evaluators/contrib/galileo/pyproject.toml | 2 +- .../__init__.py | 2 + .../records/__init__.py | 2 + .../records/factory.py | 100 +-- .../records/normalization.py | 570 ++++++++++++++---- .../galileo/tests/test_records_factory.py | 233 +++++-- 6 files changed, 704 insertions(+), 205 deletions(-) diff --git a/evaluators/contrib/galileo/pyproject.toml b/evaluators/contrib/galileo/pyproject.toml index 953ab04c..6f0a19c4 100644 --- a/evaluators/contrib/galileo/pyproject.toml +++ b/evaluators/contrib/galileo/pyproject.toml @@ -9,8 +9,8 @@ authors = [{ name = "Agent Control Team" }] dependencies = [ "agent-control-evaluators>=8.6.0", "agent-control-models>=8.6.0", + "galileo-core>=4.4.0,<5.0.0", "httpx>=0.24.0", - "splunk-ao==0.4.0", "pydantic>=2.12.4", ] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index e3bea664..51f0a65b 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -32,6 +32,7 @@ ) from agent_control_evaluator_galileo.records import ( GalileoRecord, + GalileoRecordNormalizer, RecordFactoryError, UnsupportedStepTypeError, build_galileo_record, @@ -51,6 +52,7 @@ "LunaOperator", "LUNA_AVAILABLE", "GalileoRecord", + "GalileoRecordNormalizer", "RecordFactoryError", "UnsupportedStepTypeError", "build_galileo_record", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py index 6cd089ed..7265e8a1 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py @@ -9,9 +9,11 @@ record_from_scorer_invoke_record, record_from_step, ) +from .normalization import GalileoRecordNormalizer __all__ = [ "GalileoRecord", + "GalileoRecordNormalizer", "RecordFactoryError", "UnsupportedStepTypeError", "build_galileo_record", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py index 64de8dca..4b7959a2 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py @@ -6,25 +6,12 @@ from typing import Any, cast from agent_control_models import Step +from galileo_core.schemas.logging.session import Session +from galileo_core.schemas.logging.span import LlmSpan, RetrieverSpan, ToolSpan +from galileo_core.schemas.logging.trace import Trace from pydantic import BaseModel -from splunk_ao import ( # type: ignore[import-untyped] - LlmSpan, - RetrieverSpan, - Session, - ToolSpan, - Trace, -) - -from .normalization import ( - documents, - message_value, - session_value, - string_metadata, - text_value, - tool_value, - trace_input_value, - trace_output_value, -) + +from .normalization import GalileoRecordNormalizer type GalileoRecord = LlmSpan | ToolSpan | RetrieverSpan | Trace | Session type GalileoSpan = LlmSpan | ToolSpan | RetrieverSpan @@ -111,7 +98,9 @@ def _record_from_mapping(raw: Mapping[str, Any], record_type: str) -> GalileoRec context = _mapping(raw.get("context")) or {} common = { "name": "" if raw.get("name") is None else str(raw["name"]), - "user_metadata": string_metadata(raw.get("user_metadata", raw.get("metadata"))), + "user_metadata": GalileoRecordNormalizer.metadata( + raw.get("user_metadata", raw.get("metadata")) + ), } for field in ( "tags", @@ -126,9 +115,9 @@ def _record_from_mapping(raw: Mapping[str, Any], record_type: str) -> GalileoRec ): if raw.get(field) is not None: common[field] = ( - text_value(raw[field]) + GalileoRecordNormalizer.json_text(raw[field]) if field in {"dataset_input", "dataset_output"} - else string_metadata(raw[field]) + else GalileoRecordNormalizer.metadata(raw[field]) if field == "dataset_metadata" else raw[field] ) @@ -138,39 +127,55 @@ def _record_from_mapping(raw: Mapping[str, Any], record_type: str) -> GalileoRec if record_type == "llm": kwargs = { **common, - "input": message_value(input_value), - "output": message_value(output_value, output=True), + "input": GalileoRecordNormalizer.llm_input(input_value), + "output": GalileoRecordNormalizer.llm_output(output_value), } if raw.get("tools") is not None: - kwargs["tools"] = raw["tools"] + kwargs["tools"] = GalileoRecordNormalizer.llm_tools(raw["tools"]) for field in ("events", "model", "temperature", "finish_reason"): if raw.get(field) is not None: kwargs[field] = raw[field] if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = message_value(raw["redacted_input"]) + kwargs["redacted_input"] = GalileoRecordNormalizer.llm_redacted_input( + raw["redacted_input"] + ) if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = message_value(raw["redacted_output"], output=True) + kwargs["redacted_output"] = GalileoRecordNormalizer.llm_redacted_output( + raw["redacted_output"] + ) return LlmSpan(**kwargs) if record_type == "tool": kwargs = { **common, - "input": tool_value(input_value) or "", - "output": tool_value(output_value), + "input": GalileoRecordNormalizer.tool_input(input_value), + "output": GalileoRecordNormalizer.tool_output(output_value), } if raw.get("tool_call_id") is not None: kwargs["tool_call_id"] = str(raw["tool_call_id"]) if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = tool_value(raw["redacted_input"]) + kwargs["redacted_input"] = GalileoRecordNormalizer.tool_redacted_input( + raw["redacted_input"] + ) if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = tool_value(raw["redacted_output"]) + kwargs["redacted_output"] = GalileoRecordNormalizer.tool_redacted_output( + raw["redacted_output"] + ) kwargs["spans"] = [_nested_span(item) for item in _span_payloads(raw, context)] return ToolSpan(**kwargs) if record_type == "retriever": - kwargs = {**common, "input": text_value(input_value), "output": documents(output_value)} + kwargs = { + **common, + "input": GalileoRecordNormalizer.retriever_input(input_value), + "output": GalileoRecordNormalizer.retriever_output(output_value), + } if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = text_value(raw["redacted_input"]) + kwargs["redacted_input"] = GalileoRecordNormalizer.retriever_redacted_input( + raw["redacted_input"] + ) if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = documents(raw["redacted_output"]) + kwargs["redacted_output"] = GalileoRecordNormalizer.retriever_redacted_output( + raw["redacted_output"] + ) kwargs["spans"] = [_nested_span(item) for item in _span_payloads(raw, context)] return RetrieverSpan(**kwargs) if record_type == "trace": @@ -179,34 +184,45 @@ def _record_from_mapping(raw: Mapping[str, Any], record_type: str) -> GalileoRec raise RecordFactoryError("Galileo trace context must include a 'spans' list.") trace_kwargs: dict[str, Any] = { **common, - "input": trace_input_value(input_value), - "output": trace_output_value(output_value), + "input": GalileoRecordNormalizer.trace_input(input_value), + "output": GalileoRecordNormalizer.trace_output(output_value), "spans": [_nested_span(item) for item in spans], } if raw.get("redacted_input") is not None: - trace_kwargs["redacted_input"] = trace_input_value(raw["redacted_input"]) + trace_kwargs["redacted_input"] = GalileoRecordNormalizer.trace_redacted_input( + raw["redacted_input"] + ) if raw.get("redacted_output") is not None: - trace_kwargs["redacted_output"] = trace_output_value(raw["redacted_output"]) + trace_kwargs["redacted_output"] = GalileoRecordNormalizer.trace_redacted_output( + raw["redacted_output"] + ) return Trace( **trace_kwargs, ) traces = raw.get("traces", context.get("traces")) if not isinstance(traces, Sequence) or isinstance(traces, str | bytes | bytearray): raise RecordFactoryError("Galileo session context must include a 'traces' list.") - nested_records = [_nested_record(item) for item in traces] + nested_records = GalileoRecordNormalizer.session_traces( + traces, + normalize_record=_nested_record, + ) nested_traces = [item for item in nested_records if isinstance(item, Trace)] if len(nested_traces) != len(nested_records): raise RecordFactoryError("Galileo session 'traces' must contain trace records.") session_kwargs: dict[str, Any] = { **common, - "input": session_value(input_value), - "output": session_value(output_value), + "input": GalileoRecordNormalizer.session_input(input_value), + "output": GalileoRecordNormalizer.session_output(output_value), "traces": cast(list[Trace], nested_traces), } if raw.get("redacted_input") is not None: - session_kwargs["redacted_input"] = session_value(raw["redacted_input"]) + session_kwargs["redacted_input"] = GalileoRecordNormalizer.session_redacted_input( + raw["redacted_input"] + ) if raw.get("redacted_output") is not None: - session_kwargs["redacted_output"] = session_value(raw["redacted_output"]) + session_kwargs["redacted_output"] = GalileoRecordNormalizer.session_redacted_output( + raw["redacted_output"] + ) return Session( **session_kwargs, ) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py index cfa44b6e..d9fb41ab 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py @@ -1,150 +1,474 @@ -"""Value normalization shared by the Galileo record factory. - -The conversions in this module deliberately follow the published ``splunk-ao`` -helpers. The factory owns the Agent Control-to-record mapping, while the SDK -continues to own serialization and document coercion. - -Orbit's scorer-input normalizer retains list-of-message dictionaries as an -intermediate LLM output, but the published ``LlmSpan`` model accepts one output -message. The factory therefore serializes the complete sequence before public -model validation; this is an intentional Orbit/SDK boundary difference, not a -last-item selection rule. Likewise, retriever ``None`` follows the SDK logger -helper (one empty document), rather than the direct model default (an empty -list). -""" +"""Normalize Agent Control values for canonical Galileo record fields.""" from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Any, cast +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, is_dataclass +from datetime import UTC, date, datetime +from enum import Enum +from pathlib import Path +from typing import Any from uuid import UUID -from splunk_ao import Document, Message, SplunkAOLogger # type: ignore[import-untyped] -from splunk_ao.utils.retrievers import convert_to_documents # type: ignore[import-untyped] -from splunk_ao.utils.serialization import serialize_to_str # type: ignore[import-untyped] +from galileo_core.schemas.logging.llm import Message +from galileo_core.schemas.shared.content_parts import ContentPart, FileContentPart, TextContentPart +from galileo_core.schemas.shared.document import Document +from pydantic import BaseModel, TypeAdapter, ValidationError +_CONTENT_PART_ADAPTER: TypeAdapter[ContentPart] = TypeAdapter(ContentPart) +_MAX_SAFE_INTEGER = 2**53 - 1 -def text_value(value: Any, *, default: str = "") -> str: - """Use the SDK serializer for a value that a Galileo field stores as text.""" - if value is None: - return default - return cast(str, serialize_to_str(value)) +class _GalileoJSONEncoder(json.JSONEncoder): + """Serialize values using the stable subset of Splunk AO's JSON behavior.""" -def tool_value(value: Any) -> str | None: - """Serialize tool arguments/results using the SDK's canonical serializer.""" - if value is None: - return None - return cast(str, serialize_to_str(value)) + def default(self, value: Any) -> Any: + if isinstance(value, BaseModel): + return value.model_dump( + mode="json", + exclude_none=True, + exclude_unset=True, + exclude_defaults=True, + ) + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.now().astimezone().tzinfo) + serialized = value.isoformat() + if value.tzinfo is not None and value.tzinfo.tzname(None) == UTC.tzname(None): + return serialized.replace("+00:00", "Z") + return serialized + if isinstance(value, date): + return value.isoformat() + if isinstance(value, UUID | Path): + return str(value) + if isinstance(value, Enum): + return value.value + if isinstance(value, bytes): + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return "" + if is_dataclass(value) and not isinstance(value, type): + return asdict(value) + if isinstance(value, int) and not isinstance(value, bool): + return value if -_MAX_SAFE_INTEGER <= value <= _MAX_SAFE_INTEGER else str(value) + if isinstance(value, set | frozenset): + return list(value) + return f"<{type(value).__name__}>" -def message_value(value: Any, *, output: bool = False) -> Any: - """Prepare values for the public ``LlmSpan`` model. +def _sdk_json_value(value: Any, *, active: set[int] | None = None) -> Any: + """Apply SDK-compatible conversion recursively, including mapping keys.""" + if value is None or isinstance(value, str | bool | float): + return value + if isinstance(value, int): + return value if -_MAX_SAFE_INTEGER <= value <= _MAX_SAFE_INTEGER else str(value) + if isinstance(value, UUID | Path): + return str(value) + if isinstance(value, datetime): + return _GalileoJSONEncoder().default(value) + if isinstance(value, date): + return value.isoformat() + if isinstance(value, Enum): + return _sdk_json_value(value.value, active=active) + if isinstance(value, bytes): + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return "" - LLM inputs are passed through as message sequences so the canonical model - can validate each message. The public model has a single-message output - contract, so a list or tuple output is serialized as one complete JSON - string. This preserves every item and lets the canonical model construct - the assistant message. - """ - if value is None or isinstance(value, (str, Mapping, Message)): + seen = active if active is not None else set() + value_id = id(value) + if value_id in seen: + return type(value).__name__ + seen.add(value_id) + try: + if isinstance(value, BaseModel): + return _sdk_json_value( + value.model_dump( + mode="json", + exclude_none=True, + exclude_unset=True, + exclude_defaults=True, + ), + active=seen, + ) + if is_dataclass(value) and not isinstance(value, type): + return _sdk_json_value(asdict(value), active=seen) + if isinstance(value, Mapping): + return { + _sdk_json_value(key, active=seen): _sdk_json_value(item, active=seen) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return [_sdk_json_value(item, active=seen) for item in value] + if isinstance(value, set | frozenset): + return [_sdk_json_value(item, active=seen) for item in value] + if hasattr(value, "__slots__") and value.__slots__: + return _sdk_json_value( + {slot: getattr(value, slot, None) for slot in value.__slots__}, active=seen + ) + if hasattr(value, "__dict__"): + return _sdk_json_value( + {key: item for key, item in vars(value).items() if not key.startswith("_")}, + active=seen, + ) + return f"<{type(value).__name__}>" + finally: + seen.discard(value_id) + + +def _json_text(value: Any) -> str: + """Return the Galileo text representation of a structured value.""" + if isinstance(value, str): return value - if output and isinstance(value, Sequence) and not isinstance(value, bytes | bytearray): - return serialize_to_str(value) - return value + if value is None or isinstance(value, bool | int | float): + return json.dumps(value) + return json.dumps(_sdk_json_value(value)) -def documents(value: Any) -> Any: - """Use the SDK retriever helper, including its fallback/error behavior. +def _json_compatible(value: Any) -> Any: + """Convert models and containers into values accepted by Galileo Pydantic models.""" + if isinstance(value, BaseModel): + return value.model_dump(mode="python", exclude_none=True) + if isinstance(value, Mapping): + return {key: _json_compatible(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return [_json_compatible(item) for item in value] + if isinstance(value, UUID | datetime | date | Enum): + return _GalileoJSONEncoder().default(value) + return value - ``splunk_ao`` currently exposes an attrs-based ``Document`` while its - retriever helper consumes the logging document representation. Convert - that public resource model to its public dictionary form before delegating - to the SDK helper; no Galileo core type is imported here. - """ - if isinstance(value, Document): - value = _document_dict(value) - elif isinstance(value, list) and any(isinstance(item, Document) for item in value): - value = [_document_dict(item) if isinstance(item, Document) else item for item in value] - return convert_to_documents(value) - - -def _document_dict(value: Document) -> dict[str, Any]: - """Convert either public SDK Document metadata representation to a dict.""" - metadata = getattr(value, "metadata", None) - metadata_to_dict = getattr(metadata, "to_dict", None) - if callable(metadata_to_dict): - metadata = cast(Any, metadata_to_dict)() - elif isinstance(metadata, Mapping): - metadata = dict(metadata) - else: - metadata = None - - result: dict[str, Any] = {"content": value.content} - if metadata is not None: - result["metadata"] = metadata - return result - - -def string_metadata(value: Any) -> dict[str, str]: - """Apply the SDK's flat metadata conversion.""" + +def _content_part(value: Any) -> dict[str, Any] | None: + """Validate and serialize a public Galileo text or file content part.""" + if isinstance(value, (TextContentPart, FileContentPart)): + return value.model_dump(mode="json", exclude_none=True) if not isinstance(value, Mapping): - return {} - return { - str(key): "None" if item is None else item if isinstance(item, str) else str(item) - for key, item in value.items() - } + return None + try: + part = _CONTENT_PART_ADAPTER.validate_python(dict(value)) + except ValidationError: + return None + return part.model_dump(mode="json", exclude_none=True) -def trace_input_value(value: Any) -> Any: - """Delegate trace-input coercion to the SDK logger implementation.""" - if _is_public_trace_content_parts(value): - return value - return _public_content_blocks(SplunkAOLogger._coerce_trace_input("input", value)) +def _message_content(value: Any) -> Any: + if isinstance(value, Message): + return value.content + if isinstance(value, Mapping) and ("role" in value or "content" in value): + return value.get("content", "") + return None -def trace_output_value(value: Any) -> Any: - """Delegate trace-output coercion to the SDK logger implementation.""" - if value is None: +def _normalize_content_items(value: Any) -> list[dict[str, Any]] | None: + if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray): return None - if _is_public_trace_content_parts(value): - return value - return _public_content_blocks(SplunkAOLogger._coerce_output(value)) + normalized: list[dict[str, Any]] = [] + for item in value: + block = _content_part(item) + if block is None: + return None + normalized.append(block) + return normalized -def _is_public_trace_content_parts(value: Any) -> bool: - """Recognize content parts already emitted by a public ``Trace`` dump.""" - if not isinstance(value, list): - return False - for item in value: - if not isinstance(item, Mapping): - return False - if item.get("type") == "text" and isinstance(item.get("text"), str): - continue - if item.get("type") == "file": - try: - if UUID(str(item["file_id"])).version == 4: - continue - except (KeyError, ValueError, AttributeError): - pass - return False - return True - - -def _public_content_blocks(value: Any) -> Any: - """Convert SDK ingestion blocks to the public model's content-part dicts.""" - if isinstance(value, list): - return [ - item.model_dump(exclude_none=True) if hasattr(item, "model_dump") else item - for item in value - ] - return value +def _message_sequence(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray) and all( + isinstance(item, Message) + or (isinstance(item, Mapping) and ("role" in item or "content" in item)) + for item in value + ) -def session_value(value: Any) -> Any: - """Preserve sequences accepted by the public ``Session`` validator.""" +def _flatten_message_sequence(value: Sequence[Any]) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [] + for message in value: + content = _message_content(message) + if isinstance(content, str): + if content: + blocks.append(TextContentPart(text=content).model_dump(mode="json")) + elif isinstance(content, Sequence) and not isinstance(content, str | bytes | bytearray): + for item in content: + block = _content_part(item) + if block is not None: + blocks.append(block) + elif isinstance(item, str): + blocks.append(TextContentPart(text=item).model_dump(mode="json")) + else: + # Preserve unsupported multimodal payloads instead of dropping them. + blocks.append(TextContentPart(text=_json_text(item)).model_dump(mode="json")) + elif content is not None: + blocks.append(TextContentPart(text=_json_text(content)).model_dump(mode="json")) + return blocks + + +def _normalize_document(value: Any) -> Document: if isinstance(value, Document): - return _document_dict(value) - if isinstance(value, list) and any(isinstance(item, Document) for item in value): - return [_document_dict(item) if isinstance(item, Document) else item for item in value] - return value + return value + if isinstance(value, BaseModel): + value = value.model_dump(mode="python", exclude_none=True) + if isinstance(value, Mapping): + try: + return Document.model_validate(dict(value)) + except ValidationError: + return Document(content=_json_text(value)) + if isinstance(value, str): + return Document(content=value) + return Document(content=_json_text(value)) + + +class GalileoRecordNormalizer: + """Normalize destination fields for public Galileo record models. + + The normalizer owns conversions that differ by record destination. It uses + only public ``galileo-core`` models and preserves values that cannot be + represented natively as serialized text rather than discarding them. + """ + + @staticmethod + def json_text(value: Any) -> str: + """Serialize structured data while preserving strings and SDK JSON semantics.""" + return _json_text(value) + + @staticmethod + def metadata(value: Any) -> dict[str, str]: + """Convert metadata to the flat string mapping required by Galileo records.""" + if not isinstance(value, Mapping): + return {} + return { + str(key): "None" if item is None else item if isinstance(item, str) else str(item) + for key, item in value.items() + } + + @classmethod + def llm_input(cls, value: Any) -> Any: + """Normalize an LLM input without losing any message in a sequence.""" + if value is None: + return "" + if isinstance(value, str | Message): + return value + if isinstance(value, Mapping): + return _json_compatible(value) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + if all(isinstance(item, Message | Mapping) for item in value): + return [_json_compatible(item) for item in value] + return cls.json_text(value) + + @classmethod + def llm_output(cls, value: Any) -> Any: + """Normalize an LLM output, serializing full message sequences as one message.""" + if value is None: + return "" + if isinstance(value, str | Message): + return value + if isinstance(value, Mapping): + return _json_compatible(value) + if _message_sequence(value): + return cls.json_text(value) + return cls.json_text(value) + + @classmethod + def llm_redacted_input(cls, value: Any) -> Any: + return None if value is None else cls.llm_input(value) + + @classmethod + def llm_redacted_output(cls, value: Any) -> Any: + return None if value is None else cls.llm_output(value) + + @staticmethod + def llm_tools(value: Any) -> Any: + """Normalize tool definitions nested in an LLM span.""" + return None if value is None else _sdk_json_value(value) + + @classmethod + def tool_input(cls, value: Any) -> str: + """Serialize tool input as text, using an empty string for missing input.""" + return "" if value is None else cls.json_text(value) + + @classmethod + def tool_output(cls, value: Any) -> str | None: + """Serialize a present tool output as text and preserve missing output.""" + return None if value is None else cls.json_text(value) + + @classmethod + def tool_redacted_input(cls, value: Any) -> str | None: + return None if value is None else cls.json_text(value) + + @classmethod + def tool_redacted_output(cls, value: Any) -> str | None: + return None if value is None else cls.json_text(value) + + @classmethod + def retriever_input(cls, value: Any) -> str: + return "" if value is None else cls.json_text(value) + + @staticmethod + def retriever_output(value: Any) -> list[Document]: + """Convert supported retrieval values into canonical Galileo Documents.""" + if value is None: + return [Document(content="")] + if isinstance(value, Document): + return [value] + if isinstance(value, BaseModel): + return [_normalize_document(value)] + if isinstance(value, str): + return [Document(content=value)] + if isinstance(value, Mapping): + return [_normalize_document(value)] + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + items = list(value) + if not items: + return [] + if all(isinstance(item, Document) for item in items): + return list(items) + if all(isinstance(item, str) for item in items): + return [Document(content=item) for item in items] + if all(isinstance(item, Mapping) for item in items): + return [_normalize_document(item) for item in items] + if all(isinstance(item, BaseModel) for item in items): + return [_normalize_document(item) for item in items] + raise ValueError( + "Invalid document output. Expected a list of strings, dictionaries, or Documents." + ) + return [Document(content="")] + + @classmethod + def retriever_redacted_input(cls, value: Any) -> str | None: + return None if value is None else cls.json_text(value) + + @classmethod + def retriever_redacted_output(cls, value: Any) -> list[Document] | None: + return None if value is None else cls.retriever_output(value) + + @classmethod + def trace_input(cls, value: Any) -> Any: + """Normalize trace input to a string or validated public content parts.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, Mapping): + return cls.json_text(value) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + normalized = _normalize_content_items(value) + if normalized is not None: + return normalized + if all(isinstance(item, Mapping) for item in value): + return cls.json_text(value) + raise TypeError("Trace input must be text, a mapping, or a list of content blocks.") + raise TypeError(f"Trace input does not support {type(value).__name__} values.") + + @classmethod + def trace_output(cls, value: Any) -> Any: + """Normalize trace output and flatten message sequences to content parts.""" + if value is None or isinstance(value, str): + return value + if _message_sequence(value): + return _flatten_message_sequence(value) + normalized = _normalize_content_items(value) + if normalized is not None: + return normalized + return cls.json_text(value) + + @classmethod + def trace_redacted_input(cls, value: Any) -> Any: + return None if value is None else cls.trace_input(value) + + @classmethod + def trace_redacted_output(cls, value: Any) -> Any: + return None if value is None else cls.trace_output(value) + + @classmethod + def session_input(cls, value: Any) -> Any: + """Normalize the session input variants accepted by the public model.""" + if value is None or isinstance(value, str | Message): + return value + if isinstance(value, TextContentPart | FileContentPart): + return [value] + if isinstance(value, Document): + return cls.json_text(value) + if isinstance(value, BaseModel): + return value + if isinstance(value, Mapping): + if "role" in value: + return _json_compatible(value) + part = _content_part(value) + if part is not None: + return [part] + return cls.json_text(value) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + if not value: + return [] + if all(isinstance(item, Message) for item in value): + return list(value) + if all(isinstance(item, Mapping) for item in value): + if all("role" in item for item in value): + return [_json_compatible(item) for item in value] + content_parts = _normalize_content_items(value) + if content_parts is not None: + return content_parts + if all(isinstance(item, BaseModel) for item in value): + return cls.json_text(value) + return [_json_compatible(item) for item in value] + return cls.json_text(value) + + @classmethod + def session_output(cls, value: Any) -> Any: + """Normalize session output while preserving messages, documents and parts.""" + if value is None or isinstance(value, str | Message): + return value + if isinstance(value, Document): + return [value] + if isinstance(value, TextContentPart | FileContentPart): + return [value] + if isinstance(value, BaseModel): + return value + if isinstance(value, Mapping): + if "role" in value: + return _json_compatible(value) + if "content" in value or "page_content" in value: + return _normalize_document(value) + part = _content_part(value) + if part is not None: + return [part] + return cls.json_text(value) + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + if not value: + return [] + if all(isinstance(item, Document) for item in value): + return list(value) + if all(isinstance(item, Mapping) and "role" in item for item in value): + return _flatten_message_sequence(value) + if all( + isinstance(item, Mapping) and ("content" in item or "page_content" in item) + for item in value + ): + return [_normalize_document(item) for item in value] + if _message_sequence(value): + return _flatten_message_sequence(value) + content_parts = _normalize_content_items(value) + if content_parts is not None: + return content_parts + if all(isinstance(item, Mapping) for item in value): + return [_json_compatible(item) for item in value] + return [_json_compatible(item) for item in value] + return cls.json_text(value) + + @classmethod + def session_redacted_input(cls, value: Any) -> Any: + return None if value is None else cls.session_input(value) + + @classmethod + def session_redacted_output(cls, value: Any) -> Any: + return None if value is None else cls.session_output(value) + + @staticmethod + def session_traces( + values: Sequence[Any], + *, + normalize_record: Callable[[Any], Any], + ) -> list[Any]: + """Normalize every nested session trace through the shared record path.""" + return [normalize_record(value) for value in values] diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index dd489fdf..b4086919 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -8,6 +8,7 @@ import pytest from agent_control_evaluator_galileo.records import ( + GalileoRecordNormalizer, RecordFactoryError, UnsupportedStepTypeError, build_galileo_record, @@ -16,10 +17,13 @@ record_from_step, ) from agent_control_models import Step +from galileo_core.schemas.logging.llm import Message +from galileo_core.schemas.logging.session import Session +from galileo_core.schemas.logging.span import LlmSpan, RetrieverSpan, ToolSpan +from galileo_core.schemas.logging.trace import Trace +from galileo_core.schemas.shared.content_parts import FileContentPart, TextContentPart +from galileo_core.schemas.shared.document import Document from pydantic import BaseModel, ConfigDict -from splunk_ao import Document, LlmSpan, Message, RetrieverSpan, Session, ToolSpan, Trace -from splunk_ao.utils.retrievers import convert_to_documents -from splunk_ao.utils.serialization import serialize_to_str class _RecordPayload(BaseModel): @@ -37,7 +41,7 @@ class _RecordPayload(BaseModel): tool_call_id: object | None = None -def test_required_splunk_ao_public_exports_are_importable() -> None: +def test_required_galileo_core_public_exports_are_importable() -> None: assert all((LlmSpan, ToolSpan, RetrieverSpan, Trace, Session, Document, Message)) @@ -55,7 +59,7 @@ def test_llm_messages_use_public_canonical_validation() -> None: assert isinstance(record, LlmSpan) assert record.input[0].role.value == "user" assert record.output.role.value == "assistant" - assert record.dataset_output == serialize_to_str({"expected": "answer"}) + assert record.dataset_output == json.dumps({"expected": "answer"}) @pytest.mark.parametrize( @@ -101,7 +105,7 @@ def test_llm_tuple_output_uses_the_same_sdk_serializer() -> None: record = record_from_step(Step(type="llm", name="answer", input="question", output=output)) assert isinstance(record, LlmSpan) - assert record.output.content == serialize_to_str(output) + assert record.output.content == json.dumps(list(output)) assert json.loads(record.output.content) == list(output) @@ -133,8 +137,8 @@ def test_tool_values_are_json_strings_and_missing_output_stays_missing() -> None expected = ToolSpan( name="search", - input=serialize_to_str({"query": "q"}), - output=serialize_to_str({"hits": [1, 2]}), + input=json.dumps({"query": "q"}), + output=json.dumps({"hits": [1, 2]}), ) assert type(record) is type(expected) assert record.input == expected.input @@ -146,21 +150,20 @@ def test_tool_values_are_json_strings_and_missing_output_stays_missing() -> None [None, "text", {"content": "document"}, {"invalid": True}, 42, ["one", "two"]], ) def test_retriever_matches_sdk_document_coercion(output: object) -> None: - expected = convert_to_documents(output) - - try: - record = record_from_step( - Step(type="retriever", name="retrieve", input="question", output=output) - ) - except (TypeError, ValueError): - with pytest.raises((TypeError, ValueError)): - RetrieverSpan(input="question", output=expected) - return - + record = record_from_step( + Step(type="retriever", name="retrieve", input="question", output=output) + ) assert isinstance(record, RetrieverSpan) - assert [document.model_dump() for document in record.output] == [ - document.model_dump() for document in expected - ] + if output is None or isinstance(output, int): + assert [document.content for document in record.output] == [""] + elif isinstance(output, str): + assert [document.content for document in record.output] == [output] + elif isinstance(output, dict) and "content" in output: + assert [document.content for document in record.output] == [output["content"]] + elif output == ["one", "two"]: + assert [document.content for document in record.output] == output + else: + assert record.output[0].content == json.dumps(output) def test_retriever_rejects_mixed_lists_like_the_sdk_helper() -> None: @@ -190,7 +193,7 @@ def test_retriever_results_become_documents_with_scalar_metadata_only() -> None: assert record.output[0].metadata == {"score": 0.9, "source": "kb"} -def test_retriever_accepts_the_public_splunk_ao_document_model() -> None: +def test_retriever_accepts_the_public_galileo_core_document_model() -> None: from pydantic import BaseModel document = Document(content="context", metadata={"source": "kb"}) @@ -210,7 +213,7 @@ class RecordInput(BaseModel): def test_public_document_metadata_variants_are_supported() -> None: document_without_metadata = Document(content="plain") - document_with_model_metadata = Document.from_dict( + document_with_model_metadata = Document.model_validate( {"content": "model metadata", "metadata": {"source": "kb"}} ) @@ -406,8 +409,8 @@ def test_malformed_trace_content_parts_follow_sdk_serialization() -> None: assert isinstance(scalar_item, Trace) assert isinstance(invalid_file, Trace) - assert scalar_item.output == serialize_to_str([1]) - assert invalid_file.output == serialize_to_str([{"type": "file", "file_id": "not-a-uuid"}]) + assert scalar_item.output == json.dumps([1]) + assert invalid_file.output == json.dumps([{"type": "file", "file_id": "not-a-uuid"}]) def test_nested_record_errors_and_existing_models_are_explicit() -> None: @@ -469,14 +472,9 @@ def test_invalid_nested_context_shapes_are_rejected() -> None: ) -def test_trace_structured_values_match_sdk_logger_coercion() -> None: - from splunk_ao import SplunkAOLogger - +def test_trace_structured_values_are_normalized_without_losing_messages() -> None: trace_input = {"question": "hello"} trace_output = [{"role": "assistant", "content": "answer"}] - expected_input = SplunkAOLogger._coerce_trace_input("input", trace_input) - expected_output = SplunkAOLogger._coerce_output(trace_output) - record = record_from_step( Step( type="trace", @@ -488,8 +486,161 @@ def test_trace_structured_values_match_sdk_logger_coercion() -> None: ) assert isinstance(record, Trace) - assert record.input == expected_input - assert [block.text for block in record.output] == [block.text for block in expected_output] + assert record.input == json.dumps(trace_input) + assert [block.text for block in record.output] == ["answer"] + + +def test_trace_content_blocks_preserve_text_files_and_unrepresentable_data() -> None: + file_id = uuid4() + blocks = [ + {"type": "text", "text": "look at this"}, + {"type": "file", "file_id": str(file_id)}, + ] + trace = record_from_step( + Step( + type="trace", + name="multimodal", + input=blocks, + output=blocks, + context={"spans": []}, + ) + ) + serialized_data_block = { + "type": "data", + "modality": "image", + "url": "https://example/image.png", + } + data_trace = record_from_step( + Step( + type="trace", + name="inline-image", + input=[serialized_data_block], + output=[serialized_data_block], + context={"spans": []}, + ) + ) + + assert isinstance(trace, Trace) + assert isinstance(trace.input[0], TextContentPart) + assert isinstance(trace.input[1], FileContentPart) + assert trace.input[1].file_id == file_id + assert isinstance(trace.output[0], TextContentPart) + assert isinstance(trace.output[1], FileContentPart) + assert isinstance(data_trace, Trace) + assert json.loads(data_trace.input) == [serialized_data_block] + assert json.loads(data_trace.output or "") == [serialized_data_block] + + +def test_trace_message_sequences_flatten_every_text_and_content_part() -> None: + file_id = uuid4() + output = [ + {"role": "assistant", "content": "first message"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "second message"}, + {"type": "file", "file_id": str(file_id)}, + ], + }, + ] + + trace = record_from_step( + Step(type="trace", name="messages", input="question", output=output, context={"spans": []}) + ) + + assert isinstance(trace, Trace) + assert [part.text for part in trace.output if isinstance(part, TextContentPart)] == [ + "first message", + "second message", + ] + file_parts = [part for part in trace.output if isinstance(part, FileContentPart)] + assert len(file_parts) == 1 + assert file_parts[0].file_id == file_id + + +def test_record_normalizer_defines_none_and_empty_value_behavior() -> None: + assert GalileoRecordNormalizer.llm_input(None) == "" + assert GalileoRecordNormalizer.llm_output(None) == "" + assert GalileoRecordNormalizer.tool_input(None) == "" + assert GalileoRecordNormalizer.tool_output(None) is None + assert GalileoRecordNormalizer.retriever_output(None)[0].content == "" + assert GalileoRecordNormalizer.retriever_output([]) == [] + assert GalileoRecordNormalizer.trace_input(None) == "" + assert GalileoRecordNormalizer.trace_output(None) is None + assert GalileoRecordNormalizer.trace_output([]) == [] + assert GalileoRecordNormalizer.session_input(None) is None + assert GalileoRecordNormalizer.session_output(None) is None + assert GalileoRecordNormalizer.session_input([]) == [] + assert GalileoRecordNormalizer.session_output([]) == [] + + +def test_session_content_parts_are_preserved_by_input_and_output_normalizers() -> None: + parts = [TextContentPart(text="text"), FileContentPart(file_id=uuid4())] + + assert GalileoRecordNormalizer.session_input(parts) == [ + part.model_dump(mode="json") for part in parts + ] + assert GalileoRecordNormalizer.session_output(parts) == [ + part.model_dump(mode="json") for part in parts + ] + + +def test_record_normalizer_serializes_nested_models_uuids_and_timestamps() -> None: + identifier = uuid4() + timestamp = datetime(2025, 1, 2, 3, 4, 5, tzinfo=UTC) + + class NestedPayload(BaseModel): + identifier: object + timestamp: datetime + + payload = { + "payload": NestedPayload(identifier=identifier, timestamp=timestamp), + "uuid": identifier, + "timestamp": timestamp, + } + serialized = GalileoRecordNormalizer.tool_output(payload) + + assert serialized is not None + parsed = json.loads(serialized) + assert parsed == { + "payload": { + "identifier": str(identifier), + "timestamp": timestamp.isoformat().replace("+00:00", "Z"), + }, + "uuid": str(identifier), + "timestamp": timestamp.isoformat().replace("+00:00", "Z"), + } + + +def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values() -> None: + identifier = uuid4() + timestamp = datetime(2025, 1, 2, tzinfo=UTC) + source = _RecordPayload( + type="llm", + input="question", + tools=[{"id": identifier, "created_at": timestamp}], + ) + + record = record_from_scorer_invoke_record(source) + + assert isinstance(record, LlmSpan) + assert record.tools == [ + {"id": str(identifier), "created_at": timestamp.isoformat().replace("+00:00", "Z")} + ] + + +def test_serialization_and_document_conversion_match_sdk_when_available() -> None: + sdk_serialization = pytest.importorskip("splunk_ao.utils.serialization") + sdk_retrievers = pytest.importorskip("splunk_ao.utils.retrievers") + payload = {"uuid": uuid4(), "timestamp": datetime(2025, 1, 2, tzinfo=UTC)} + documents = [{"content": "document", "metadata": {"source": "kb"}}] + + assert GalileoRecordNormalizer.tool_output(payload) == sdk_serialization.serialize_to_str( + payload + ) + assert GalileoRecordNormalizer.retriever_output( + documents + ) == sdk_retrievers.convert_to_documents(documents) def test_trace_and_session_envelopes_are_supported() -> None: @@ -660,7 +811,7 @@ def test_record_serialization_is_json_safe() -> None: payload = record.model_dump(mode="json", exclude_none=True) assert payload["type"] == "tool" - assert payload["input"] == serialize_to_str({"q": "x"}) + assert payload["input"] == json.dumps({"q": "x"}) def test_factory_accepts_the_existing_pydantic_luna_record() -> None: @@ -675,12 +826,16 @@ def test_factory_accepts_the_existing_pydantic_luna_record() -> None: def test_factory_rejects_invalid_boundaries_and_normalizes_missing_text() -> None: - from agent_control_evaluator_galileo.records.normalization import session_value, text_value + from agent_control_evaluator_galileo.records import GalileoRecordNormalizer with pytest.raises(RecordFactoryError, match="complete Agent Control Step"): record_from_step(object()) # type: ignore[arg-type] with pytest.raises(UnsupportedStepTypeError, match="unsupported"): record_from_scorer_invoke_record(_RecordPayload(type="unsupported")) - assert text_value(None) == "" - assert session_value(Document(content="plain")) == {"content": "plain"} - assert session_value([Document(content="plain")]) == [{"content": "plain"}] + assert GalileoRecordNormalizer.retriever_input(None) == "" + assert GalileoRecordNormalizer.session_input(Document(content="plain")) == json.dumps( + {"content": "plain"} + ) + assert GalileoRecordNormalizer.session_input([Document(content="plain")]) == json.dumps( + [{"content": "plain"}] + ) From 50ab93f5ea87c9147104259ed6cda24be9e33ccf Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Thu, 24 Sep 2026 16:06:42 -0700 Subject: [PATCH 03/17] test(galileo): remove SDK parity imports --- .../galileo/tests/test_records_factory.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index b4086919..a57f45a3 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -594,22 +594,20 @@ class NestedPayload(BaseModel): timestamp: datetime payload = { + "nested": {"answer": "42"}, "payload": NestedPayload(identifier=identifier, timestamp=timestamp), "uuid": identifier, "timestamp": timestamp, } serialized = GalileoRecordNormalizer.tool_output(payload) - assert serialized is not None - parsed = json.loads(serialized) - assert parsed == { - "payload": { - "identifier": str(identifier), - "timestamp": timestamp.isoformat().replace("+00:00", "Z"), - }, - "uuid": str(identifier), - "timestamp": timestamp.isoformat().replace("+00:00", "Z"), - } + assert serialized == ( + '{"nested": {"answer": "42"}, "payload": {"identifier": "' + + str(identifier) + + '", "timestamp": "2025-01-02T03:04:05Z"}, "uuid": "' + + str(identifier) + + '", "timestamp": "2025-01-02T03:04:05Z"}' + ) def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values() -> None: @@ -629,18 +627,15 @@ def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values ] -def test_serialization_and_document_conversion_match_sdk_when_available() -> None: - sdk_serialization = pytest.importorskip("splunk_ao.utils.serialization") - sdk_retrievers = pytest.importorskip("splunk_ao.utils.retrievers") - payload = {"uuid": uuid4(), "timestamp": datetime(2025, 1, 2, tzinfo=UTC)} - documents = [{"content": "document", "metadata": {"source": "kb"}}] - - assert GalileoRecordNormalizer.tool_output(payload) == sdk_serialization.serialize_to_str( - payload +def test_retriever_dictionaries_become_canonical_documents() -> None: + documents = GalileoRecordNormalizer.retriever_output( + [{"content": "document", "metadata": {"source": "kb"}}] ) - assert GalileoRecordNormalizer.retriever_output( - documents - ) == sdk_retrievers.convert_to_documents(documents) + + assert len(documents) == 1 + assert isinstance(documents[0], Document) + assert documents[0].content == "document" + assert documents[0].metadata == {"source": "kb"} def test_trace_and_session_envelopes_are_supported() -> None: From 4ce13fb8dad601a8a47c5f7bd029706f552d5217 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Sun, 27 Sep 2026 11:02:48 -0700 Subject: [PATCH 04/17] support tools in control decorator --- sdks/python/README.md | 57 +++++ sdks/python/src/agent_control/__init__.py | 1 + .../src/agent_control/control_decorators.py | 66 +++++- sdks/python/tests/test_control_decorators.py | 211 +++++++++++++++++- sdks/python/tests/test_evaluation.py | 49 ++++ 5 files changed, 372 insertions(+), 12 deletions(-) diff --git a/sdks/python/README.md b/sdks/python/README.md index f9a608b0..46dd9ce2 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -25,6 +25,63 @@ async def search_database(query: str) -> str: Controls are defined centrally and enforced automatically at runtime. See [Python SDK Documentation](https://docs.agentcontrol.dev/sdk/python-sdk) for complete reference. +## Supplying available tools to an LLM control + +The `@control` decorator is a convenience path for function-based steps. An LLM +control can include static tool definitions, or a provider can build them from +the function arguments for each pre- and post-execution check: + +```python +AVAILABLE_TOOLS = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search internal documents", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } +] + +@agent_control.control(step_type="llm", tools=AVAILABLE_TOOLS) +async def answer(message: str) -> str: + ... +``` + +For per-call tools, pass a provider such as +`tools=lambda args, kwargs: tools_for(kwargs.get("model"))`. The provider may +return `None` when the step has no available tools. + +For framework adapters and advanced callers, construct the shared `Step` +directly and send it through the SDK's `check_evaluation` helper. This keeps +framework-specific message and tool formats in the adapter, without requiring +the decorator or a framework dependency: + +```python +step = agent_control.Step( + type="llm", + name="openai_agent", + input=messages, + output=response, + tools=available_tools, +) + +async with agent_control.AgentControlClient() as client: + result = await agent_control.evaluation.check_evaluation( + client, + agent_name="my-agent", + step=step, + stage="post", + ) +``` + +The existing `evaluate_controls` helper remains available for callers that +prefer its field-based convenience arguments. + ## Sharing an OpenTelemetry provider with Google ADK When Google ADK and Agent Control should export through the same OpenTelemetry diff --git a/sdks/python/src/agent_control/__init__.py b/sdks/python/src/agent_control/__init__.py index 6c6e117f..04d040f6 100644 --- a/sdks/python/src/agent_control/__init__.py +++ b/sdks/python/src/agent_control/__init__.py @@ -1650,6 +1650,7 @@ async def main(): # Models (re-exported when available) "Agent", "Step", + "JSONObject", "StepSchema", "EvaluationRequest", "EvaluationResult", diff --git a/sdks/python/src/agent_control/control_decorators.py b/sdks/python/src/agent_control/control_decorators.py index fee78333..6c96b674 100644 --- a/sdks/python/src/agent_control/control_decorators.py +++ b/sdks/python/src/agent_control/control_decorators.py @@ -31,11 +31,12 @@ async def chat(message: str) -> str: import functools import inspect import time -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy from dataclasses import dataclass, field from typing import Any, TypeVar -from agent_control_models import Step, normalize_action +from agent_control_models import JSONObject, Step, normalize_action from agent_control_telemetry import get_trace_context_from_provider from agent_control import AgentControlClient @@ -58,6 +59,10 @@ async def chat(message: str) -> str: logger = get_logger(__name__) F = TypeVar("F", bound=Callable[..., Any]) +type ToolsProvider = Callable[ + [tuple[Any, ...], Mapping[str, Any]], Sequence[JSONObject] | None +] +type ToolsConfiguration = Sequence[JSONObject] | ToolsProvider | None def _resolve_control_trace_context() -> tuple[str, str]: @@ -98,6 +103,7 @@ class ControlContext: start_time: float step_name: str | None = None step_type: str | None = None + tools: ToolsConfiguration = None # Stats (mutually exclusive: errors vs matches vs non_matches) total_executions: int = 0 @@ -134,6 +140,7 @@ def pre_payload(self) -> dict[str, Any]: output=None, step_name=self.step_name, explicit_step_type=self.step_type, + tools=self.tools, ) def post_payload(self, output: Any) -> dict[str, Any]: @@ -145,6 +152,7 @@ def post_payload(self, output: Any) -> dict[str, Any]: output=output, step_name=self.step_name, explicit_step_type=self.step_type, + tools=self.tools, ) def process_result(self, result: dict[str, Any], check_stage: str) -> None: @@ -247,6 +255,10 @@ def __init__( ) +class _InvalidToolsConfigurationError(ValueError): + """Raised when decorator tools are configured for a non-LLM step.""" + + def _get_current_agent() -> Any | None: """Get the current agent from agent_control module.""" try: @@ -433,11 +445,13 @@ def _unexpected_control_failure_message(stage: str, error: Exception) -> str: async def _run_control_check( ctx: ControlContext, stage: str, - payload: dict[str, Any], + payload: dict[str, Any] | Callable[[], dict[str, Any]], controls: list[dict[str, Any]] | None, ) -> None: """Run one control stage and enforce fail-closed behavior on unexpected errors.""" try: + if callable(payload): + payload = payload() result = await _evaluate( ctx.agent_name, payload, @@ -451,6 +465,8 @@ async def _run_control_check( ctx.process_result(result, stage) except (ControlViolationError, ControlSteerError): raise + except _InvalidToolsConfigurationError: + raise except Exception as e: stage_name = "Pre" if stage == "pre" else "Post" logger.error("%s-execution control check failed: %s", stage_name, e, exc_info=True) @@ -492,6 +508,7 @@ def _create_evaluation_payload( output: Any = None, step_name: str | None = None, explicit_step_type: str | None = None, + tools: ToolsConfiguration = None, ) -> dict[str, Any]: """ Create evaluation payload for server, detecting if it's a tool step or LLM step. @@ -505,6 +522,7 @@ def _create_evaluation_payload( output: Function output (None for pre-execution) step_name: Optional explicit step name to override auto-detection explicit_step_type: Optional explicit step type to override inference + tools: Optional static tool definitions or a provider called with args/kwargs """ sig = inspect.signature(func) bound = sig.bind(*args, **kwargs) @@ -536,33 +554,45 @@ def _create_evaluation_payload( else inferred_type ) + if tools is not None and step_type != "llm": + raise _InvalidToolsConfigurationError( + f"@control(tools=...) requires an LLM step; received step_type={step_type!r}." + ) + + resolved_tools = tools(args, kwargs) if callable(tools) else tools + + def validate_payload(payload: dict[str, Any]) -> dict[str, Any]: + """Validate decorator payloads against the shared runtime Step model.""" + Step.model_validate(payload) + return payload + if step_type not in ("llm", "tool"): # Custom step types use the complete bound argument mapping. Unlike an # LLM step, there is no generic convention for selecting one argument # as the prompt/input for a retriever, trace, session, etc. - return { + return validate_payload({ "type": step_type, "name": determined_name, "input": dict(bound.arguments), "output": output if isinstance(output, (str, int, float, bool, dict, list)) else ( None if output is None else str(output) ), - } + }) if step_type == "tool": # This is a tool step - return { + return validate_payload({ "type": step_type, "name": determined_name, "input": dict(bound.arguments), "output": output if isinstance(output, (str, int, float, bool, dict, list)) else ( None if output is None else str(output) ), - } + }) # This is an LLM step input_data = _extract_input_from_args(func, args, kwargs) - return { + payload = { "type": step_type, "name": determined_name, "input": input_data, @@ -570,6 +600,9 @@ def _create_evaluation_payload( None if output is None else str(output) ), } + if resolved_tools is not None: + payload["tools"] = deepcopy(list(resolved_tools)) + return validate_payload(payload) def _handle_evaluation_result(result: dict[str, Any]) -> None: @@ -733,6 +766,7 @@ async def _execute_with_control( is_async: bool, step_name: str | None = None, step_type: str | None = None, + tools: ToolsConfiguration = None, ) -> Any: """ Core control execution logic for both async and sync functions. @@ -787,12 +821,13 @@ async def _execute_with_control( start_time=time.perf_counter(), step_name=step_name, step_type=step_type, + tools=tools, ) ctx.log_start() try: # PRE-EXECUTION: Check controls with check_stage="pre" - await _run_control_check(ctx, "pre", ctx.pre_payload(), controls) + await _run_control_check(ctx, "pre", ctx.pre_payload, controls) # Execute the function if is_async: @@ -801,7 +836,7 @@ async def _execute_with_control( output = func(*args, **kwargs) # POST-EXECUTION: Check controls with check_stage="post" - await _run_control_check(ctx, "post", ctx.post_payload(output), controls) + await _run_control_check(ctx, "post", lambda: ctx.post_payload(output), controls) return output finally: @@ -812,6 +847,10 @@ def control( policy: str | None = None, step_name: str | None = None, step_type: str | None = None, + *, + tools: Sequence[JSONObject] | Callable[ + [tuple[Any, ...], Mapping[str, Any]], Sequence[JSONObject] | None + ] | None = None, ) -> Callable[[F], F]: """ Decorator to apply server-defined controls at this code location. @@ -827,6 +866,9 @@ def control( step_type: Optional custom type for this step. If not provided, tool-like functions are classified as ``tool`` and other functions as ``llm``. + tools: Optional tool definitions for LLM steps. Static definitions are + copied when the decorator is created. A provider receives the + function's positional and keyword arguments at each check. Returns: Decorated function @@ -885,6 +927,8 @@ async def handle_user_input(user_message: str) -> str: # controls associated with the agent via policy and direct links. _ = policy + configured_tools = deepcopy(list(tools)) if tools is not None and not callable(tools) else tools + def decorator(func: F) -> F: # Register this function's step schema for auto-discovery by init() from agent_control._control_registry import register @@ -900,6 +944,7 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: is_async=True, step_name=step_name, step_type=step_type, + tools=configured_tools, ) # Copy over ALL attributes from the original function (important for LangChain tools) @@ -920,6 +965,7 @@ def sync_wrapper(*args: Any, **kwargs: Any) -> Any: is_async=False, step_name=step_name, step_type=step_type, + tools=configured_tools, ) ) diff --git a/sdks/python/tests/test_control_decorators.py b/sdks/python/tests/test_control_decorators.py index 808c30c4..c883b03f 100644 --- a/sdks/python/tests/test_control_decorators.py +++ b/sdks/python/tests/test_control_decorators.py @@ -1,12 +1,13 @@ """Tests for @control decorator.""" +from copy import deepcopy from unittest.mock import MagicMock, patch import httpx import pytest -from agent_control_telemetry import clear_trace_context_provider, set_trace_context_provider - from agent_control.control_decorators import ControlSteerError, ControlViolationError, control +from agent_control_telemetry import clear_trace_context_provider, set_trace_context_provider +from pydantic import ValidationError # ============================================================================= # FIXTURES @@ -218,6 +219,212 @@ async def retrieve(query: str, top_k: int, filters: dict[str, str]) -> str: "filters": {"language": "en"}, } + @pytest.mark.asyncio + async def test_omitted_tools_preserves_legacy_payload(self, mock_agent, mock_safe_response): + payloads = [] + + async def capture_payload(*args, **kwargs): + payloads.append(args[1]) + return mock_safe_response + + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate", side_effect=capture_payload), + ): + + @control() + async def answer(message: str) -> str: + return "answer" + + await answer("question") + + assert payloads == [ + {"type": "llm", "name": "answer", "input": "question", "output": None}, + {"type": "llm", "name": "answer", "input": "question", "output": "answer"}, + ] + + @pytest.mark.asyncio + async def test_static_tools_are_copied_into_pre_and_post_payloads( + self, mock_agent, mock_safe_response + ): + original_tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search documents", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + expected_tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search documents", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + payloads = [] + + async def capture_payload(*args, **kwargs): + payload = args[1] + payloads.append( + { + "type": payload["type"], + "name": payload["name"], + "input": payload["input"], + "output": payload["output"], + "tools": deepcopy(payload["tools"]), + } + ) + payload["tools"][0]["function"]["parameters"]["properties"]["query"][ + "type" + ] = "integer" + return mock_safe_response + + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate", side_effect=capture_payload), + ): + @control(tools=original_tools) + async def answer(message: str) -> str: + return "answer" + + original_tools[0]["function"]["description"] = "caller mutation" + await answer("question") + + assert [payload["tools"] for payload in payloads] == [expected_tools, expected_tools] + assert original_tools[0]["function"]["description"] == "caller mutation" + assert original_tools[0]["function"]["parameters"]["properties"]["query"][ + "type" + ] == "string" + + @pytest.mark.asyncio + async def test_dynamic_tools_receive_call_arguments(self, mock_agent, mock_safe_response): + calls = [] + payloads = [] + available_tools = [{"name": "lookup", "parameters": {"type": "object"}}] + + def provide_tools(args, kwargs): + calls.append((args, dict(kwargs))) + return available_tools + + async def capture_payload(*args, **kwargs): + payloads.append(args[1]) + return mock_safe_response + + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate", side_effect=capture_payload), + ): + + @control(tools=provide_tools) + async def answer(message: str, *, language: str) -> str: + return "answer" + + await answer("question", language="en") + + assert calls == [ + (("question",), {"language": "en"}), + (("question",), {"language": "en"}), + ] + assert payloads[0]["tools"] == available_tools + assert payloads[1]["tools"] == available_tools + + @pytest.mark.asyncio + async def test_dynamic_tool_provider_can_return_none(self, mock_agent, mock_safe_response): + payloads = [] + + async def capture_payload(*args, **kwargs): + payloads.append(args[1]) + return mock_safe_response + + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate", side_effect=capture_payload), + ): + + @control(tools=lambda args, kwargs: None) + async def answer(message: str) -> str: + return "answer" + + await answer("question") + + assert all("tools" not in payload for payload in payloads) + + @pytest.mark.asyncio + async def test_invalid_tool_definitions_use_step_validation( + self, mock_agent + ): + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate") as mock_evaluate, + ): + + @control(tools=[{"name": "invalid", "schema": object()}]) + async def answer(message: str) -> str: + return "should not run" + + with pytest.raises(RuntimeError) as exc_info: + await answer("question") + + assert isinstance(exc_info.value.__cause__, ValidationError) + assert "tools" in str(exc_info.value.__cause__) + mock_evaluate.assert_not_called() + + @pytest.mark.parametrize("step_type", ["tool", "retriever"]) + @pytest.mark.asyncio + async def test_tools_are_rejected_for_non_llm_steps( + self, mock_agent, step_type + ): + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate") as mock_evaluate, + ): + + @control(step_type=step_type, tools=[]) + async def answer(message: str) -> str: + return "should not run" + + with pytest.raises(ValueError, match="requires an LLM step"): + await answer("question") + + mock_evaluate.assert_not_called() + + @pytest.mark.asyncio + async def test_dynamic_tool_provider_failure_blocks_execution(self, mock_agent): + executed = False + + def provider(args, kwargs): + raise ValueError("provider failed") + + with ( + patch("agent_control.control_decorators._get_current_agent", return_value=mock_agent), + patch("agent_control.control_decorators._evaluate") as mock_evaluate, + ): + + @control(tools=provider) + async def answer(message: str) -> str: + nonlocal executed + executed = True + return "answer" + + with pytest.raises(RuntimeError, match="Execution blocked for safety") as exc_info: + await answer("question") + + assert str(exc_info.value.__cause__) == "provider failed" + assert executed is False + mock_evaluate.assert_not_called() + def test_explicit_step_type_overrides_tool_inference( self, mock_agent, mock_safe_response ): diff --git a/sdks/python/tests/test_evaluation.py b/sdks/python/tests/test_evaluation.py index d598c118..7f41d5f8 100644 --- a/sdks/python/tests/test_evaluation.py +++ b/sdks/python/tests/test_evaluation.py @@ -6,6 +6,7 @@ import pytest from agent_control import evaluation from agent_control.evaluation import EvaluationResult +from agent_control_models import Step from pydantic import ValidationError @@ -107,6 +108,54 @@ def json(self) -> dict[str, object]: } +@pytest.mark.asyncio +async def test_check_evaluation_accepts_step_with_tool_definitions(): + """Framework callers can send a fully structured Step directly.""" + + class DummyResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"is_safe": True, "confidence": 1.0} + + client = MagicMock() + client.http_client = MagicMock() + client.http_client.post = AsyncMock(return_value=DummyResponse()) + tools = [ + { + "type": "function", + "function": { + "name": "search", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + step = Step( + type="llm", + name="openai_agent", + input=[{"role": "user", "content": "question"}], + output={"role": "assistant", "content": "answer"}, + tools=tools, + ) + + await evaluation.check_evaluation( + client=client, + agent_name="Agent-Example_01", + step=step, + stage="post", + ) + + sent_step = client.http_client.post.await_args.kwargs["json"]["step"] + assert sent_step["tools"] == tools + assert sent_step["tools"][0]["function"]["parameters"]["properties"]["query"] == { + "type": "string" + } + + @pytest.mark.asyncio async def test_evaluate_controls_requires_server_url(): """evaluate_controls should require server_url to be configured.""" From 79aafd819951910a05752c30aec4c09ccde49ed6 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Sun, 27 Sep 2026 18:30:48 -0700 Subject: [PATCH 05/17] fix evaluator --- .../luna/evaluator.py | 40 ++- .../galileo/tests/test_luna_evaluator.py | 273 +++++++++++++++++- 2 files changed, 305 insertions(+), 8 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index f8d3c15c..1d3b21a5 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -50,6 +50,16 @@ def _has_text(value: str | None) -> bool: return value is not None and value.strip() != "" +def _has_payload_value(value: JSONValue | None) -> bool: + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, (dict, list)): + return bool(value) + return True + + def _extract_dict_text(data: dict[str, Any], key: str) -> str | None: if key not in data: return None @@ -145,8 +155,20 @@ def _get_client(self) -> GalileoLunaClient: """Get the Galileo Luna client.""" return self._client - def _prepare_payload(self, data: Any) -> tuple[str | None, str | None]: - """Prepare scorer input/output fields from selected data.""" + def _prepare_payload( + self, data: Any, *, step: Step | None = None + ) -> tuple[JSONValue | None, JSONValue | None]: + """Prepare legacy scorer inputs while preserving structured LLM values.""" + if step is not None and step.type.strip().lower() == "llm": + if isinstance(data, dict) and ("input" in data or "output" in data): + return cast(JSONValue | None, data.get("input")), cast( + JSONValue | None, data.get("output") + ) + selected_value = cast(JSONValue | None, data) + if self.config.payload_field == "output": + return None, selected_value + return selected_value, None + if isinstance(data, dict): input_text = _extract_dict_text(data, "input") output_text = _extract_dict_text(data, "output") @@ -215,8 +237,8 @@ async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: """Run a Luna evaluation with optional structured runtime context.""" - input_text, output_text = self._prepare_payload(data) - if not (_has_text(input_text) or _has_text(output_text)): + input_text, output_text = self._prepare_payload(data, step=step) + if not (_has_payload_value(input_text) or _has_payload_value(output_text)): return EvaluatorResult( matched=False, confidence=1.0, @@ -243,12 +265,18 @@ async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: JSONObject, record.model_dump(mode="json", exclude_none=True), ) + if step.type.strip().lower() == "llm": + # Keep the legacy inputs in the request, but derive + # them from Orbit's canonical record so they cannot + # conflict with normalized tool-call/message fields. + input_text = cast(JSONValue | None, record_payload.get("input")) + output_text = cast(JSONValue | None, record_payload.get("output")) if record_payload is not None: scorer_kwargs["record"] = record_payload response = await self._get_client().invoke( **scorer_kwargs, - input=input_text if _has_text(input_text) else None, - output=output_text if _has_text(output_text) else None, + input=input_text if _has_payload_value(input_text) else None, + output=output_text if _has_payload_value(output_text) else None, config=self.config.scorer_config, timeout=self.get_timeout_seconds(), ) diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index fa89f6c4..50388d5e 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -1061,8 +1061,8 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: scorer_id="scorer-123", scorer_version_id="version-456", step=step, - input="selected input", - output=None, + input=record["input"], + output=record["output"], record=record, config=None, timeout=10.0, @@ -1071,6 +1071,275 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: assert record["input"][0]["content"] == "selected input" assert record["output"]["content"] == "full output" + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluate_without_context_preserves_legacy_scalar_payload(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate("selected input") + + assert result.matched is True + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_llm_direct_output_preserves_tool_call_payload(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "payload_field": "output", + "threshold": 0.5, + "operator": "gte", + } + ) + tools = [{"name": "search", "description": "Search", "input_schema": {}}] + output = { + "role": "assistant", + "content": "I will search.", + "tool_calls": [ + { + "id": "call-123", + "type": "function", + "function": {"name": "search", "arguments": "{\"query\":\"weather\"}"}, + } + ], + } + step = Step( + type="llm", + name="answer", + input="Find the weather.", + output=output, + tools=tools, + ) + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context(output, step) + + kwargs = mock_invoke.await_args.kwargs + assert kwargs["input"] == kwargs["record"]["input"] + assert kwargs["output"] == kwargs["record"]["output"] + assert kwargs["record"]["tools"] == tools + assert kwargs["record"]["output"]["tool_calls"][0]["id"] == "call-123" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_llm_preserves_structured_dual_written_tool_call(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "threshold": 0.5, + "operator": "gte", + } + ) + messages = [ + {"role": "system", "content": "Follow transfer policy."}, + {"role": "user", "content": "Review transfer 7781."}, + ] + assistant = { + "role": "assistant", + "content": "I will verify the transfer.", + "tool_calls": [ + { + "id": "call-123", + "type": "function", + "function": { + "name": "submit_transfer", + "arguments": '{"amount": 20}', + }, + } + ], + } + tools = [ + { + "type": "function", + "function": { + "name": "submit_transfer", + "parameters": {"type": "object", "properties": {"amount": {"type": "number"}}}, + }, + } + ] + step = Step( + type="llm", + name="transfer_agent", + input=messages, + output=assistant, + tools=tools, + ) + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse( + scorer_label="Tool Selection Quality (SLM)", + score=0.8, + status="success", + ) + await evaluator.evaluate_with_context( + {"input": messages, "output": assistant}, + step, + ) + + kwargs = mock_invoke.await_args.kwargs + record = kwargs["record"] + assert kwargs["input"] == messages + assert record["type"] == "llm" + assert record["input"] == messages + assert kwargs["output"] == record["output"] + assert record["output"]["role"] == "assistant" + assert record["output"]["content"] == assistant["content"] + assert record["output"]["tool_calls"][0]["id"] == "call-123" + assert record["output"]["tool_calls"][0]["function"]["name"] == "submit_transfer" + assert record["output"]["tool_calls"][0]["function"]["arguments"] == '{"amount": 20}' + assert record["tools"] == tools + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + @pytest.mark.parametrize( + "selected_input", + [ + [{"role": "user", "content": "Review transfer 20."}], + {"role": "user", "content": "Review transfer 20."}, + ], + ids=["message-list", "message-object"], + ) + async def test_contextual_llm_direct_input_preserves_selected_shape( + self, selected_input: JSONValue + ) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "payload_field": "input", + } + ) + assistant = { + "role": "assistant", + "content": "I will verify the transfer.", + "tool_calls": [ + { + "id": "call-123", + "function": { + "name": "submit_transfer", + "arguments": '{"amount": 20}', + }, + } + ], + } + tools = [ + { + "type": "function", + "function": { + "name": "submit_transfer", + "parameters": { + "type": "object", + "properties": {"amount": {"type": "number"}}, + }, + }, + } + ] + step = Step( + type="llm", + name="transfer_agent", + input=selected_input, + output=assistant, + tools=tools, + ) + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context(selected_input, step) + + kwargs = mock_invoke.await_args.kwargs + record = kwargs["record"] + expected_input = selected_input + if isinstance(selected_input, dict): + expected_input = [selected_input] + assert kwargs["input"] == record["input"] == expected_input + assert kwargs["output"] == record["output"] + assert record["type"] == "llm" + assert record["output"]["tool_calls"][0]["function"]["name"] == "submit_transfer" + assert record["tools"] == tools + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_contextual_llm_direct_output_message_preserves_selected_shape(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": "scorer-123", + "scorer_version_id": "version-456", + "payload_field": "output", + } + ) + messages = [{"role": "user", "content": "Review transfer 20."}] + selected_output = { + "role": "assistant", + "content": "I will verify the transfer.", + "tool_calls": [ + { + "id": "call-123", + "function": { + "name": "submit_transfer", + "arguments": '{"amount": 20}', + }, + } + ], + } + tools = [ + { + "type": "function", + "function": { + "name": "submit_transfer", + "parameters": { + "type": "object", + "properties": {"amount": {"type": "number"}}, + }, + }, + } + ] + step = Step( + type="llm", + name="transfer_agent", + input=messages, + output=selected_output, + tools=tools, + ) + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + await evaluator.evaluate_with_context(selected_output, step) + + kwargs = mock_invoke.await_args.kwargs + record = kwargs["record"] + assert kwargs["input"] == record["input"] == messages + assert kwargs["output"] == record["output"] + assert record["type"] == "llm" + assert record["output"]["tool_calls"][0]["id"] == "call-123" + assert record["output"]["tool_calls"][0]["function"]["name"] == "submit_transfer" + assert record["tools"] == tools + @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio async def test_contextual_trace_preserves_raw_record_and_nested_spans(self) -> None: From bc6b2d960eb67ffa582c2a4ec3c44485339a2525 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 28 Sep 2026 12:16:29 -0700 Subject: [PATCH 06/17] fix(server): annotate binding query rows --- server/src/agent_control_server/services/control_bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/agent_control_server/services/control_bindings.py b/server/src/agent_control_server/services/control_bindings.py index f6b04d44..b4118bd6 100644 --- a/server/src/agent_control_server/services/control_bindings.py +++ b/server/src/agent_control_server/services/control_bindings.py @@ -279,7 +279,7 @@ def _apply_filters(stmt): # type: ignore[no-untyped-def] if cursor is not None: page_stmt = page_stmt.where(ControlBinding.id < cursor) result = await self._db.execute(page_stmt.limit(limit + 1)) - rows = list(result.scalars().all()) + rows: list[ControlBinding] = list(result.scalars().all()) has_more = len(rows) > limit if has_more: rows = rows[:limit] From ebd2df77aae0417dbed01ef15b6624bda58b0908 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 28 Sep 2026 12:36:01 -0700 Subject: [PATCH 07/17] test(galileo): cover record normalization paths --- .../galileo/tests/test_records_factory.py | 176 +++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index a57f45a3..aae63f2b 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -3,7 +3,10 @@ from __future__ import annotations import json -from datetime import UTC, datetime +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta, timezone +from enum import Enum +from pathlib import Path from uuid import uuid4 import pytest @@ -23,7 +26,7 @@ from galileo_core.schemas.logging.trace import Trace from galileo_core.schemas.shared.content_parts import FileContentPart, TextContentPart from galileo_core.schemas.shared.document import Document -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field class _RecordPayload(BaseModel): @@ -610,6 +613,175 @@ class NestedPayload(BaseModel): ) +def test_json_normalization_supports_python_value_types_and_fallbacks() -> None: + from agent_control_evaluator_galileo.records.normalization import ( + _GalileoJSONEncoder, + _json_compatible, + _sdk_json_value, + ) + + class State(Enum): + READY = "ready" + + class NestedModel(BaseModel): + value: int + optional: str | None = None + + @dataclass + class NestedDataclass: + identifier: object + + class SlottedValue: + __slots__ = ("value", "missing") + + def __init__(self) -> None: + self.value = "slot" + + class PlainValue: + def __init__(self) -> None: + self.visible = "public" + self._hidden = "private" + + encoder = _GalileoJSONEncoder() + identifier = uuid4() + utc_timestamp = datetime(2025, 1, 2, tzinfo=UTC) + naive_timestamp = datetime(2025, 1, 2) + offset_timestamp = datetime(2025, 1, 2, tzinfo=timezone(timedelta(hours=2))) + + assert encoder.default(NestedModel(value=3)) == {"value": 3} + assert encoder.default(utc_timestamp) == "2025-01-02T00:00:00Z" + assert isinstance(encoder.default(naive_timestamp), str) + assert encoder.default(offset_timestamp) == "2025-01-02T00:00:00+02:00" + assert encoder.default(date(2025, 1, 2)) == "2025-01-02" + assert encoder.default(identifier) == str(identifier) + assert encoder.default(Path("folder/file.txt")) == "folder/file.txt" + assert encoder.default(State.READY) == "ready" + assert encoder.default(b"text") == "text" + assert encoder.default(b"\xff") == "" + assert encoder.default(NestedDataclass(identifier)) == {"identifier": identifier} + assert encoder.default(2**53) == str(2**53) + assert encoder.default({"item"}) == ["item"] + assert encoder.default(object()) == "" + + recursive: list[object] = [] + recursive.append(recursive) + normalized = _sdk_json_value( + { + identifier: NestedModel(value=4), + "date": date(2025, 1, 2), + "enum": State.READY, + "bytes": b"text", + "invalid_bytes": b"\xff", + "large_integer": 2**53, + "set": frozenset({"item"}), + "dataclass": NestedDataclass(identifier), + "slotted": SlottedValue(), + "plain": PlainValue(), + "unsupported": object(), + "recursive": recursive, + } + ) + assert normalized[str(identifier)] == {"value": 4} + assert normalized["date"] == "2025-01-02" + assert normalized["enum"] == "ready" + assert normalized["bytes"] == "text" + assert normalized["invalid_bytes"] == "" + assert normalized["large_integer"] == str(2**53) + assert normalized["set"] == ["item"] + assert normalized["dataclass"] == {"identifier": str(identifier)} + assert normalized["slotted"] == {"value": "slot", "missing": None} + assert normalized["plain"] == {"visible": "public"} + assert normalized["unsupported"] == "" + assert normalized["recursive"] == ["list"] + assert _json_compatible(NestedModel(value=5)) == {"value": 5} + assert _json_compatible({"values": [State.READY, date(2025, 1, 2)]}) == { + "values": ["ready", "2025-01-02"] + } + + +def test_normalizer_preserves_message_content_and_reports_invalid_trace_inputs() -> None: + from agent_control_evaluator_galileo.records.normalization import ( + _flatten_message_sequence, + _message_sequence, + _normalize_content_items, + ) + + messages = [ + {"role": "user", "content": ""}, + {"role": "assistant", "content": ["plain", {"unsupported": True}]}, + {"content": 7}, + {}, + ] + flattened = _flatten_message_sequence(messages) + + assert [part["text"] for part in flattened] == [ + "plain", + '{"unsupported": true}', + "7", + ] + assert _message_sequence([{"role": "user", "content": "question"}]) + assert not _message_sequence([{"other": "field"}]) + assert _normalize_content_items("not a sequence") is None + assert _normalize_content_items([{"invalid": True}]) is None + assert GalileoRecordNormalizer.llm_input(42) == "42" + assert GalileoRecordNormalizer.llm_output([1, 2]) == "[1, 2]" + with pytest.raises(TypeError, match="Trace input must be"): + GalileoRecordNormalizer.trace_input([1]) + with pytest.raises(TypeError, match="does not support int"): + GalileoRecordNormalizer.trace_input(1) + + +def test_retriever_and_session_normalizers_accept_public_models() -> None: + class DocumentModel(BaseModel): + content: str + metadata: dict[str, str] = Field(default_factory=dict) + + pydantic_document = DocumentModel(content="model document") + document = Document(content="canonical document") + part = TextContentPart(text="content part") + message = Message(role="user", content="question") + + assert GalileoRecordNormalizer.retriever_output(pydantic_document) == [ + Document(content="model document", metadata={}) + ] + assert GalileoRecordNormalizer.retriever_output([pydantic_document]) == [ + Document(content="model document", metadata={}) + ] + assert GalileoRecordNormalizer.retriever_output([document]) == [document] + assert GalileoRecordNormalizer.session_input(part) == [part] + assert GalileoRecordNormalizer.session_input(document) == '{"content": "canonical document"}' + assert GalileoRecordNormalizer.session_input(pydantic_document) is pydantic_document + assert GalileoRecordNormalizer.session_input({"role": "user", "content": "question"}) == { + "role": "user", + "content": "question", + } + assert GalileoRecordNormalizer.session_input({"type": "text", "text": "hello"}) == [ + {"type": "text", "text": "hello"} + ] + assert GalileoRecordNormalizer.session_input([message]) == [message] + assert GalileoRecordNormalizer.session_input([{"role": "user", "content": "question"}]) == [ + {"role": "user", "content": "question"} + ] + assert json.loads(GalileoRecordNormalizer.session_input([pydantic_document]))[0][ + "content" + ] == "model document" + assert GalileoRecordNormalizer.session_output(document) == [document] + assert GalileoRecordNormalizer.session_output(part) == [part] + assert GalileoRecordNormalizer.session_output(pydantic_document) is pydantic_document + assert GalileoRecordNormalizer.session_output({"role": "assistant", "content": "answer"}) == { + "role": "assistant", + "content": "answer", + } + assert GalileoRecordNormalizer.session_output({"page_content": "page"}).content == "page" + assert GalileoRecordNormalizer.session_output({"type": "text", "text": "hello"}) == [ + {"type": "text", "text": "hello"} + ] + assert GalileoRecordNormalizer.session_output([document]) == [document] + assert GalileoRecordNormalizer.session_output([{"role": "assistant", "content": "answer"}]) == [ + {"type": "text", "text": "answer"} + ] + + def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values() -> None: identifier = uuid4() timestamp = datetime(2025, 1, 2, tzinfo=UTC) From a2909f6382b158564b2a72edc57af86b36305d5a Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 28 Sep 2026 18:46:42 -0700 Subject: [PATCH 08/17] fix(galileo): address Luna request review feedback --- .../luna/client.py | 8 ++- .../luna/evaluator.py | 33 +++++----- .../galileo/tests/test_luna_evaluator.py | 61 +++++++++++++++++-- sdks/python/README.md | 6 +- .../src/agent_control/control_decorators.py | 3 +- 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index f0554894..6378f219 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -16,7 +16,7 @@ import httpx from agent_control_models import JSONObject, JSONValue, Step -from pydantic import BaseModel, Field, PrivateAttr, model_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from .config import ScorerInvokeConfig @@ -219,6 +219,8 @@ class ScorerInvokeRecord(BaseModel): represented here. Orbit hydrates those fields from trusted server context. """ + model_config = ConfigDict(extra="allow") + type: ScorerInvokeRecordType name: str | None = None input: JSONValue = None @@ -245,7 +247,7 @@ class ScorerInvokeRequest(BaseModel): scorer_version_id: str | None = Field(default=None, min_length=1) scorer_label: str | None = Field(default=None, min_length=1) inputs: ScorerInvokeInputs - record: JSONObject | ScorerInvokeRecord | None = None + record: ScorerInvokeRecord | None = None config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) @model_validator(mode="after") @@ -488,7 +490,7 @@ async def invoke( ground_truth=step.ground_truth if step is not None else None, tools=step.tools if step is not None else None, ), - record=record, + record=ScorerInvokeRecord.model_validate(record) if record is not None else None, config=invoke_config, ).to_dict() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 1d3b21a5..713211fa 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -252,25 +252,22 @@ async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: if step is not None: scorer_kwargs["step"] = step if step.type.strip().lower() in {"llm", "tool", "retriever", "trace", "session"}: - if self.config.scorer_version_id is None: - raise ValueError( - "scorer_version_id is required for structured Luna requests." + if self.config.scorer_version_id is not None: + record = build_galileo_record( + data, + step, + payload_field=self.config.payload_field, ) - record = build_galileo_record( - data, - step, - payload_field=self.config.payload_field, - ) - record_payload = cast( - JSONObject, - record.model_dump(mode="json", exclude_none=True), - ) - if step.type.strip().lower() == "llm": - # Keep the legacy inputs in the request, but derive - # them from Orbit's canonical record so they cannot - # conflict with normalized tool-call/message fields. - input_text = cast(JSONValue | None, record_payload.get("input")) - output_text = cast(JSONValue | None, record_payload.get("output")) + record_payload = cast( + JSONObject, + record.model_dump(mode="json", exclude_none=True), + ) + if step.type.strip().lower() == "llm": + # Keep the legacy inputs in the request, but derive + # them from Orbit's canonical record so they cannot + # conflict with normalized tool-call/message fields. + input_text = cast(JSONValue | None, record_payload.get("input")) + output_text = cast(JSONValue | None, record_payload.get("output")) if record_payload is not None: scorer_kwargs["record"] = record_payload response = await self._get_client().invoke( diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 50388d5e..7ede6d9f 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -248,6 +248,50 @@ def test_scorer_invoke_request_serialization_keeps_legacy_record_shape(self) -> "config": {}, } + def test_scorer_invoke_request_validates_and_preserves_record_fields(self) -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRequest + + request = ScorerInvokeRequest( + scorer_id="scorer-123", + scorer_version_id="version-123", + inputs={"query": "hello"}, + record={ + "type": "llm", + "input": "hello", + "metrics": {"duration_ns": 12}, + "custom_field": {"value": "preserved"}, + }, + ) + + assert request.to_dict()["record"] == { + "type": "llm", + "input": "hello", + "metrics": {"duration_ns": 12}, + "custom_field": {"value": "preserved"}, + } + + @pytest.mark.parametrize("record", [{}, {"type": "BOGUS_TYPE"}]) + def test_scorer_invoke_request_validates_record_discriminator(self, record: dict) -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRequest + + with pytest.raises(ValidationError, match="type"): + ScorerInvokeRequest( + scorer_id="scorer-123", + scorer_version_id="version-123", + inputs={"query": "hello"}, + record=record, + ) + + def test_scorer_invoke_request_requires_version_for_structured_record(self) -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRequest + + with pytest.raises(ValidationError, match="scorer_version_id is required"): + ScorerInvokeRequest( + scorer_id="scorer-123", + inputs={"query": "hello"}, + record={"type": "llm", "input": "hello"}, + ) + def test_scorer_invoke_request_rejects_unknown_config(self) -> None: from agent_control_evaluator_galileo.luna import ( ScorerInvokeInputs, @@ -797,6 +841,7 @@ def handler(request: httpx.Request) -> httpx.Response: "context": {"session": {"id": "s-1", "attributes": {"region": "west"}}}, "tools": [{"name": "search", "description": "Search", "input_schema": {}}], "dataset_output": {"text": "expected"}, + "metrics": {"duration_ns": 12}, }, ) finally: @@ -826,6 +871,7 @@ def handler(request: httpx.Request) -> httpx.Response: "context": {"session": {"id": "s-1", "attributes": {"region": "west"}}}, "tools": [{"name": "search", "description": "Search", "input_schema": {}}], "dataset_output": {"text": "expected"}, + "metrics": {"duration_ns": 12}, }, "config": {"request_timeout_seconds": 8.0}, } @@ -1457,18 +1503,25 @@ async def test_contextual_missing_group_context_returns_factory_error( @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio - async def test_structured_request_requires_explicit_scorer_version(self) -> None: - from agent_control_evaluator_galileo.luna import LunaEvaluator + async def test_context_without_scorer_version_uses_legacy_request(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse from agent_control_evaluator_galileo.luna.client import GalileoLunaClient evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) step = Step(type="llm", name="answer", input="question", output="answer") with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") result = await evaluator.evaluate_with_context("question", step) - assert "scorer_version_id is required" in (result.error or "") - mock_invoke.assert_not_called() + assert result.error is None + assert result.matched is True + mock_invoke.assert_awaited_once() + assert mock_invoke.await_args.kwargs["scorer_id"] == "scorer-123" + assert "scorer_version_id" not in mock_invoke.await_args.kwargs + assert "record" not in mock_invoke.await_args.kwargs + assert mock_invoke.await_args.kwargs["input"] == "question" + assert mock_invoke.await_args.kwargs["output"] is None @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio diff --git a/sdks/python/README.md b/sdks/python/README.md index 46dd9ce2..9522cf2e 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -53,8 +53,10 @@ async def answer(message: str) -> str: ``` For per-call tools, pass a provider such as -`tools=lambda args, kwargs: tools_for(kwargs.get("model"))`. The provider may -return `None` when the step has no available tools. +`tools=lambda args, kwargs: tools_for(kwargs.get("model"))`. The provider +receives the raw positional call arguments in `args` and keyword arguments in +`kwargs` for each pre- and post-execution check. The provider may return `None` +when the step has no available tools. For framework adapters and advanced callers, construct the shared `Step` directly and send it through the SDK's `check_evaluation` helper. This keeps diff --git a/sdks/python/src/agent_control/control_decorators.py b/sdks/python/src/agent_control/control_decorators.py index 6c96b674..cfd02219 100644 --- a/sdks/python/src/agent_control/control_decorators.py +++ b/sdks/python/src/agent_control/control_decorators.py @@ -868,7 +868,8 @@ def control( ``llm``. tools: Optional tool definitions for LLM steps. Static definitions are copied when the decorator is created. A provider receives the - function's positional and keyword arguments at each check. + raw call-site positional arguments as ``args`` and keyword + arguments as ``kwargs`` at each pre- and post-execution check. Returns: Decorated function From a8744463aba8d97d4651416a29bfc9b38a425556 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 5 Oct 2026 15:46:34 -0700 Subject: [PATCH 09/17] add Step.children --- models/src/agent_control_models/agent.py | 8 +++ sdks/python/src/agent_control/evaluation.py | 3 + sdks/python/tests/test_evaluation.py | 80 +++++++++++++++++++++ sdks/python/tests/test_local_evaluation.py | 37 ++++++++-- 4 files changed, 122 insertions(+), 6 deletions(-) diff --git a/models/src/agent_control_models/agent.py b/models/src/agent_control_models/agent.py index e60573c9..aa18e4f3 100644 --- a/models/src/agent_control_models/agent.py +++ b/models/src/agent_control_models/agent.py @@ -168,6 +168,11 @@ class Step(BaseModel): ground_truth: JSONValue | None = Field( None, description="Optional expected or reference output for this step" ) + children: list[Step] | None = Field( + None, + exclude_if=lambda value: value is None, + description="Optional child steps associated with this invocation", + ) @field_validator("type") @classmethod @@ -182,3 +187,6 @@ def validate_builtin_types(self) -> Step: if not isinstance(self.input, dict): raise ValueError("tool steps require object input") return self + + +Step.model_rebuild() diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index 3a8ec91b..800b4bf4 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -523,6 +523,7 @@ async def evaluate_controls( context: dict[str, Any] | None = None, tools: list[dict[str, JSONValue]] | None = None, ground_truth: JSONValue | None = None, + children: list[Step] | None = None, step_type: str = "llm", stage: Literal["pre", "post"] = "pre", agent_name: str, @@ -561,6 +562,8 @@ async def evaluate_controls( step_dict["tools"] = tools if ground_truth is not None: step_dict["ground_truth"] = ground_truth + if children is not None: + step_dict["children"] = children step_obj = Step(**step_dict) # type: ignore[arg-type] resolved_controls = state.server_controls or [] diff --git a/sdks/python/tests/test_evaluation.py b/sdks/python/tests/test_evaluation.py index d598c118..f4d5915d 100644 --- a/sdks/python/tests/test_evaluation.py +++ b/sdks/python/tests/test_evaluation.py @@ -6,6 +6,7 @@ import pytest from agent_control import evaluation from agent_control.evaluation import EvaluationResult +from agent_control_models import Step from pydantic import ValidationError @@ -228,6 +229,85 @@ async def test_evaluate_controls_preserves_explicit_tools_and_ground_truth(monke assert step.ground_truth == {"answer": "expected"} +def test_step_legacy_serialization_is_unchanged_without_children(): + """A Step without children keeps the existing serialized payload.""" + step = Step(type="llm", name="chat", input="hello") + + assert step.model_dump(mode="json") == { + "type": "llm", + "name": "chat", + "input": "hello", + "output": None, + "context": None, + "tools": None, + "ground_truth": None, + } + assert Step(type="llm", name="chat", input="hello", children=None).children is None + + +def test_step_accepts_and_preserves_recursive_children(): + """Nested child steps retain all provider-neutral runtime fields.""" + child_payload = { + "type": "llm", + "name": "child", + "input": {"prompt": "question"}, + "output": "answer", + "context": {"conversation": ["earlier"]}, + "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], + "ground_truth": {"answer": "expected"}, + "children": [ + {"type": "retriever", "name": "search", "input": {"query": "q"}} + ], + } + parent = Step(type="llm", name="parent", input="hello", children=[child_payload]) + + assert parent.children is not None + child = parent.children[0] + assert child.type == child_payload["type"] + assert child.name == child_payload["name"] + assert child.input == child_payload["input"] + assert child.output == child_payload["output"] + assert child.context == child_payload["context"] + assert child.tools == child_payload["tools"] + assert child.ground_truth == child_payload["ground_truth"] + assert child.children is not None + assert child.children[0].input == {"query": "q"} + + +def test_step_rejects_invalid_child_values_clearly(): + with pytest.raises(ValidationError, match="children"): + Step(type="llm", name="parent", input="hello", children=["not a step"]) + + +@pytest.mark.asyncio +async def test_evaluate_controls_passes_children_without_recursively_evaluating(monkeypatch): + """Children are data on the current Step and are not evaluated separately.""" + mock_check = AsyncMock(return_value=EvaluationResult(is_safe=True, confidence=1.0)) + monkeypatch.setattr(evaluation, "check_evaluation_with_local", mock_check) + children = [ + Step( + type="tool", + name="search", + input={"query": "nested"}, + children=[Step(type="llm", name="nested", input="prompt")], + ) + ] + + with patch("agent_control.state.server_url", "http://localhost:8000"): + await evaluation.evaluate_controls( + step_name="parent", + input="hello", + children=children, + stage="pre", + agent_name="test-bot", + ) + + mock_check.assert_awaited_once() + passed_step = mock_check.call_args.kwargs["step"] + assert passed_step.children == children + assert passed_step.children[0].children == children[0].children + + @pytest.mark.asyncio async def test_evaluate_controls_uses_session_api_key_header(monkeypatch): """evaluate_controls should pass init's API-key header into the client.""" diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 1763a136..0c033c3e 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -13,6 +13,11 @@ import httpx import pytest +from agent_control.client import AgentControlClient +from agent_control.evaluation import ( + _merge_results, + check_evaluation_with_local, +) from agent_control_models import ( ControlMatch, EvaluationResponse, @@ -20,12 +25,6 @@ Step, ) -from agent_control.client import AgentControlClient -from agent_control.evaluation import ( - _merge_results, - check_evaluation_with_local, -) - # ============================================================================= # Test Fixtures # ============================================================================= @@ -1316,3 +1315,29 @@ async def test_local_evaluation_includes_steering_context(self, agent_name, llm_ from agent_control_models.controls import SteeringContext as SteeringContextModel assert isinstance(match.steering_context, SteeringContextModel) assert match.steering_context.message == "Please rephrase your input" + + +@pytest.mark.asyncio +async def test_child_content_does_not_change_current_step_control_matching( + agent_name: str, +) -> None: + """Local controls still select against the current step, not its children.""" + parent = Step( + type="llm", + name="parent", + input="safe parent input", + children=[Step(type="llm", name="child", input="test secret")], + ) + client = _HttpOnlyDuckClient() + + result = await check_evaluation_with_local( + client=client, # Given: local execution with a matching pattern only in a child + agent_name=agent_name, + step=parent, + stage="pre", + controls=[make_control_dict(1, "parent-match", execution="sdk", pattern="secret")], + ) + + assert result.is_safe is True + assert result.matches is None + client.http_client.post.assert_not_awaited() From 0fe6d6f4694bbac1024d4aef6c981ac9daac1994 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 5 Oct 2026 17:14:45 -0700 Subject: [PATCH 10/17] update requiest payload contract to use steps.children --- .../__init__.py | 2 - .../luna/client.py | 34 +- .../records/__init__.py | 2 - .../records/factory.py | 319 +++------ .../records/normalization.py | 11 +- .../galileo/tests/test_luna_evaluator.py | 193 ++++- .../galileo/tests/test_records_factory.py | 674 ++++++++++++------ 7 files changed, 716 insertions(+), 519 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index 51f0a65b..0183ad97 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -37,7 +37,6 @@ UnsupportedStepTypeError, build_galileo_record, build_record, - record_from_scorer_invoke_record, record_from_step, ) @@ -57,6 +56,5 @@ "UnsupportedStepTypeError", "build_galileo_record", "build_record", - "record_from_scorer_invoke_record", "record_from_step", ] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index 645ccceb..f14087d2 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -11,13 +11,14 @@ from hmac import new as hmac_new from json import dumps from time import time -from typing import Literal, cast, get_args +from typing import Literal, get_args from urllib.parse import urlsplit import httpx from agent_control_models import JSONObject, JSONValue, Step from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator +from ..records import UnsupportedStepTypeError, record_from_step from .config import ScorerInvokeConfig logger = logging.getLogger(__name__) @@ -274,7 +275,7 @@ def to_dict(self) -> JSONObject: return request -def _orbit_record_from_step( +def _scorer_invoke_record_from_step( step: Step | None, *, selected_input: JSONValue, @@ -299,21 +300,20 @@ def _orbit_record_from_step( Returns: An Orbit-compatible record, or ``None`` for absent/unsupported steps. """ - if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: + if step is None: return None - - # The membership check above narrows the runtime value to Orbit's known - # discriminator set, but static type checkers cannot infer that relationship. - record_type = cast(ScorerInvokeRecordType, step.type) - return ScorerInvokeRecord( - type=record_type, - name=step.name, - input=selected_input if selected_input is not None else step.input, - output=selected_output if selected_output is not None else step.output, - context=step.context, - tools=step.tools, - dataset_output=step.ground_truth, - ) + try: + record = record_from_step( + step, + selected_input=selected_input, + selected_output=selected_output, + ) + except UnsupportedStepTypeError: + return None + canonical_record = record.model_dump(mode="json", exclude_none=True) + if canonical_record.get("type") not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: + return None + return ScorerInvokeRecord.model_validate(canonical_record) class ScorerInvokeResponse(BaseModel): @@ -543,7 +543,7 @@ async def invoke( ground_truth=step.ground_truth if step is not None else None, tools=step.tools if step is not None else None, ), - record=_orbit_record_from_step( + record=_scorer_invoke_record_from_step( step, selected_input=input, selected_output=output, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py index 7265e8a1..4260202c 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/__init__.py @@ -6,7 +6,6 @@ UnsupportedStepTypeError, build_galileo_record, build_record, - record_from_scorer_invoke_record, record_from_step, ) from .normalization import GalileoRecordNormalizer @@ -18,6 +17,5 @@ "UnsupportedStepTypeError", "build_galileo_record", "build_record", - "record_from_scorer_invoke_record", "record_from_step", ] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py index d34047f5..5b67261c 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py @@ -2,20 +2,18 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Any, cast +from collections.abc import Mapping +from typing import Any from agent_control_models import Step from galileo_core.schemas.logging.session import Session from galileo_core.schemas.logging.span import LlmSpan, RetrieverSpan, ToolSpan from galileo_core.schemas.logging.trace import Trace -from pydantic import BaseModel from .normalization import GalileoRecordNormalizer type GalileoRecord = LlmSpan | ToolSpan | RetrieverSpan | Trace | Session type GalileoSpan = LlmSpan | ToolSpan | RetrieverSpan - _MISSING = object() _SPAN_TYPES = {"llm", "tool", "retriever"} _RECORD_TYPES = _SPAN_TYPES | {"trace", "session"} @@ -47,185 +45,13 @@ def _selected_values( return selected_data, None -def _child_payloads(context: Mapping[str, Any], *, key: str) -> list[Any]: - value: Any = context.get(key) - if value is None and key == "spans": - value = context.get("children") - if value is None: - return [] - if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray): - raise RecordFactoryError(f"Galileo {key} context must be a list.") - return list(value) - - -def _span_payloads(raw: Mapping[str, Any], context: Mapping[str, Any]) -> list[Any]: - """Read child spans from either a public model dump or step context.""" - if "spans" in raw: - value = raw["spans"] - if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray): - raise RecordFactoryError("Galileo span context must include a 'spans' list.") - return list(value) - return _child_payloads(context, key="spans") - - -def _nested_record(value: Any) -> GalileoRecord: - if isinstance(value, (LlmSpan, ToolSpan, RetrieverSpan, Trace, Session)): - return value - raw = _mapping(value) - if raw is None: - raise RecordFactoryError("Nested Galileo records must be objects with a supported type.") - raw_type_value = raw.get("type") - raw_type = ( - raw_type_value - if isinstance(raw_type_value, str) - else getattr(raw_type_value, "value", None) - ) - if not isinstance(raw_type, str): - raise RecordFactoryError("Nested Galileo records require a string 'type'.") - if raw_type not in _RECORD_TYPES: - raise UnsupportedStepTypeError(f"Unsupported Galileo nested step type: {raw_type!r}") - return _record_from_mapping(raw, raw_type) - - -def _nested_span(value: Any) -> GalileoSpan: - record = _nested_record(value) - if not isinstance(record, (LlmSpan, ToolSpan, RetrieverSpan)): - raise RecordFactoryError("Nested span context must contain a span record.") - return record - - -def _record_from_mapping(raw: Mapping[str, Any], record_type: str) -> GalileoRecord: - context = _mapping(raw.get("context")) or {} - common = { - "name": "" if raw.get("name") is None else str(raw["name"]), - "user_metadata": GalileoRecordNormalizer.metadata( - raw.get("user_metadata", raw.get("metadata")) - ), - } - for field in ( - "tags", - "status_code", - "external_id", - "step_number", - "created_at", - "metrics", - "dataset_input", - "dataset_output", - "dataset_metadata", - ): - if raw.get(field) is not None: - common[field] = ( - GalileoRecordNormalizer.json_text(raw[field]) - if field in {"dataset_input", "dataset_output"} - else GalileoRecordNormalizer.metadata(raw[field]) - if field == "dataset_metadata" - else raw[field] - ) - - input_value = raw.get("input") - output_value = raw.get("output") - if record_type == "llm": - kwargs = { - **common, - "input": GalileoRecordNormalizer.llm_input(input_value), - "output": GalileoRecordNormalizer.llm_output(output_value), - } - if raw.get("tools") is not None: - kwargs["tools"] = GalileoRecordNormalizer.llm_tools(raw["tools"]) - for field in ("events", "model", "temperature", "finish_reason"): - if raw.get(field) is not None: - kwargs[field] = raw[field] - if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = GalileoRecordNormalizer.llm_redacted_input( - raw["redacted_input"] - ) - if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = GalileoRecordNormalizer.llm_redacted_output( - raw["redacted_output"] - ) - return LlmSpan(**kwargs) - if record_type == "tool": - kwargs = { - **common, - "input": GalileoRecordNormalizer.tool_input(input_value), - "output": GalileoRecordNormalizer.tool_output(output_value), - } - if raw.get("tool_call_id") is not None: - kwargs["tool_call_id"] = str(raw["tool_call_id"]) - if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = GalileoRecordNormalizer.tool_redacted_input( - raw["redacted_input"] - ) - if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = GalileoRecordNormalizer.tool_redacted_output( - raw["redacted_output"] - ) - kwargs["spans"] = [_nested_span(item) for item in _span_payloads(raw, context)] - return ToolSpan(**kwargs) - if record_type == "retriever": - kwargs = { - **common, - "input": GalileoRecordNormalizer.retriever_input(input_value), - "output": GalileoRecordNormalizer.retriever_output(output_value), - } - if raw.get("redacted_input") is not None: - kwargs["redacted_input"] = GalileoRecordNormalizer.retriever_redacted_input( - raw["redacted_input"] - ) - if raw.get("redacted_output") is not None: - kwargs["redacted_output"] = GalileoRecordNormalizer.retriever_redacted_output( - raw["redacted_output"] - ) - kwargs["spans"] = [_nested_span(item) for item in _span_payloads(raw, context)] - return RetrieverSpan(**kwargs) - if record_type == "trace": - spans = raw.get("spans", context.get("spans", context.get("children"))) - if not isinstance(spans, Sequence) or isinstance(spans, str | bytes | bytearray): - raise RecordFactoryError("Galileo trace context must include a 'spans' list.") - trace_kwargs: dict[str, Any] = { - **common, - "input": GalileoRecordNormalizer.trace_input(input_value), - "output": GalileoRecordNormalizer.trace_output(output_value), - "spans": [_nested_span(item) for item in spans], - } - if raw.get("redacted_input") is not None: - trace_kwargs["redacted_input"] = GalileoRecordNormalizer.trace_redacted_input( - raw["redacted_input"] - ) - if raw.get("redacted_output") is not None: - trace_kwargs["redacted_output"] = GalileoRecordNormalizer.trace_redacted_output( - raw["redacted_output"] - ) - return Trace( - **trace_kwargs, - ) - traces = raw.get("traces", context.get("traces")) - if not isinstance(traces, Sequence) or isinstance(traces, str | bytes | bytearray): - raise RecordFactoryError("Galileo session context must include a 'traces' list.") - nested_records = GalileoRecordNormalizer.session_traces( - traces, - normalize_record=_nested_record, - ) - nested_traces = [item for item in nested_records if isinstance(item, Trace)] - if len(nested_traces) != len(nested_records): - raise RecordFactoryError("Galileo session 'traces' must contain trace records.") - session_kwargs: dict[str, Any] = { - **common, - "input": GalileoRecordNormalizer.session_input(input_value), - "output": GalileoRecordNormalizer.session_output(output_value), - "traces": cast(list[Trace], nested_traces), - } - if raw.get("redacted_input") is not None: - session_kwargs["redacted_input"] = GalileoRecordNormalizer.session_redacted_input( - raw["redacted_input"] - ) - if raw.get("redacted_output") is not None: - session_kwargs["redacted_output"] = GalileoRecordNormalizer.session_redacted_output( - raw["redacted_output"] - ) - return Session( - **session_kwargs, - ) +def _span_children(children: list[GalileoRecord], *, parent_type: str) -> list[GalileoSpan]: + spans: list[GalileoSpan] = [] + for child in children: + if not isinstance(child, (LlmSpan, ToolSpan, RetrieverSpan)): + raise RecordFactoryError(f"Galileo {parent_type} children must be span steps.") + spans.append(child) + return spans def record_from_step( @@ -241,7 +67,7 @@ def record_from_step( The complete step determines the record discriminator. Selector-selected values override only the corresponding input/output fields, preserving the selected evaluator payload while the unselected side comes from ``step``. - Trace and session records require structured child context; scalar selected + Trace and session records require child steps; scalar selected values are never promoted into those record types. The result is a concrete Galileo Core step model (a ``BaseStep`` subclass) that can be JSON-serialized and revalidated using that subtype's schema. The factory doesn't assign @@ -259,26 +85,39 @@ def record_from_step( payload_field=payload_field, ) context = dict(_mapping(step.context) or {}) - step_envelope = _mapping(step.input) - if record_type == "trace" and not context and step_envelope is not None: - if "spans" in step_envelope or "children" in step_envelope: - context = dict(step_envelope) - if record_type == "session" and not context and step_envelope is not None: - if "traces" in step_envelope: - context = dict(step_envelope) + hierarchy_fields = {"children", "spans", "traces"}.intersection(context) + if hierarchy_fields: + fields = ", ".join(sorted(hierarchy_fields)) + raise RecordFactoryError( + f"Step.context cannot define Galileo hierarchy fields ({fields}); " + "use Step.children instead." + ) + children = step.children or [] + allowed_child_types = { + "trace": _SPAN_TYPES, + "session": {"trace"}, + "tool": _SPAN_TYPES, + "retriever": _SPAN_TYPES, + "llm": set(), + }[record_type] + if record_type in {"trace", "session"} and step.children is None: + raise RecordFactoryError(f"Galileo {record_type} step is missing Step.children.") + child_records: list[GalileoRecord] = [] + for child in children: + if not isinstance(child, Step): + raise RecordFactoryError("Step.children must contain Agent Control Step objects.") + child_type = child.type.strip().lower() + if child_type not in allowed_child_types: + raise RecordFactoryError( + f"Galileo {record_type} step cannot contain {child_type!r} child steps." + ) + child_records.append(record_from_step(child)) input_value = ( step.input if selected_input is _MISSING or selected_input is None else selected_input ) output_value = ( step.output if selected_output is _MISSING or selected_output is None else selected_output ) - if selected_input is _MISSING or selected_input is None: - if step_envelope is not None and record_type in {"trace", "session"}: - input_value = step_envelope.get("input", step.input) - if selected_output is _MISSING or selected_output is None: - if step_envelope is not None and record_type in {"trace", "session"}: - output_value = step_envelope.get("output", step.output) - if record_type in {"trace", "session"}: selected_value = selected_data if selected_data is _MISSING: @@ -287,52 +126,54 @@ def record_from_step( selected_value, Mapping ): raise RecordFactoryError( - f"Galileo {record_type} records require structured context; " + f"Galileo {record_type} records require structured selector values; " "an untyped scalar selector value is not sufficient." ) - if record_type == "trace" and not ( - "spans" in context or "children" in context or isinstance(context.get("trace"), Mapping) - ): - raise RecordFactoryError("Galileo trace context is missing; provide a 'spans' list.") - if record_type == "session" and not ( - "traces" in context or isinstance(context.get("session"), Mapping) - ): - raise RecordFactoryError("Galileo session context is missing; provide a 'traces' list.") - - source: dict[str, Any] = { - "type": record_type, + common: dict[str, Any] = { "name": step.name, - "input": input_value, - "output": output_value, - "context": context, - "tools": step.tools, + "user_metadata": GalileoRecordNormalizer.metadata(context.get("metadata")), } - if record_type == "trace" and isinstance(context.get("trace"), Mapping): - source.update(context["trace"]) - if record_type == "session" and isinstance(context.get("session"), Mapping): - source.update(context["session"]) - if selected_input is not _MISSING and selected_input is not None: - source["input"] = input_value - if selected_output is not _MISSING and selected_output is not None: - source["output"] = output_value if step.ground_truth is not None: - source["dataset_output"] = step.ground_truth - source["metadata"] = context.get("metadata") - return _record_from_mapping(source, record_type) - - -def record_from_scorer_invoke_record(record: BaseModel) -> GalileoRecord: - """Convert the legacy Pydantic Luna request record to a canonical record. - - The Luna wire contract intentionally remains owned by - :class:`ScorerInvokeRecord`. This adapter accepts that existing request - model at the boundary and uses the published SDK models internally. - """ - raw = record.model_dump(mode="python", exclude_none=True) - record_type = raw.get("type") - if not isinstance(record_type, str) or record_type not in _RECORD_TYPES: - raise UnsupportedStepTypeError(f"Unsupported Galileo record type: {record_type!r}") - return _record_from_mapping(raw, record_type) + common["dataset_output"] = GalileoRecordNormalizer.json_text(step.ground_truth) + if record_type == "llm": + kwargs = { + **common, + "input": GalileoRecordNormalizer.llm_input(input_value), + "output": GalileoRecordNormalizer.llm_output(output_value), + } + if step.tools is not None: + kwargs["tools"] = GalileoRecordNormalizer.llm_tools(step.tools) + return LlmSpan(**kwargs) + if record_type == "tool": + return ToolSpan( + **common, + input=GalileoRecordNormalizer.tool_input(input_value), + output=GalileoRecordNormalizer.tool_output(output_value), + spans=_span_children(child_records, parent_type=record_type), + ) + if record_type == "retriever": + return RetrieverSpan( + **common, + input=GalileoRecordNormalizer.retriever_input(input_value), + output=GalileoRecordNormalizer.retriever_output(output_value), + spans=_span_children(child_records, parent_type=record_type), + ) + if record_type == "trace": + return Trace( + **common, + input=GalileoRecordNormalizer.trace_input(input_value), + output=GalileoRecordNormalizer.trace_output(output_value), + spans=_span_children(child_records, parent_type=record_type), + ) + traces = [child for child in child_records if isinstance(child, Trace)] + if len(traces) != len(child_records): + raise RecordFactoryError("Galileo session children must be trace steps.") + return Session( + **common, + input=GalileoRecordNormalizer.session_input(input_value), + output=GalileoRecordNormalizer.session_output(output_value), + traces=traces, + ) def build_record(selected_data: Any, step: Step, *, payload_field: str = "input") -> GalileoRecord: diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py index 5144cbbb..c491602a 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import asdict, is_dataclass from datetime import UTC, date, datetime from enum import Enum @@ -459,12 +459,3 @@ def session_redacted_input(cls, value: Any) -> Any: @classmethod def session_redacted_output(cls, value: Any) -> Any: return None if value is None else cls.session_output(value) - - @staticmethod - def session_traces( - values: Sequence[Any], - *, - normalize_record: Callable[[Any], Any], - ) -> list[Any]: - """Normalize every nested session trace through the shared record path.""" - return [normalize_record(value) for value in values] diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 45f4c946..1e5473b7 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -781,6 +781,7 @@ def handler(_request: httpx.Request) -> httpx.Response: @pytest.mark.asyncio async def test_client_dual_writes_legacy_inputs_and_structured_record(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient + from agent_control_evaluator_galileo.records import record_from_step captured: dict[str, object] = {} @@ -804,7 +805,7 @@ def handler(request: httpx.Request) -> httpx.Response: name="answer", input={"messages": [{"role": "user", "content": "question"}]}, output={"text": "answer"}, - context={"session": {"id": "s-1", "attributes": {"region": "west"}}}, + context={"metadata": {"source": "runtime"}, "request_id": "r-1"}, tools=[{"name": "search", "description": "Search", "input_schema": {}}], ground_truth={"text": "expected"}, ) @@ -825,7 +826,7 @@ def handler(request: httpx.Request) -> httpx.Response: finally: await client.close() - # Then: old inputs and an equivalent Orbit record are sent together + # Then: legacy inputs and the shared factory's canonical record are sent together assert response.score == 0.9 assert response.scorer_label == "toxicity" assert response.status == "success" @@ -841,15 +842,11 @@ def handler(request: httpx.Request) -> httpx.Response: "ground_truth": {"text": "expected"}, "tools": [{"name": "search", "description": "Search", "input_schema": {}}], }, - "record": { - "type": "llm", - "name": "answer", - "input": "selected question", - "output": "selected answer", - "context": {"session": {"id": "s-1", "attributes": {"region": "west"}}}, - "tools": [{"name": "search", "description": "Search", "input_schema": {}}], - "dataset_output": {"text": "expected"}, - }, + "record": record_from_step( + step, + selected_input="selected question", + selected_output="selected answer", + ).model_dump(mode="json", exclude_none=True), "config": {"request_timeout_seconds": 8.0}, } assert captured["body"] == expected_body @@ -857,14 +854,15 @@ def handler(request: httpx.Request) -> httpx.Response: # The emitted body also satisfies the exact additive Orbit #1720 shape. orbit_request = _Orbit1720Request.model_validate(expected_body) assert orbit_request.record is not None - assert orbit_request.inputs.query == orbit_request.record.input - assert orbit_request.inputs.response == orbit_request.record.output + assert orbit_request.inputs.query == "selected question" + assert orbit_request.inputs.response == "selected answer" + assert "selected question" in json.dumps(orbit_request.record.input) + assert "selected answer" in json.dumps(orbit_request.record.output) assert orbit_request.inputs.tools == orbit_request.record.tools assert orbit_request.record.dataset_output == {"text": "expected"} assert orbit_request.inputs.ground_truth == {"text": "expected"} - assert orbit_request.record.context == { - "session": {"id": "s-1", "attributes": {"region": "west"}} - } + assert orbit_request.record.user_metadata == {"source": "runtime"} + assert "context" not in expected_body["record"] assert orbit_request.config is not None assert orbit_request.config.request_timeout_seconds == 8.0 @@ -907,6 +905,169 @@ def handler(request: httpx.Request) -> httpx.Response: "config": {"request_timeout_seconds": 8.0}, } + @pytest.mark.parametrize( + ("step", "record_type"), + [ + (Step(type="llm", name="answer", input="question", output="answer"), "llm"), + (Step(type="tool", name="search", input={"query": "q"}, output="result"), "tool"), + ( + Step(type="retriever", name="retrieve", input="question", output=["document"]), + "retriever", + ), + ], + ) + @pytest.mark.asyncio + async def test_client_uses_canonical_factory_for_flat_steps( + self, + step: Step, + record_type: str, + ) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + await client.invoke( + scorer_id="scorer-123", + input="selected input", + output="selected output", + step=step, + ) + finally: + await client.close() + + body = captured["body"] + assert isinstance(body, dict) + record = body["record"] + assert isinstance(record, dict) + assert record["type"] == record_type + assert "context" not in record + if record_type in {"tool", "retriever"}: + assert record["spans"] == [] + + @pytest.mark.asyncio + async def test_client_serializes_nested_session_trace_and_tool_spans(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + step = Step( + type="session", + name="conversation", + input="session input", + children=[ + Step( + type="trace", + name="request", + input="trace input", + children=[ + Step( + type="tool", + name="search", + input={"query": "q"}, + output="result", + children=[Step(type="tool", name="nested", input={"id": 1})], + ) + ], + ) + ], + ) + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + await client.invoke( + scorer_id="scorer-123", + input={"query": "selected"}, + output={"answer": "selected"}, + step=step, + ) + finally: + await client.close() + + body = captured["body"] + assert isinstance(body, dict) + record = body["record"] + assert isinstance(record, dict) + assert record["type"] == "session" + traces = record["traces"] + assert isinstance(traces, list) + trace = traces[0] + assert isinstance(trace, dict) + assert trace["type"] == "trace" + spans = trace["spans"] + assert isinstance(spans, list) + tool = spans[0] + assert isinstance(tool, dict) + assert tool["type"] == "tool" + nested_spans = tool["spans"] + assert isinstance(nested_spans, list) + assert nested_spans[0]["name"] == "nested" + + @pytest.mark.parametrize("field", ["children", "spans", "traces"]) + @pytest.mark.asyncio + async def test_client_rejects_hierarchy_fields_from_step_context(self, field: str) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + from agent_control_evaluator_galileo.records import RecordFactoryError + + step = Step( + type="tool", + name="search", + input={"query": "q"}, + context={field: []}, + ) + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"score": 0.5, "status": "success"}) + ) + ) + + try: + with pytest.raises(RecordFactoryError, match="Step.context cannot define"): + await client.invoke(scorer_id="scorer-123", input="selected", step=step) + finally: + await client.close() + + @pytest.mark.asyncio + async def test_client_uses_legacy_inputs_when_step_is_missing(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + await client.invoke(scorer_id="scorer-123", input="legacy query") + finally: + await client.close() + + assert captured["body"] == { + "scorer_id": "scorer-123", + "inputs": {"query": "legacy query", "response": ""}, + "config": {"request_timeout_seconds": 8.0}, + } + @pytest.mark.asyncio async def test_client_forwards_scorer_version_id_when_configured(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index f47f4c91..eadd112b 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -16,7 +16,6 @@ UnsupportedStepTypeError, build_galileo_record, build_record, - record_from_scorer_invoke_record, record_from_step, ) from agent_control_models import Step @@ -28,22 +27,7 @@ from galileo_core.schemas.shared.content_parts import FileContentPart, TextContentPart from galileo_core.schemas.shared.document import Document from galileo_core.schemas.shared.records import BaseRecord, RecordTypeAdapter -from pydantic import BaseModel, ConfigDict, Field - - -class _RecordPayload(BaseModel): - """Flexible Pydantic boundary model used to exercise the adapter contract.""" - - model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") - - type: str - name: str | None = None - input: object | None = None - output: object | None = None - context: object | None = None - spans: object | None = None - traces: object | None = None - tool_call_id: object | None = None +from pydantic import BaseModel, Field def test_required_galileo_core_public_exports_are_importable() -> None: @@ -64,7 +48,7 @@ def test_factory_records_are_serializable_concrete_base_steps() -> None: name="request", input="question", output="answer", - context={"spans": [{"type": "llm", "name": "answer", "input": "question"}]}, + children=[Step(type="llm", name="answer", input="question")], ) ), record_from_step( @@ -72,16 +56,14 @@ def test_factory_records_are_serializable_concrete_base_steps() -> None: type="session", name="conversation", input="question", - context={ - "traces": [ - { - "type": "trace", - "name": "request", - "input": "question", - "spans": [{"type": "llm", "name": "answer", "input": "question"}], - } - ] - }, + children=[ + Step( + type="trace", + name="request", + input="question", + children=[Step(type="llm", name="answer", input="question")], + ) + ], ) ), ] @@ -117,10 +99,10 @@ def test_factory_steps_convert_to_core_records_after_execution_ids_are_added() - Step(type="retriever", name="retrieve", input="question", output=["document"]) ), record_from_step( - Step(type="trace", name="request", input="question", context={"spans": []}) + Step(type="trace", name="request", input="question", children=[]) ), record_from_step( - Step(type="session", name="conversation", input="question", context={"traces": []}) + Step(type="session", name="conversation", input="question", children=[]) ), ] project_id = uuid4() @@ -314,17 +296,10 @@ def test_retriever_results_become_documents_with_scalar_metadata_only() -> None: def test_retriever_accepts_the_public_galileo_core_document_model() -> None: - from pydantic import BaseModel - document = Document(content="context", metadata={"source": "kb"}) - - class RecordInput(BaseModel): - type: str = "retriever" - name: str = "retrieve" - input: str = "question" - output: object - - record = record_from_scorer_invoke_record(RecordInput(output=document)) + record = RetrieverSpan.model_validate( + {"name": "retrieve", "input": "question", "output": [document]} + ) assert isinstance(record, RetrieverSpan) assert record.output[0].content == "context" @@ -337,12 +312,11 @@ def test_public_document_metadata_variants_are_supported() -> None: {"content": "model metadata", "metadata": {"source": "kb"}} ) - record = record_from_scorer_invoke_record( - _RecordPayload( - type="retriever", - input="question", - output=[document_without_metadata, document_with_model_metadata], - ) + record = RetrieverSpan.model_validate( + { + "input": "question", + "output": [document_without_metadata, document_with_model_metadata], + } ) assert isinstance(record, RetrieverSpan) @@ -358,17 +332,15 @@ def test_nested_trace_and_session_records_are_preserved() -> None: type="trace", name="request", input="question", - context={ - "spans": [ - { - "type": "tool", - "name": "search", - "input": {"q": "x"}, - "spans": [{"type": "tool", "name": "nested", "input": {"q": "y"}}], - }, - {"type": "retriever", "name": "retrieve", "input": "x", "output": []}, - ] - }, + children=[ + Step( + type="tool", + name="search", + input={"q": "x"}, + children=[Step(type="tool", name="nested", input={"q": "y"})], + ), + Step(type="retriever", name="retrieve", input="x", output=[]), + ], ) ) session = record_from_step( @@ -376,7 +348,12 @@ def test_nested_trace_and_session_records_are_preserved() -> None: type="session", name="conversation", input="question", - context={"traces": [trace.model_dump(mode="json")]}, + children=[Step(type="trace", name="request", input="question", children=[ + Step(type="tool", name="search", input={"q": "x"}, children=[ + Step(type="tool", name="nested", input={"q": "y"}) + ]), + Step(type="retriever", name="retrieve", input="x", output=[]), + ])], ) ) @@ -438,20 +415,14 @@ def test_public_model_dump_preserves_canonical_fields() -> None: traces=[], ) - rebuilt_llm = record_from_scorer_invoke_record( - _RecordPayload(**llm_source.model_dump(mode="json", exclude_none=True)) - ) - rebuilt_tool = record_from_scorer_invoke_record( - _RecordPayload(**tool_source.model_dump(mode="json", exclude_none=True)) - ) - rebuilt_retriever = record_from_scorer_invoke_record( - _RecordPayload(**retriever_source.model_dump(mode="json", exclude_none=True)) + rebuilt_llm = LlmSpan.model_validate(llm_source.model_dump(mode="json", exclude_none=True)) + rebuilt_tool = ToolSpan.model_validate(tool_source.model_dump(mode="json", exclude_none=True)) + rebuilt_retriever = RetrieverSpan.model_validate( + retriever_source.model_dump(mode="json", exclude_none=True) ) - rebuilt_trace = record_from_scorer_invoke_record( - _RecordPayload(**trace_source.model_dump(mode="json", exclude_none=True)) - ) - rebuilt_session = record_from_scorer_invoke_record( - _RecordPayload(**session_source.model_dump(mode="json", exclude_none=True)) + rebuilt_trace = Trace.model_validate(trace_source.model_dump(mode="json", exclude_none=True)) + rebuilt_session = Session.model_validate( + session_source.model_dump(mode="json", exclude_none=True) ) assert isinstance(rebuilt_llm, LlmSpan) @@ -491,9 +462,7 @@ def test_trace_public_content_parts_are_not_reserialized() -> None: spans=[], ) - rebuilt = record_from_scorer_invoke_record( - _RecordPayload(**source.model_dump(mode="json", exclude_none=True)) - ) + rebuilt = Trace.model_validate(source.model_dump(mode="json", exclude_none=True)) assert isinstance(rebuilt, Trace) assert rebuilt.model_dump(mode="json", exclude_none=True) == source.model_dump( @@ -505,9 +474,7 @@ def test_trace_public_content_parts_are_not_reserialized() -> None: output=[{"type": "file", "file_id": str(uuid4())}], spans=[], ) - file_rebuilt = record_from_scorer_invoke_record( - _RecordPayload(**file_source.model_dump(mode="json", exclude_none=True)) - ) + file_rebuilt = Trace.model_validate(file_source.model_dump(mode="json", exclude_none=True)) assert isinstance(file_rebuilt, Trace) assert file_rebuilt.model_dump(mode="json", exclude_none=True) == file_source.model_dump( mode="json", exclude_none=True @@ -515,81 +482,41 @@ def test_trace_public_content_parts_are_not_reserialized() -> None: def test_malformed_trace_content_parts_follow_sdk_serialization() -> None: - scalar_item = record_from_scorer_invoke_record( - _RecordPayload(type="trace", input="input", output=[1], spans=[]) - ) - invalid_file = record_from_scorer_invoke_record( - _RecordPayload( - type="trace", - input="input", - output=[{"type": "file", "file_id": "not-a-uuid"}], - spans=[], + from pydantic import ValidationError + + with pytest.raises(ValidationError): + Trace.model_validate({"input": "input", "output": [1], "spans": []}) + with pytest.raises(ValidationError): + Trace.model_validate( + { + "input": "input", + "output": [{"type": "file", "file_id": "not-a-uuid"}], + "spans": [], + } ) - ) - - assert isinstance(scalar_item, Trace) - assert isinstance(invalid_file, Trace) - assert scalar_item.output == json.dumps([1]) - assert invalid_file.output == json.dumps([{"type": "file", "file_id": "not-a-uuid"}]) - -def test_nested_record_errors_and_existing_models_are_explicit() -> None: - from agent_control_evaluator_galileo.records.factory import _record_from_mapping - existing_child = ToolSpan(name="existing", input="input") - with pytest.raises(RecordFactoryError, match="objects"): - record_from_scorer_invoke_record( - _RecordPayload(type="tool", input="input", spans=[1]) - ) - with pytest.raises(RecordFactoryError, match="string 'type'"): - record_from_scorer_invoke_record( - _RecordPayload(type="tool", input="input", spans=[{}]) - ) - with pytest.raises(UnsupportedStepTypeError, match="custom"): - record_from_scorer_invoke_record( - _RecordPayload(type="tool", input="input", spans=[{"type": "custom"}]) - ) - with pytest.raises(RecordFactoryError, match="span record"): - record_from_scorer_invoke_record( - _RecordPayload( - type="tool", - input="input", - spans=[{"type": "trace", "input": "input", "spans": []}], +def test_context_hierarchy_fields_are_rejected_and_metadata_remains_supported() -> None: + for hierarchy_field in ("children", "spans", "traces"): + with pytest.raises(RecordFactoryError, match="Step.context cannot define.*Step.children"): + record_from_step( + Step( + type="tool", + name="tool", + input={"q": "input"}, + context={hierarchy_field: []}, + ) ) + metadata_record = record_from_step( + Step( + type="tool", + name="tool", + input={"q": "input"}, + context={"metadata": {"provider": "test"}, "request_id": "abc"}, ) - - record = _record_from_mapping( - {"type": "tool", "input": "input", "spans": [existing_child]}, "tool" - ) - assert isinstance(record, ToolSpan) - assert record.spans[0].name == existing_child.name - - -def test_invalid_nested_context_shapes_are_rejected() -> None: - valid_context_record = record_from_step( - Step(type="tool", name="tool", input={"q": "input"}, context={"spans": []}) - ) - children_context_record = record_from_step( - Step(type="tool", name="tool", input={"q": "input"}, context={"children": []}) ) - assert isinstance(valid_context_record, ToolSpan) - assert isinstance(children_context_record, ToolSpan) - with pytest.raises(RecordFactoryError, match="context must be a list"): - record_from_step( - Step(type="tool", name="tool", input={"q": "input"}, context={"spans": "invalid"}) - ) - with pytest.raises(RecordFactoryError, match="span context"): - record_from_scorer_invoke_record( - _RecordPayload(type="tool", input="input", spans="invalid") - ) - with pytest.raises(RecordFactoryError, match="trace context"): - record_from_scorer_invoke_record(_RecordPayload(type="trace", input="input")) - with pytest.raises(RecordFactoryError, match="session context"): - record_from_scorer_invoke_record(_RecordPayload(type="session", input="input")) - with pytest.raises(RecordFactoryError, match="trace records"): - record_from_scorer_invoke_record( - _RecordPayload(type="session", input="input", traces=[{"type": "tool"}]) - ) + assert isinstance(metadata_record, ToolSpan) + assert metadata_record.user_metadata == {"provider": "test"} def test_trace_structured_values_are_normalized_without_losing_messages() -> None: @@ -601,7 +528,7 @@ def test_trace_structured_values_are_normalized_without_losing_messages() -> Non name="request", input=trace_input, output=trace_output, - context={"spans": []}, + children=[], ) ) @@ -622,7 +549,7 @@ def test_trace_content_blocks_preserve_text_files_and_unrepresentable_data() -> name="multimodal", input=blocks, output=blocks, - context={"spans": []}, + children=[], ) ) serialized_data_block = { @@ -636,7 +563,7 @@ def test_trace_content_blocks_preserve_text_files_and_unrepresentable_data() -> name="inline-image", input=[serialized_data_block], output=[serialized_data_block], - context={"spans": []}, + children=[], ) ) @@ -665,7 +592,7 @@ def test_trace_message_sequences_flatten_every_text_and_content_part() -> None: ] trace = record_from_step( - Step(type="trace", name="messages", input="question", output=output, context={"spans": []}) + Step(type="trace", name="messages", input="question", output=output, children=[]) ) assert isinstance(trace, Trace) @@ -953,13 +880,12 @@ class DocumentModel(BaseModel): def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values() -> None: identifier = uuid4() timestamp = datetime(2025, 1, 2, tzinfo=UTC) - source = _RecordPayload( - type="llm", + source = LlmSpan( input="question", tools=[{"id": identifier, "created_at": timestamp}], ) - - record = record_from_scorer_invoke_record(source) + wire_json = source.model_dump(mode="json", exclude_none=True) + record = LlmSpan.model_validate(wire_json) assert isinstance(record, LlmSpan) assert record.tools == [ @@ -978,89 +904,115 @@ def test_retriever_dictionaries_become_canonical_documents() -> None: assert documents[0].metadata == {"source": "kb"} -def test_trace_and_session_envelopes_are_supported() -> None: +def test_trace_and_session_use_children_and_selectors_only_change_the_root() -> None: trace = record_from_step( Step( type="trace", name="outer", - input={"input": "question", "spans": []}, - ) + input="base question", + children=[ + Step( + type="tool", + name="child", + input={"query": "child query"}, + output="child result", + context={"metadata": {"owner": "child"}}, + ), + Step( + type="llm", + name="child llm", + input="child prompt", + output="child response", + tools=[{"name": "lookup"}], + ) + ], + ), + selected_data={"input": "selected question", "output": "selected answer"}, ) session = record_from_step( Step( type="session", - name="outer", - input={"input": "question", "traces": []}, + name="conversation", + input="base session", + children=[ + Step( + type="trace", + name="child trace", + input="child question", + children=[Step(type="llm", name="child llm", input="child input")], + ) + ], ) ) - wrapped_trace = record_from_step( - Step( - type="trace", - name="outer", - input="ignored", - context={"trace": {"name": "inner", "input": "question", "spans": []}}, + + assert isinstance(trace, Trace) + assert trace.input == "selected question" + assert trace.output == "selected answer" + assert isinstance(trace.spans[0], ToolSpan) + assert json.loads(trace.spans[0].input) == {"query": "child query"} + assert trace.spans[0].output == "child result" + assert trace.spans[0].user_metadata == {"owner": "child"} + assert isinstance(trace.spans[1], LlmSpan) + assert trace.spans[1].tools == [{"name": "lookup"}] + assert trace.spans[1].output.content == "child response" + assert isinstance(session, Session) + assert session.traces[0].name == "child trace" + assert session.traces[0].input == "child question" + assert isinstance(session.traces[0].spans[0], LlmSpan) + + +def test_children_must_be_valid_for_parent_and_trace_session_children_are_required() -> None: + with pytest.raises(RecordFactoryError, match="cannot contain 'session'"): + record_from_step( + Step(type="trace", name="trace", input="q", children=[ + Step(type="session", name="session", input="q", children=[]) + ]) ) - ) - wrapped_session = record_from_step( - Step( - type="session", - name="outer", - input="ignored", - context={"session": {"name": "inner", "input": "question", "traces": []}}, + with pytest.raises(RecordFactoryError, match="cannot contain 'tool'"): + record_from_step( + Step(type="session", name="session", input="q", children=[ + Step(type="tool", name="tool", input={}) + ]) ) - ) - selected_trace = record_from_step( - Step( - type="trace", - name="outer", - input="ignored", - context={ - "trace": { - "name": "inner", - "input": "wrapped input", - "output": "wrapped output", - "spans": [], - } - }, - ), - selected_data={"input": "selected input", "output": "selected output"}, - ) - selected_session = record_from_step( + with pytest.raises(RecordFactoryError, match="missing Step.children"): + record_from_step( + Step(type="trace", name="trace", input={"input": "q", "spans": []}) + ) + with pytest.raises(RecordFactoryError, match="missing Step.children"): + record_from_step( + Step(type="session", name="session", input={"input": "q", "traces": []}) + ) + + +def test_nested_tool_and_retriever_children_are_recursively_converted() -> None: + record = record_from_step( Step( - type="session", + type="tool", name="outer", - input="ignored", - context={ - "session": { - "name": "inner", - "input": "wrapped input", - "output": "wrapped output", - "traces": [], - } - }, - ), - selected_data={"input": "selected input", "output": "selected output"}, + input={}, + children=[ + Step( + type="retriever", + name="inner retriever", + input="query", + output=["document"], + children=[Step(type="tool", name="leaf tool", input={"x": 1})], + ) + ], + ) ) - assert isinstance(trace, Trace) - assert isinstance(session, Session) - assert isinstance(wrapped_trace, Trace) - assert isinstance(wrapped_session, Session) - assert isinstance(selected_trace, Trace) - assert isinstance(selected_session, Session) - assert trace.input == "question" - assert session.input == "question" - assert wrapped_trace.name == "inner" - assert wrapped_session.name == "inner" - assert selected_trace.input == "selected input" - assert selected_trace.output == "selected output" - assert selected_session.input == "selected input" - assert selected_session.output == "selected output" + assert isinstance(record, ToolSpan) + retriever = record.spans[0] + assert isinstance(retriever, RetrieverSpan) + assert retriever.output[0].content == "document" + assert isinstance(retriever.spans[0], ToolSpan) + assert json.loads(retriever.spans[0].input) == {"x": 1} def test_trace_missing_output_stays_none() -> None: record = record_from_step( - Step(type="trace", name="request", input="question", context={"spans": []}) + Step(type="trace", name="request", input="question", children=[]) ) assert isinstance(record, Trace) @@ -1076,9 +1028,7 @@ def test_optional_fields_and_aliases_are_translated() -> None: tools=[{"name": "search"}], ) ) - tool = record_from_scorer_invoke_record( - _RecordPayload(type="tool", input="input", tool_call_id=123) - ) + tool = ToolSpan.model_validate({"tool_call_id": "123"}) alias = build_galileo_record("question", Step(type="llm", name="answer", input="base")) assert isinstance(llm, LlmSpan) @@ -1089,7 +1039,6 @@ def test_optional_fields_and_aliases_are_translated() -> None: def test_session_message_and_document_sequences_use_public_validators() -> None: - traces = [{"type": "trace", "name": "request", "input": "question", "spans": []}] messages = [{"role": "user", "content": "question"}] documents = [{"content": "answer", "metadata": {"source": "kb"}}] @@ -1099,7 +1048,7 @@ def test_session_message_and_document_sequences_use_public_validators() -> None: name="conversation", input=messages, output=documents, - context={"traces": traces}, + children=[Step(type="trace", name="request", input="question", children=[])], ) ) @@ -1123,15 +1072,15 @@ def test_session_message_and_document_sequences_use_public_validators() -> None: ] -def test_trace_and_session_require_structured_context() -> None: - with pytest.raises(RecordFactoryError, match="trace context is missing"): +def test_trace_and_session_require_children_and_selector_values_must_be_structured() -> None: + with pytest.raises(RecordFactoryError, match="missing Step.children"): record_from_step(Step(type="trace", name="request", input="question")) - with pytest.raises(RecordFactoryError, match="session context is missing"): + with pytest.raises(RecordFactoryError, match="missing Step.children"): record_from_step(Step(type="session", name="conversation", input="question")) with pytest.raises(RecordFactoryError, match="untyped scalar"): build_record( "question", - Step(type="trace", name="request", input="question", context={"spans": []}), + Step(type="trace", name="request", input="question", children=[]), ) @@ -1149,15 +1098,276 @@ def test_record_serialization_is_json_safe() -> None: assert payload["input"] == json.dumps({"q": "x"}) -def test_factory_accepts_the_existing_pydantic_luna_record() -> None: +def test_public_core_model_validates_luna_request_record_payload() -> None: from agent_control_evaluator_galileo.luna import ScorerInvokeRecord - record = record_from_scorer_invoke_record( - ScorerInvokeRecord(type="tool", name="search", input={"query": "q"}) + source = ToolSpan(name="search", input=json.dumps({"query": "q"})) + request_record = ScorerInvokeRecord.model_validate( + source.model_dump(mode="json", exclude_none=True) ) + record = ToolSpan.model_validate(request_record.model_dump(mode="json", exclude_none=True)) - assert isinstance(record, ToolSpan) - assert json.loads(record.input) == {"query": "q"} + assert record == source + + +def test_canonical_records_round_trip_through_luna_wire_model() -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRecord + + llm_step = Step( + type="llm", + name="answer", + input=[{"role": "user", "content": "question"}], + output="answer", + tools=[{"name": "lookup"}], + context={"metadata": {"source": "llm"}}, + ) + nested_tool = Step( + type="tool", + name="search", + input={"query": "q"}, + output="result", + context={"metadata": {"source": "tool"}}, + children=[llm_step], + ) + records = [ + ( + llm_step, + LlmSpan, + ), + ( + Step(type="tool", name="flat tool", input={"id": 1}, output="done"), + ToolSpan, + ), + ( + Step( + type="retriever", + name="retrieve", + input="query", + output=[{"content": "document", "metadata": {"source": "kb"}}], + ), + RetrieverSpan, + ), + ( + Step( + type="trace", + name="trace", + input="trace input", + output="trace output", + children=[nested_tool], + ), + Trace, + ), + ( + Step( + type="session", + name="session", + input="session input", + output="session output", + children=[ + Step( + type="trace", + name="session trace", + input="trace input", + children=[ + Step( + type="tool", + name="session tool", + input={"query": "session query"}, + output="session result", + ) + ], + ) + ], + ), + Session, + ), + ] + + for step, core_model in records: + canonical = record_from_step(step) + wire_json = canonical.model_dump(mode="json", exclude_none=True) + request_record = ScorerInvokeRecord.model_validate(wire_json) + reconstructed = core_model.model_validate( + request_record.model_dump(mode="json", exclude_none=True) + ) + + assert type(reconstructed) is type(canonical) + assert reconstructed.model_dump(mode="json", exclude_none=True) == wire_json + + trace = record_from_step(records[3][0]) + assert isinstance(trace, Trace) + assert isinstance(trace.spans[0], ToolSpan) + assert isinstance(trace.spans[0].spans[0], LlmSpan) + assert trace.spans[0].user_metadata == {"source": "tool"} + assert trace.spans[0].spans[0].tools == [{"name": "lookup"}] + + session = record_from_step(records[4][0]) + assert isinstance(session, Session) + assert isinstance(session.traces[0], Trace) + assert isinstance(session.traces[0].spans[0], ToolSpan) + + +def test_luna_wire_hierarchy_uses_top_level_fields_not_context() -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRecord + from pydantic import ValidationError + + canonical_trace = record_from_step( + Step( + type="trace", + name="canonical trace", + input="trace input", + children=[Step(type="tool", name="canonical tool", input={})], + ) + ) + canonical_session = record_from_step( + Step( + type="session", + name="canonical session", + input="session input", + children=[ + Step(type="trace", name="canonical child trace", input="trace input", children=[]) + ], + ) + ) + trace_payload = canonical_trace.model_dump(mode="json", exclude_none=True) + session_payload = canonical_session.model_dump(mode="json", exclude_none=True) + assert "spans" in trace_payload + assert "traces" in session_payload + assert "context" not in trace_payload + assert "context" not in session_payload + + wire_trace = ScorerInvokeRecord.model_validate( + { + "type": "trace", + "name": "trace", + "input": "trace input", + "spans": [{"type": "tool", "name": "wire tool", "input": "{}"}], + "context": { + "children": [{"type": "llm", "name": "ignored child"}], + "spans": [{"type": "llm", "name": "ignored span"}], + "traces": [{"type": "trace", "name": "ignored trace"}], + }, + } + ) + wire_trace_json = wire_trace.model_dump(mode="json", exclude_none=True) + try: + rebuilt_trace = Trace.model_validate(wire_trace_json) + except ValidationError as exc: + assert any("context" in error["loc"] for error in exc.errors()) + else: + assert len(rebuilt_trace.spans) == 1 + assert isinstance(rebuilt_trace.spans[0], ToolSpan) + assert rebuilt_trace.spans[0].name == "wire tool" + assert all(span.name != "ignored span" for span in rebuilt_trace.spans) + + context_only_session = ScorerInvokeRecord.model_validate( + { + "type": "session", + "name": "session", + "context": {"traces": [{"type": "trace", "name": "ignored trace"}]}, + } + ) + context_only_json = context_only_session.model_dump(mode="json", exclude_none=True) + with pytest.raises(ValidationError) as missing_spans: + Trace.model_validate( + ScorerInvokeRecord.model_validate( + { + "type": "trace", + "name": "context only trace", + "context": { + "children": [{"type": "tool", "name": "ignored child"}], + "spans": [{"type": "tool", "name": "ignored span"}], + }, + } + ).model_dump(mode="json", exclude_none=True) + ) + assert any(error["loc"] == ("spans",) for error in missing_spans.value.errors()) + + with pytest.raises(ValidationError) as missing_traces: + Session.model_validate(context_only_json) + assert any(error["loc"] == ("traces",) for error in missing_traces.value.errors()) + + +def test_luna_wire_round_trip_preserves_all_trace_span_types_and_nested_tool_span() -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRecord + + trace_step = Step( + type="trace", + name="trace", + input="question", + children=[ + Step(type="llm", name="answer", input="question", output="answer"), + Step(type="tool", name="search", input={"query": "q"}, output="result"), + Step( + type="retriever", + name="retrieve", + input="query", + output=[{"content": "document"}], + ), + ], + ) + tool_step = Step( + type="tool", + name="outer tool", + input={"query": "q"}, + children=[Step(type="llm", name="inner llm", input="tool prompt")], + ) + + for step, core_model in ((trace_step, Trace), (tool_step, ToolSpan)): + canonical = record_from_step(step) + wire_json = canonical.model_dump(mode="json", exclude_none=True) + request_record = ScorerInvokeRecord.model_validate(wire_json) + reconstructed = core_model.model_validate( + request_record.model_dump(mode="json", exclude_none=True) + ) + + assert type(reconstructed) is type(canonical) + assert reconstructed.model_dump(mode="json", exclude_none=True) == wire_json + + trace = record_from_step(trace_step) + assert isinstance(trace, Trace) + assert [type(span) for span in trace.spans] == [LlmSpan, ToolSpan, RetrieverSpan] + + tool = record_from_step(tool_step) + assert isinstance(tool, ToolSpan) + assert isinstance(tool.spans[0], LlmSpan) + assert tool.spans[0].name == "inner llm" + + +@pytest.mark.parametrize( + ("payload", "core_model"), + [ + ({"type": "trace", "name": "trace", "input": "question"}, Trace), + ({"type": "session", "name": "session"}, Session), + ( + { + "type": "session", + "name": "session", + "traces": [{"type": "tool", "name": "invalid child"}], + }, + Session, + ), + ( + { + "type": "trace", + "name": "trace", + "spans": [{"type": "session", "name": "invalid child", "traces": []}], + }, + Trace, + ), + ], +) +def test_luna_wire_records_reject_missing_or_invalid_hierarchy( + payload: dict[str, object], + core_model: type[BaseModel], +) -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeRecord + from pydantic import ValidationError + + request_record = ScorerInvokeRecord.model_validate(payload) + + with pytest.raises(ValidationError): + core_model.model_validate(request_record.model_dump(mode="json", exclude_none=True)) def test_factory_rejects_invalid_boundaries_and_normalizes_missing_text() -> None: @@ -1165,8 +1375,6 @@ def test_factory_rejects_invalid_boundaries_and_normalizes_missing_text() -> Non with pytest.raises(RecordFactoryError, match="complete Agent Control Step"): record_from_step(object()) # type: ignore[arg-type] - with pytest.raises(UnsupportedStepTypeError, match="unsupported"): - record_from_scorer_invoke_record(_RecordPayload(type="unsupported")) assert GalileoRecordNormalizer.retriever_input(None) == "" assert GalileoRecordNormalizer.session_input(Document(content="plain")) == json.dumps( {"content": "plain"} From 7d2f4a59eb87670ed2cfe2c3c8a90723bef5fce7 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 5 Oct 2026 17:17:22 -0700 Subject: [PATCH 11/17] fix(sdk-ts): update generated Step children model --- sdks/typescript/src/generated/models/step.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdks/typescript/src/generated/models/step.ts b/sdks/typescript/src/generated/models/step.ts index e5dd9775..cee84a13 100644 --- a/sdks/typescript/src/generated/models/step.ts +++ b/sdks/typescript/src/generated/models/step.ts @@ -9,6 +9,10 @@ import { remap as remap$ } from "../lib/primitives.js"; * Runtime payload for an agent step invocation. */ export type Step = { + /** + * Optional child steps associated with this invocation + */ + children?: Array | null | undefined; /** * Optional context (conversation history, metadata, etc.) */ @@ -41,6 +45,7 @@ export type Step = { /** @internal */ export type Step$Outbound = { + children?: Array | null | undefined; context?: { [k: string]: any } | null | undefined; ground_truth?: any | null | undefined; input: any; @@ -53,6 +58,9 @@ export type Step$Outbound = { /** @internal */ export const Step$outboundSchema: z.ZodMiniType = z.pipe( z.object({ + children: z.optional( + z.nullable(z.array(z.lazy(() => Step$outboundSchema))), + ), context: z.optional(z.nullable(z.record(z.string(), z.any()))), groundTruth: z.optional(z.nullable(z.any())), input: z.any(), From 38e3d9506e9ba3632b667fc41478eb302e709880 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 5 Oct 2026 17:35:51 -0700 Subject: [PATCH 12/17] test(galileo): align record contract assertions --- .../records/factory.py | 14 ++++-- .../galileo/tests/test_luna_evaluator.py | 13 ++++- .../galileo/tests/test_records_factory.py | 49 +++++++++---------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py index 5b67261c..c7623ad6 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py @@ -7,13 +7,21 @@ from agent_control_models import Step from galileo_core.schemas.logging.session import Session -from galileo_core.schemas.logging.span import LlmSpan, RetrieverSpan, ToolSpan +from galileo_core.schemas.logging.span import ( + AgentSpan, + ControlSpan, + LlmSpan, + RetrieverSpan, + ToolSpan, + WorkflowSpan, +) from galileo_core.schemas.logging.trace import Trace from .normalization import GalileoRecordNormalizer type GalileoRecord = LlmSpan | ToolSpan | RetrieverSpan | Trace | Session type GalileoSpan = LlmSpan | ToolSpan | RetrieverSpan +type GalileoCoreSpan = AgentSpan | WorkflowSpan | LlmSpan | RetrieverSpan | ToolSpan | ControlSpan _MISSING = object() _SPAN_TYPES = {"llm", "tool", "retriever"} _RECORD_TYPES = _SPAN_TYPES | {"trace", "session"} @@ -45,8 +53,8 @@ def _selected_values( return selected_data, None -def _span_children(children: list[GalileoRecord], *, parent_type: str) -> list[GalileoSpan]: - spans: list[GalileoSpan] = [] +def _span_children(children: list[GalileoRecord], *, parent_type: str) -> list[GalileoCoreSpan]: + spans: list[GalileoCoreSpan] = [] for child in children: if not isinstance(child, (LlmSpan, ToolSpan, RetrieverSpan)): raise RecordFactoryError(f"Galileo {parent_type} children must be span steps.") diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 1e5473b7..86880af0 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -849,7 +849,16 @@ def handler(request: httpx.Request) -> httpx.Response: ).model_dump(mode="json", exclude_none=True), "config": {"request_timeout_seconds": 8.0}, } - assert captured["body"] == expected_body + actual_body = captured["body"] + assert isinstance(actual_body, dict) + actual_record = actual_body["record"] + expected_record = expected_body["record"] + assert isinstance(actual_record, dict) + assert isinstance(expected_record, dict) + # Each factory call assigns a fresh created_at timestamp. + actual_record.pop("created_at", None) + expected_record.pop("created_at", None) + assert actual_body == expected_body # The emitted body also satisfies the exact additive Orbit #1720 shape. orbit_request = _Orbit1720Request.model_validate(expected_body) @@ -859,7 +868,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert "selected question" in json.dumps(orbit_request.record.input) assert "selected answer" in json.dumps(orbit_request.record.output) assert orbit_request.inputs.tools == orbit_request.record.tools - assert orbit_request.record.dataset_output == {"text": "expected"} + assert json.loads(orbit_request.record.dataset_output) == {"text": "expected"} assert orbit_request.inputs.ground_truth == {"text": "expected"} assert orbit_request.record.user_metadata == {"source": "runtime"} assert "context" not in expected_body["record"] diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index eadd112b..2be55d58 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -880,12 +880,10 @@ class DocumentModel(BaseModel): def test_llm_tool_definitions_normalize_nested_pydantic_uuid_and_datetime_values() -> None: identifier = uuid4() timestamp = datetime(2025, 1, 2, tzinfo=UTC) - source = LlmSpan( - input="question", - tools=[{"id": identifier, "created_at": timestamp}], + normalized_tools = GalileoRecordNormalizer.llm_tools( + [{"id": identifier, "created_at": timestamp}] ) - wire_json = source.model_dump(mode="json", exclude_none=True) - record = LlmSpan.model_validate(wire_json) + record = LlmSpan.model_validate({"input": "question", "tools": normalized_tools}) assert isinstance(record, LlmSpan) assert record.tools == [ @@ -1260,6 +1258,23 @@ def test_luna_wire_hierarchy_uses_top_level_fields_not_context() -> None: assert rebuilt_trace.spans[0].name == "wire tool" assert all(span.name != "ignored span" for span in rebuilt_trace.spans) + # Galileo Core supplies empty hierarchy defaults when top-level fields are absent. + # Context-only hierarchy must not be treated as canonical spans or traces. + context_only_trace = ScorerInvokeRecord.model_validate( + { + "type": "trace", + "name": "context only trace", + "context": { + "children": [{"type": "tool", "name": "ignored child"}], + "spans": [{"type": "tool", "name": "ignored span"}], + }, + } + ) + rebuilt_context_only_trace = Trace.model_validate( + context_only_trace.model_dump(mode="json", exclude_none=True) + ) + assert rebuilt_context_only_trace.spans == [] + context_only_session = ScorerInvokeRecord.model_validate( { "type": "session", @@ -1268,24 +1283,8 @@ def test_luna_wire_hierarchy_uses_top_level_fields_not_context() -> None: } ) context_only_json = context_only_session.model_dump(mode="json", exclude_none=True) - with pytest.raises(ValidationError) as missing_spans: - Trace.model_validate( - ScorerInvokeRecord.model_validate( - { - "type": "trace", - "name": "context only trace", - "context": { - "children": [{"type": "tool", "name": "ignored child"}], - "spans": [{"type": "tool", "name": "ignored span"}], - }, - } - ).model_dump(mode="json", exclude_none=True) - ) - assert any(error["loc"] == ("spans",) for error in missing_spans.value.errors()) - - with pytest.raises(ValidationError) as missing_traces: - Session.model_validate(context_only_json) - assert any(error["loc"] == ("traces",) for error in missing_traces.value.errors()) + rebuilt_context_only_session = Session.model_validate(context_only_json) + assert rebuilt_context_only_session.traces == [] def test_luna_wire_round_trip_preserves_all_trace_span_types_and_nested_tool_span() -> None: @@ -1337,8 +1336,6 @@ def test_luna_wire_round_trip_preserves_all_trace_span_types_and_nested_tool_spa @pytest.mark.parametrize( ("payload", "core_model"), [ - ({"type": "trace", "name": "trace", "input": "question"}, Trace), - ({"type": "session", "name": "session"}, Session), ( { "type": "session", @@ -1357,7 +1354,7 @@ def test_luna_wire_round_trip_preserves_all_trace_span_types_and_nested_tool_spa ), ], ) -def test_luna_wire_records_reject_missing_or_invalid_hierarchy( +def test_luna_wire_records_reject_invalid_hierarchy( payload: dict[str, object], core_model: type[BaseModel], ) -> None: From 4d32733753970605c239865c36b9b8871c4fc130 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 6 Oct 2026 17:37:38 -0700 Subject: [PATCH 13/17] fix(galileo): preserve structured Luna selector data --- .../luna/client.py | 29 ++- .../luna/evaluator.py | 2 + .../galileo/tests/test_luna_evaluator.py | 222 ++++++++++++++++++ 3 files changed, 247 insertions(+), 6 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index f14087d2..9ed4367b 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -11,7 +11,7 @@ from hmac import new as hmac_new from json import dumps from time import time -from typing import Literal, get_args +from typing import Any, Literal, get_args from urllib.parse import urlsplit import httpx @@ -58,6 +58,7 @@ "session", ] SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) +_MISSING_SELECTED_DATA = object() def _b64url(data: bytes) -> str: @@ -280,6 +281,8 @@ def _scorer_invoke_record_from_step( *, selected_input: JSONValue, selected_output: JSONValue, + selected_data: Any = _MISSING_SELECTED_DATA, + payload_field: str = "input", ) -> ScorerInvokeRecord | None: """Translate a generic Agent Control step into Orbit's record contract. @@ -303,11 +306,18 @@ def _scorer_invoke_record_from_step( if step is None: return None try: - record = record_from_step( - step, - selected_input=selected_input, - selected_output=selected_output, - ) + if selected_data is _MISSING_SELECTED_DATA: + record = record_from_step( + step, + selected_input=selected_input, + selected_output=selected_output, + ) + else: + record = record_from_step( + step, + selected_data=selected_data, + payload_field=payload_field, + ) except UnsupportedStepTypeError: return None canonical_record = record.model_dump(mode="json", exclude_none=True) @@ -489,6 +499,8 @@ async def invoke( input: JSONValue = None, output: JSONValue = None, step: Step | None = None, + selected_data: Any = _MISSING_SELECTED_DATA, + selected_data_payload_field: str = "input", execution_context: GalileoExecutionContext | None = None, config: ScorerInvokeConfig | JSONObject | None = None, timeout: float = DEFAULT_TIMEOUT_SECS, @@ -504,6 +516,9 @@ async def invoke( input: Optional user/system prompt text. output: Optional model response text. step: Optional complete runtime step used for structured dual-write. + selected_data: Raw selector-selected value used to construct the record, + preserving its structure independently from serialized legacy inputs. + selected_data_payload_field: Record input/output side for scalar selector values. execution_context: Optional authenticated Galileo scorer context. config: Optional Orbit-supported scorer invocation configuration. timeout: Request timeout in seconds. @@ -547,6 +562,8 @@ async def invoke( step, selected_input=input, selected_output=output, + selected_data=selected_data, + payload_field=selected_data_payload_field, ), execution_context=execution_context, config=invoke_config, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 14b58d88..a09b4ccf 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -281,6 +281,8 @@ async def _evaluate( scorer_kwargs = self._scorer_kwargs() if step is not None: scorer_kwargs["step"] = step + scorer_kwargs["selected_data"] = data + scorer_kwargs["selected_data_payload_field"] = self.config.payload_field if execution_context is not None: scorer_kwargs["execution_context"] = execution_context response = await self._get_client().invoke( diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 86880af0..74a8a709 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -1026,6 +1026,220 @@ def handler(request: httpx.Request) -> httpx.Response: assert isinstance(nested_spans, list) assert nested_spans[0]["name"] == "nested" + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_posts_trace_with_raw_structured_selector_data(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.8, "status": "success"}) + + step = Step( + type="trace", + name="account_lookup_trace", + input={"messages": [{"role": "user", "content": "Find my account information."}]}, + output={"content": "Your account belongs to Jane Doe."}, + children=[ + Step( + type="llm", + name="account_agent", + input=[{"role": "user", "content": "Find my account information."}], + output={"content": "Looking up the account."}, + ), + Step( + type="tool", + name="lookup_account", + input={"account_id": "123"}, + output={"name": "Jane Doe"}, + children=[ + Step( + type="tool", + name="account_service", + input={"account_id": "123"}, + output={"name": "Jane Doe"}, + ) + ], + ), + Step( + type="retriever", + name="account_docs", + input={"query": "account 123"}, + output=[{"content": "Account owner: Jane Doe", "metadata": {"id": "d1"}}], + ), + ], + tools=[{"name": "lookup_account", "description": "Lookup", "input_schema": {}}], + ground_truth={"owner": "Jane Doe"}, + ) + selected = step.model_dump(mode="json") + evaluator = LunaEvaluator.from_dict( + { + "scorer_id": SCORER_ID, + "scorer_version_id": SCORER_VERSION_ID, + "scorer_label": "account-check", + "threshold": 0.5, + "config": {"request_timeout_seconds": 6}, + "timeout_ms": 9000, + } + ) + evaluator._client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + result = await evaluator.evaluate_with_extensions( + selected, + step, + { + "namespace_key": "org-1", + "caller_id": "user-2", + "target_type": "log_stream", + "target_id": "run-3", + "metadata": {"project_id": "project-4"}, + }, + ) + finally: + await evaluator._client.close() + + assert result.error is None + body = captured["body"] + assert isinstance(body, dict) + assert body["scorer_id"] == SCORER_ID + assert body["scorer_version_id"] == SCORER_VERSION_ID + assert body["scorer_label"] == "account-check" + assert body["execution_context"] == { + "organization_id": "org-1", + "user_id": "user-2", + "project_id": "project-4", + "run_id": "run-3", + } + assert body["inputs"] == { + "query": json.dumps(step.input, ensure_ascii=False, sort_keys=True), + "response": json.dumps(step.output, ensure_ascii=False, sort_keys=True), + "ground_truth": {"owner": "Jane Doe"}, + "tools": step.tools, + } + assert body["config"] == {"request_timeout_seconds": 6.0} + record = body["record"] + assert isinstance(record, dict) + assert record["type"] == "trace" + assert json.loads(record["input"]) == step.input + assert json.loads(record["output"]) == step.output + spans = record["spans"] + assert isinstance(spans, list) + assert [child["type"] for child in spans] == ["llm", "tool", "retriever"] + assert spans[0]["input"][0]["content"] == "Find my account information." + assert json.loads(spans[1]["input"]) == {"account_id": "123"} + assert spans[1]["spans"][0]["name"] == "account_service" + assert spans[2]["output"][0]["content"] == "Account owner: Jane Doe" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_posts_session_with_raw_structured_selector_data(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.8, "status": "success"}) + + step = Step( + type="session", + name="support_session", + input={"customer": "Jane Doe"}, + output={"resolution": "account found"}, + children=[ + Step( + type="trace", + name="account_lookup", + input={"query": "account info"}, + output={"owner": "Jane Doe"}, + children=[ + Step( + type="tool", + name="lookup_account", + input={"account_id": "123"}, + output={"name": "Jane Doe"}, + ) + ], + ) + ], + ) + evaluator = LunaEvaluator.from_dict( + {"scorer_id": SCORER_ID, "scorer_version_id": SCORER_VERSION_ID} + ) + evaluator._client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + result = await evaluator.evaluate_with_context(step.model_dump(mode="json"), step) + finally: + await evaluator._client.close() + + assert result.error is None + body = captured["body"] + assert isinstance(body, dict) + assert body["inputs"]["query"] == json.dumps( + step.input, ensure_ascii=False, sort_keys=True + ) + assert body["inputs"]["response"] == json.dumps( + step.output, ensure_ascii=False, sort_keys=True + ) + record = body["record"] + assert isinstance(record, dict) + assert record["type"] == "session" + assert json.loads(record["input"]) == step.input + assert json.loads(record["output"]) == step.output + traces = record["traces"] + assert isinstance(traces, list) + assert traces[0]["type"] == "trace" + assert traces[0]["spans"][0]["type"] == "tool" + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_does_not_post_trace_for_scalar_selector_or_invalid_hierarchy( + self, + ) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + + request_count = 0 + + def handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response(200, json={"score": 0.8, "status": "success"}) + + evaluator = LunaEvaluator.from_dict( + {"scorer_id": SCORER_ID, "scorer_version_id": SCORER_VERSION_ID} + ) + evaluator._client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + trace = Step( + type="trace", + name="trace", + input={"question": "q"}, + children=[Step(type="llm", name="answer", input="q", output="a")], + ) + invalid_session = Step( + type="session", + name="session", + input="q", + children=[Step(type="tool", name="invalid-child", input={"q": "q"})], + ) + + try: + scalar_result = await evaluator.evaluate_with_context("response text", trace) + hierarchy_result = await evaluator.evaluate_with_context( + invalid_session.model_dump(mode="json"), invalid_session + ) + finally: + await evaluator._client.close() + + assert scalar_result.error is not None + assert "structured selector values" in scalar_result.error + assert hierarchy_result.error is not None + assert "cannot contain 'tool' child steps" in hierarchy_result.error + assert request_count == 0 + @pytest.mark.parametrize("field", ["children", "spans", "traces"]) @pytest.mark.asyncio async def test_client_rejects_hierarchy_fields_from_step_context(self, field: str) -> None: @@ -1247,6 +1461,8 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", step=step, + selected_data="selected input", + selected_data_payload_field="input", input="selected input", output=None, config=None, @@ -1284,6 +1500,8 @@ async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", step=Step(type="llm", name="answer", input="prompt"), + selected_data="selected input", + selected_data_payload_field="input", execution_context=GalileoExecutionContext( organization_id="org-1", user_id="verified-user-2", @@ -1323,6 +1541,8 @@ async def test_evaluator_allows_missing_caller_id(self) -> None: mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", step=Step(type="llm", name="answer", input="prompt"), + selected_data="selected input", + selected_data_payload_field="input", execution_context=GalileoExecutionContext( organization_id="org-1", user_id=None, @@ -1363,6 +1583,8 @@ async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None: mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", step=Step(type="llm", name="answer", input="prompt"), + selected_data="selected input", + selected_data_payload_field="input", execution_context=GalileoExecutionContext( organization_id="org-1", user_id="verified-user-2", From 66c3697c2d70d084083573f964e269338968e3da Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 6 Oct 2026 17:47:10 -0700 Subject: [PATCH 14/17] fix(galileo): allow empty trace and session children --- .../records/factory.py | 2 -- .../galileo/tests/test_records_factory.py | 26 +++++++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py index c7623ad6..b0d5d245 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py @@ -108,8 +108,6 @@ def record_from_step( "retriever": _SPAN_TYPES, "llm": set(), }[record_type] - if record_type in {"trace", "session"} and step.children is None: - raise RecordFactoryError(f"Galileo {record_type} step is missing Step.children.") child_records: list[GalileoRecord] = [] for child in children: if not isinstance(child, Step): diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index 2be55d58..761f8d45 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -959,7 +959,7 @@ def test_trace_and_session_use_children_and_selectors_only_change_the_root() -> assert isinstance(session.traces[0].spans[0], LlmSpan) -def test_children_must_be_valid_for_parent_and_trace_session_children_are_required() -> None: +def test_children_must_be_valid_for_parent() -> None: with pytest.raises(RecordFactoryError, match="cannot contain 'session'"): record_from_step( Step(type="trace", name="trace", input="q", children=[ @@ -972,14 +972,16 @@ def test_children_must_be_valid_for_parent_and_trace_session_children_are_requir Step(type="tool", name="tool", input={}) ]) ) - with pytest.raises(RecordFactoryError, match="missing Step.children"): - record_from_step( - Step(type="trace", name="trace", input={"input": "q", "spans": []}) - ) - with pytest.raises(RecordFactoryError, match="missing Step.children"): - record_from_step( - Step(type="session", name="session", input={"input": "q", "traces": []}) - ) + + +def test_trace_and_session_without_children_have_empty_child_collections() -> None: + trace = record_from_step(Step(type="trace", name="request", input="question")) + session = record_from_step(Step(type="session", name="conversation", input="question")) + + assert isinstance(trace, Trace) + assert trace.spans == [] + assert isinstance(session, Session) + assert session.traces == [] def test_nested_tool_and_retriever_children_are_recursively_converted() -> None: @@ -1070,11 +1072,7 @@ def test_session_message_and_document_sequences_use_public_validators() -> None: ] -def test_trace_and_session_require_children_and_selector_values_must_be_structured() -> None: - with pytest.raises(RecordFactoryError, match="missing Step.children"): - record_from_step(Step(type="trace", name="request", input="question")) - with pytest.raises(RecordFactoryError, match="missing Step.children"): - record_from_step(Step(type="session", name="conversation", input="question")) +def test_trace_and_session_selector_values_must_be_structured() -> None: with pytest.raises(RecordFactoryError, match="untyped scalar"): build_record( "question", From 215fdd7d148a55e3e842f77cc5c56ebe522923cf Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 6 Oct 2026 17:53:36 -0700 Subject: [PATCH 15/17] fix the records creation for traces and sessions --- .../luna/client.py | 19 ++++--- .../galileo/tests/test_luna_evaluator.py | 52 ++++++++++++++++++- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index 9ed4367b..f0f67b93 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -548,6 +548,17 @@ async def invoke( invoke_config, http_timeout_seconds=timeout, ) + record = ( + _scorer_invoke_record_from_step( + step, + selected_input=input, + selected_output=output, + selected_data=selected_data, + payload_field=selected_data_payload_field, + ) + if step is not None and scorer_version_id is not None + else None + ) request_body = ScorerInvokeRequest( scorer_id=scorer_id, scorer_version_id=scorer_version_id, @@ -558,13 +569,7 @@ async def invoke( ground_truth=step.ground_truth if step is not None else None, tools=step.tools if step is not None else None, ), - record=_scorer_invoke_record_from_step( - step, - selected_input=input, - selected_output=output, - selected_data=selected_data, - payload_field=selected_data_payload_field, - ), + record=record, execution_context=execution_context, config=invoke_config, ).to_dict() diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 74a8a709..f56c0828 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -885,6 +885,49 @@ def handler(request: httpx.Request) -> httpx.Response: "config": {"request_timeout_seconds": 8.0}, } + @pytest.mark.asyncio + async def test_client_omits_record_when_step_has_no_pinned_version(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + step = Step( + type="llm", + name="answer", + input="complete prompt", + output="complete response", + tools=[{"name": "search", "input_schema": {}}], + ground_truth="expected answer", + ) + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + await client.invoke( + scorer_id=SCORER_ID, + input="selected prompt", + output="selected response", + step=step, + ) + finally: + await client.close() + + assert captured["body"] == { + "scorer_id": SCORER_ID, + "inputs": { + "query": "selected prompt", + "response": "selected response", + "ground_truth": "expected answer", + "tools": [{"name": "search", "input_schema": {}}], + }, + "config": {"request_timeout_seconds": 8.0}, + } + @pytest.mark.asyncio async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient @@ -946,6 +989,7 @@ def handler(request: httpx.Request) -> httpx.Response: try: await client.invoke( scorer_id="scorer-123", + scorer_version_id=SCORER_VERSION_ID, input="selected input", output="selected output", step=step, @@ -1000,6 +1044,7 @@ def handler(request: httpx.Request) -> httpx.Response: try: await client.invoke( scorer_id="scorer-123", + scorer_version_id=SCORER_VERSION_ID, input={"query": "selected"}, output={"answer": "selected"}, step=step, @@ -1262,7 +1307,12 @@ async def test_client_rejects_hierarchy_fields_from_step_context(self, field: st try: with pytest.raises(RecordFactoryError, match="Step.context cannot define"): - await client.invoke(scorer_id="scorer-123", input="selected", step=step) + await client.invoke( + scorer_id="scorer-123", + scorer_version_id=SCORER_VERSION_ID, + input="selected", + step=step, + ) finally: await client.close() From 77a4763b1ac207763acdfae3885ab64745abac65 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 6 Oct 2026 19:53:42 -0700 Subject: [PATCH 16/17] add messages as part of input for llm records --- .../records/normalization.py | 48 ++++++++++- .../galileo/tests/test_luna_evaluator.py | 43 ++++++++++ .../galileo/tests/test_records_factory.py | 86 +++++++++++++++++++ 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py index c491602a..53f118bf 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/normalization.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import json from collections.abc import Mapping, Sequence from dataclasses import asdict, is_dataclass @@ -180,6 +181,36 @@ def _message_sequence(value: Any) -> bool: ) +def _llm_message_sequence(value: Any) -> Sequence[Any] | None: + """Extract a role-bearing LLM message list from structured or string input.""" + if isinstance(value, str): + try: + parsed = json.loads(value) + except (RecursionError, ValueError): + try: + parsed = ast.literal_eval(value) + except (SyntaxError, ValueError, TypeError, RecursionError): + return None + if isinstance(parsed, Mapping): + candidates = parsed.get("messages") + if not isinstance(candidates, list): + return None + elif isinstance(parsed, list): + candidates = parsed + else: + return None + else: + candidates = value.get("messages") if isinstance(value, Mapping) else value + if not isinstance(candidates, Sequence) or isinstance(candidates, str | bytes | bytearray): + return None + if not all( + isinstance(item, Message) or (isinstance(item, Mapping) and "role" in item) + for item in candidates + ): + return None + return candidates + + def _flatten_message_sequence(value: Sequence[Any]) -> list[dict[str, Any]]: blocks: list[dict[str, Any]] = [] for message in value: @@ -246,11 +277,18 @@ def llm_input(cls, value: Any) -> Any: if value is None: return "" if isinstance(value, str | Message): - return value + messages = _llm_message_sequence(value) + if messages is None: + return value + return [_json_compatible(item) for item in messages] if isinstance(value, Mapping): + messages = _llm_message_sequence(value) + if messages is not None: + return [_json_compatible(item) for item in messages] return _json_compatible(value) if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): - if all(isinstance(item, Message | Mapping) for item in value): + messages = _llm_message_sequence(value) + if messages is not None: return [_json_compatible(item) for item in value] return cls.json_text(value) @@ -300,7 +338,11 @@ def tool_redacted_output(cls, value: Any) -> str | None: @classmethod def retriever_input(cls, value: Any) -> str: - return "" if value is None else cls.json_text(value) + if value is None: + return "" + if isinstance(value, Mapping) and "query" in value: + return cls.json_text(value["query"]) + return cls.json_text(value) @staticmethod def retriever_output(value: Any) -> list[Document]: diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index f56c0828..3d986216 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -928,6 +928,49 @@ def handler(request: httpx.Request) -> httpx.Response: "config": {"request_timeout_seconds": 8.0}, } + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_normalizes_record_without_changing_legacy_inputs(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + + captured: dict[str, object] = {} + raw_messages = "{'messages': [{'role': 'user', 'content': 'wire policy'}]}" + raw_response = "{'answer': 'approved'}" + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + step = Step( + type="llm", + name="answer", + input=raw_messages, + output=raw_response, + ) + evaluator = LunaEvaluator.from_dict( + {"scorer_id": SCORER_ID, "scorer_version_id": SCORER_VERSION_ID} + ) + evaluator._client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + result = await evaluator.evaluate_with_context( + {"input": raw_messages, "output": raw_response}, + step, + ) + finally: + await evaluator._client.close() + + assert result.error is None + body = captured["body"] + assert isinstance(body, dict) + assert body["inputs"]["query"] == raw_messages + assert body["inputs"]["response"] == raw_response + assert body["scorer_version_id"] == SCORER_VERSION_ID + record = body["record"] + assert isinstance(record, dict) + assert record["input"] == [{"role": "user", "content": "wire policy"}] + assert record["output"] == {"content": raw_response, "role": "assistant"} + @pytest.mark.asyncio async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient diff --git a/evaluators/contrib/galileo/tests/test_records_factory.py b/evaluators/contrib/galileo/tests/test_records_factory.py index 761f8d45..61f0be74 100644 --- a/evaluators/contrib/galileo/tests/test_records_factory.py +++ b/evaluators/contrib/galileo/tests/test_records_factory.py @@ -164,6 +164,60 @@ def test_llm_messages_use_public_canonical_validation() -> None: assert record.dataset_output == json.dumps({"expected": "answer"}) +@pytest.mark.parametrize( + ("source", "expected_messages"), + [ + ("A plain prompt", None), + ( + '[{"role": "user", "content": "question"}]', + [{"role": "user", "content": "question"}], + ), + ( + "{'messages': [{'role': 'user', 'content': 'question'}]}", + [{"role": "user", "content": "question"}], + ), + ("{'messages': [", None), + ('{"content": "document without a role"}', None), + ('[{"content": "document without a role"}]', None), + ], +) +def test_llm_string_input_only_parses_role_bearing_message_shapes( + source: str, + expected_messages: list[dict[str, str]] | None, +) -> None: + record = record_from_step(Step(type="llm", name="answer", input=source)) + + assert isinstance(record, LlmSpan) + if expected_messages is None: + assert len(record.input) == 1 + assert record.input[0].content == source + else: + assert [ + {"role": message.role.value, "content": message.content} + for message in record.input + ] == expected_messages + + restored = LlmSpan.model_validate(record.model_dump(mode="json", exclude_none=True)) + assert [message.content for message in restored.input] == [ + message.content for message in record.input + ] + + +def test_llm_mapping_with_messages_key_becomes_public_message_models() -> None: + record = record_from_step( + Step( + type="llm", + name="answer", + input={"messages": [{"role": "user", "content": "question"}]}, + ) + ) + + assert isinstance(record, LlmSpan) + assert [message.role.value for message in record.input] == ["user"] + restored = LlmSpan.model_validate(record.model_dump(mode="json", exclude_none=True)) + assert [message.content for message in restored.input] == ["question"] + + @pytest.mark.parametrize( ("output", "expected"), [ @@ -902,6 +956,38 @@ def test_retriever_dictionaries_become_canonical_documents() -> None: assert documents[0].metadata == {"source": "kb"} +def test_retriever_input_mapping_extracts_query_and_round_trips_documents() -> None: + record = record_from_step( + Step( + type="retriever", + name="policy_search", + input={"query": "wire policy", "partition": "banking"}, + output=[{"content": "Policy text", "metadata": {"source": "kb"}}], + ) + ) + + assert isinstance(record, RetrieverSpan) + assert record.input == "wire policy" + assert len(record.output) == 1 + assert isinstance(record.output[0], Document) + restored = RetrieverSpan.model_validate(record.model_dump(mode="json", exclude_none=True)) + assert restored.input == "wire policy" + assert isinstance(restored.output[0], Document) + assert restored.output[0].content == "Policy text" + + +def test_retriever_input_mapping_without_query_uses_json_fallback() -> None: + record = record_from_step( + Step(type="retriever", name="search", input={"partition": "banking"}) + ) + + assert isinstance(record, RetrieverSpan) + assert record.input == '{"partition": "banking"}' + assert GalileoRecordNormalizer.retriever_input({"query": {"phrase": "wire"}}) == ( + '{"phrase": "wire"}' + ) + + def test_trace_and_session_use_children_and_selectors_only_change_the_root() -> None: trace = record_from_step( Step( From 094b0a6b8899e518998d6b64c9fb2bd9da7939d5 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Wed, 7 Oct 2026 12:54:30 -0700 Subject: [PATCH 17/17] address comment --- .../luna/client.py | 9 +++++--- .../records/factory.py | 1 + .../src/agent_control/control_decorators.py | 22 ++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index f0f67b93..d6cc72c1 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -307,20 +307,23 @@ def _scorer_invoke_record_from_step( return None try: if selected_data is _MISSING_SELECTED_DATA: - record = record_from_step( + galileo_core_record = record_from_step( step, selected_input=selected_input, selected_output=selected_output, ) else: - record = record_from_step( + galileo_core_record = record_from_step( step, selected_data=selected_data, payload_field=payload_field, ) except UnsupportedStepTypeError: return None - canonical_record = record.model_dump(mode="json", exclude_none=True) + # record_from_step() returns Galileo Core models (LlmSpan, Trace, etc.). Dumping + # and revalidating coerces the model into the Luna request shape while preserving + # extra canonical fields that Orbit accepts but ScorerInvokeRecord does not declare. + canonical_record = galileo_core_record.model_dump(mode="json", exclude_none=True) if canonical_record.get("type") not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: return None return ScorerInvokeRecord.model_validate(canonical_record) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py index b0d5d245..75a3e5b6 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/records/factory.py @@ -36,6 +36,7 @@ class UnsupportedStepTypeError(RecordFactoryError): def _mapping(value: Any) -> Mapping[str, Any] | None: + """Return mapping values unchanged, or ``None`` for non-mapping inputs.""" return value if isinstance(value, Mapping) else None diff --git a/sdks/python/src/agent_control/control_decorators.py b/sdks/python/src/agent_control/control_decorators.py index cfd02219..8c12ae4c 100644 --- a/sdks/python/src/agent_control/control_decorators.py +++ b/sdks/python/src/agent_control/control_decorators.py @@ -445,13 +445,11 @@ def _unexpected_control_failure_message(stage: str, error: Exception) -> str: async def _run_control_check( ctx: ControlContext, stage: str, - payload: dict[str, Any] | Callable[[], dict[str, Any]], + payload: dict[str, Any], controls: list[dict[str, Any]] | None, ) -> None: """Run one control stage and enforce fail-closed behavior on unexpected errors.""" try: - if callable(payload): - payload = payload() result = await _evaluate( ctx.agent_name, payload, @@ -827,7 +825,14 @@ async def _execute_with_control( try: # PRE-EXECUTION: Check controls with check_stage="pre" - await _run_control_check(ctx, "pre", ctx.pre_payload, controls) + try: + pre_payload = ctx.pre_payload() + except _InvalidToolsConfigurationError: + raise + except Exception as e: + logger.error("Pre-execution control payload creation failed: %s", e, exc_info=True) + raise RuntimeError(_unexpected_control_failure_message("pre", e)) from e + await _run_control_check(ctx, "pre", pre_payload, controls) # Execute the function if is_async: @@ -836,7 +841,14 @@ async def _execute_with_control( output = func(*args, **kwargs) # POST-EXECUTION: Check controls with check_stage="post" - await _run_control_check(ctx, "post", lambda: ctx.post_payload(output), controls) + try: + post_payload = ctx.post_payload(output) + except _InvalidToolsConfigurationError: + raise + except Exception as e: + logger.error("Post-execution control payload creation failed: %s", e, exc_info=True) + raise RuntimeError(_unexpected_control_failure_message("post", e)) from e + await _run_control_check(ctx, "post", post_payload, controls) return output finally: