diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index cca27c66..45d65333 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -229,12 +229,12 @@ class ScorerInvokeRecord(BaseModel): class GalileoExecutionContext(BaseModel): - """Authenticated identity fields required by Galileo scorer invocation.""" + """Authenticated organization and optional identity fields for Galileo scorer invocation.""" organization_id: str = Field(min_length=1) - user_id: str = Field(min_length=1) - project_id: str = Field(min_length=1) - run_id: str = Field(min_length=1) + user_id: str | None = Field(default=None, min_length=1) + project_id: str | None = Field(default=None, min_length=1) + run_id: str | None = Field(default=None, min_length=1) class ScorerInvokeRequest(BaseModel): @@ -266,7 +266,10 @@ def ensure_required_values(self) -> ScorerInvokeRequest: def to_dict(self) -> JSONObject: """Convert to the Luna scorer invoke request shape.""" - return self.model_dump(mode="json", exclude_none=True) + request = self.model_dump(mode="json", exclude_none=True) + if self.execution_context is not None: + request["execution_context"] = self.execution_context.model_dump(mode="json") + return request def _orbit_record_from_step( diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 99085e26..14b58d88 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -236,38 +236,26 @@ def _execution_context_from_extensions( metadata = extensions.get("metadata") metadata_obj = metadata if isinstance(metadata, dict) else {} organization_id = extensions.get("namespace_key") - # Orbit's runtime-auth contract defines caller_id as the authenticated - # user ID for this flow. Keep that contract mapping inside Galileo. + # The runtime envelope's caller_id is supplied by the authenticated + # principal and is the caller identity Galileo associates with this run. user_id = extensions.get("caller_id") project_id = metadata_obj.get("project_id") target_type = extensions.get("target_type") target_id = extensions.get("target_id") - run_id = target_id if target_type == "log_stream" else None - - missing = [ - field - for field, value in ( - ("organization_id", organization_id), - ("user_id", user_id), - ("project_id", project_id), - ("run_id", run_id), - ) - if not isinstance(value, str) or not value - ] - if missing: - raise ValueError( - "Authenticated execution metadata is missing required fields: " - + ", ".join(missing) - ) + run_id = ( + target_id + if target_type in ("log_stream", "agent_stream") + and isinstance(target_id, str) + and target_id + else None + ) + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("Authenticated execution metadata is missing organization_id") - assert isinstance(organization_id, str) - assert isinstance(user_id, str) - assert isinstance(project_id, str) - assert isinstance(run_id, str) return GalileoExecutionContext( organization_id=organization_id, - user_id=user_id, - project_id=project_id, + user_id=user_id if isinstance(user_id, str) and user_id else None, + project_id=project_id if isinstance(project_id, str) and project_id else None, run_id=run_id, ) diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index b368f24d..cf3d5423 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -246,6 +246,23 @@ def test_scorer_invoke_request_serialization_keeps_legacy_record_shape(self) -> "config": {}, } + def test_execution_context_serializes_missing_optional_claims_as_null(self) -> None: + from agent_control_evaluator_galileo.luna import ScorerInvokeInputs, ScorerInvokeRequest + from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext + + request = ScorerInvokeRequest( + scorer_id="scorer-123", + inputs=ScorerInvokeInputs(query="hello"), + execution_context=GalileoExecutionContext(organization_id="org-1"), + ) + + assert request.to_dict()["execution_context"] == { + "organization_id": "org-1", + "user_id": None, + "project_id": None, + "run_id": None, + } + def test_scorer_invoke_request_rejects_unknown_config(self) -> None: from agent_control_evaluator_galileo.luna import ( ScorerInvokeInputs, @@ -1064,7 +1081,8 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio - async def test_evaluator_consumes_verified_opaque_extensions(self) -> None: + @pytest.mark.parametrize("target_type", ["log_stream", "agent_stream"]) + async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: str) -> None: from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse from agent_control_evaluator_galileo.luna.client import ( GalileoExecutionContext, @@ -1077,7 +1095,7 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None: extensions = { "namespace_key": "org-1", "caller_id": "verified-user-2", - "target_type": "log_stream", + "target_type": target_type, "target_id": "run-4", "metadata": {"project_id": "project-3"}, } @@ -1106,25 +1124,105 @@ async def test_evaluator_consumes_verified_opaque_extensions(self) -> None: @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio - async def test_evaluator_requires_authenticated_caller_id(self) -> None: - from agent_control_evaluator_galileo.luna import LunaEvaluator - from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + async def test_evaluator_allows_missing_caller_id(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import ( + GalileoExecutionContext, + GalileoLunaClient, + ) evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) extensions = { "namespace_key": "org-1", "target_type": "log_stream", "target_id": "run-4", + "metadata": {}, + } + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate_with_extensions( + "selected input", Step(type="llm", name="answer", input="prompt"), extensions + ) + + assert result.error is None + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=Step(type="llm", name="answer", input="prompt"), + execution_context=GalileoExecutionContext( + organization_id="org-1", + user_id=None, + project_id=None, + run_id="run-4", + ), + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import ( + GalileoExecutionContext, + GalileoLunaClient, + ) + + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + extensions = { + "namespace_key": "org-1", + "caller_id": "verified-user-2", + "target_type": "environment", + "target_id": "prod", "metadata": {"project_id": "project-3"}, } with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") result = await evaluator.evaluate_with_extensions( "selected input", Step(type="llm", name="answer", input="prompt"), extensions ) + assert result.error is None + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=Step(type="llm", name="answer", input="prompt"), + execution_context=GalileoExecutionContext( + organization_id="org-1", + user_id="verified-user-2", + project_id="project-3", + run_id=None, + ), + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_rejects_missing_organization_id(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + extensions = { + "target_type": "log_stream", + "target_id": "run-4", + "metadata": {"project_id": "project-3"}, + } + + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + result = await evaluator.evaluate_with_extensions( + "selected input", + Step(type="llm", name="answer", input="prompt"), + extensions, + ) + assert result.error is not None - assert "user_id" in result.error + assert "organization_id" in result.error mock_invoke.assert_not_called() @patch.dict(os.environ, LUNA_ENV)