feature flag and request runner's only when step and execution_context present

Author: wrisa

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py

@@ -522,11 +522,12 @@ async def invoke(
             scorer_label: Optional display/metadata label.
             input: Optional user/system prompt text.
             output: Optional model response text.
-            step: Optional complete runtime step used for structured dual-write.
+            step: Required runtime step; invocation is rejected without it.
             selected_data: Raw selector-selected value used to construct the record,
                 preserving its structure independently from serialized legacy inputs.
             selected_data_payload_field: Record input/output side for scalar selector values.
-            execution_context: Optional authenticated Galileo scorer context.
+            execution_context: Required authenticated Galileo scorer context; Orbit uses
+                it to fetch LLM credentials.
             config: Optional scorer invocation configuration.
             timeout: Request timeout in seconds.
             headers: Additional request headers.
@@ -535,12 +536,16 @@ async def invoke(
             Parsed scorer invocation response.
 
         Raises:
-            ValueError: If neither input nor output is provided, or the timeout
-                ordering is invalid.
+            ValueError: If step or execution_context are absent, neither input nor
+                output is provided, or the timeout ordering is invalid.
             RuntimeError: If the API response is not a JSON object.
             httpx.HTTPStatusError: If the invoke endpoint returns an error status code.
             httpx.RequestError: If the request fails before a response is received.
         """
+        if step is None:
+            raise ValueError("LLM scorer invocation requires a runtime step.")
+        if execution_context is None:
+            raise ValueError("LLM scorer invocation requires an authenticated execution context.")
         if not (_has_value(input) or _has_value(output)):
             raise ValueError("At least one of input or output must be provided.")
 
@@ -557,7 +562,7 @@ async def invoke(
                 selected_data=selected_data,
                 payload_field=selected_data_payload_field,
             )
-            if step is not None and scorer_version_id is not None
+            if scorer_version_id is not None
             else None
         )
         request_body = ScorerInvokeRequest(
@@ -567,8 +572,8 @@ async def invoke(
             inputs=ScorerInvokeInputs(
                 query="" if input is None else input,
                 response="" if output is None else output,
-                ground_truth=step.ground_truth if step is not None else None,
-                tools=step.tools if step is not None else None,
+                ground_truth=step.ground_truth,
+                tools=step.tools,
             ),
             record=record,
             execution_context=execution_context,

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import Literal
 
 from agent_control_evaluators import EvaluatorConfig
@@ -12,6 +13,17 @@
 LlmPayloadField = Literal["input", "output"]
 
 _NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"})
+LLM_INVOKE_RUNTIME_FLAG_ENV = "GALILEO_FEATURE_FLAG_LLM_INVOKE_RUNTIME"
+
+
+def llm_invoke_runtime_enabled() -> bool:
+    """Return whether the LLM scorer invoke runtime is enabled.
+
+    The LLM evaluator requires Orbit support for ``execution_context`` to fetch
+    LLM credentials. Set ``GALILEO_FEATURE_FLAG_LLM_INVOKE_RUNTIME=enabled`` once
+    the Orbit-side support is confirmed ready.
+    """
+    return os.getenv(LLM_INVOKE_RUNTIME_FLAG_ENV) == "enabled"
 
 
 class ScorerInvokeConfig(BaseModel):

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py

@@ -13,7 +13,7 @@
 from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step
 
 from .client import GalileoExecutionContext, GalileoLLMClient, ScorerInvokeResponse
-from .config import LlmEvaluatorConfig, coerce_number
+from .config import LlmEvaluatorConfig, coerce_number, llm_invoke_runtime_enabled
 
 logger = logging.getLogger(__name__)
 
@@ -117,8 +117,13 @@ class LlmEvaluator(Evaluator[LlmEvaluatorConfig]):
 
     @classmethod
     def is_available(cls) -> bool:
-        """Check whether required runtime dependencies are available."""
-        return LLM_AVAILABLE
+        """Return True only when the LLM invoke runtime feature flag is enabled.
+
+        The LLM evaluator requires Orbit-side support for ``execution_context`` to
+        fetch LLM credentials. It is unavailable until
+        ``GALILEO_FEATURE_FLAG_LLM_INVOKE_RUNTIME=enabled`` is set.
+        """
+        return LLM_AVAILABLE and llm_invoke_runtime_enabled()
 
     def __init__(self, config: LlmEvaluatorConfig) -> None:
         """Initialize the direct LLM-as-judge evaluator.
@@ -267,6 +272,28 @@ async def _evaluate(
         extensions: JSONObject | None = None,
     ) -> EvaluatorResult:
         """Run an LLM scorer evaluation with optional structured runtime context."""
+        if step is None:
+            return EvaluatorResult(
+                matched=False,
+                confidence=0.0,
+                message="LLM scorer requires a runtime step; evaluation skipped.",
+                metadata=self._base_metadata(),
+                error="LLM scorer requires a runtime step; evaluation skipped.",
+            )
+
+        try:
+            execution_context = self._execution_context_from_extensions(extensions)
+        except ValueError as exc:
+            return self._handle_error(exc)
+        if execution_context is None:
+            return EvaluatorResult(
+                matched=False,
+                confidence=0.0,
+                message="LLM scorer requires authenticated execution context; evaluation skipped.",
+                metadata=self._base_metadata(),
+                error="LLM scorer requires authenticated execution context; evaluation skipped.",
+            )
+
         input_text, output_text = self._prepare_payload(data)
         if not (_has_text(input_text) or _has_text(output_text)):
             return EvaluatorResult(
@@ -277,14 +304,11 @@ async def _evaluate(
             )
 
         try:
-            execution_context = self._execution_context_from_extensions(extensions)
             scorer_kwargs = self._scorer_kwargs()
-            if step is not None:
-                scorer_kwargs["step"] = step
-                scorer_kwargs["selected_data"] = data
-                scorer_kwargs["selected_data_payload_field"] = self.config.payload_field
-            if execution_context is not None:
-                scorer_kwargs["execution_context"] = execution_context
+            scorer_kwargs["step"] = step
+            scorer_kwargs["selected_data"] = data
+            scorer_kwargs["selected_data_payload_field"] = self.config.payload_field
+            scorer_kwargs["execution_context"] = execution_context
             response = await self._get_client().invoke(
                 **scorer_kwargs,
                 input=input_text if _has_text(input_text) else None,

evaluators/contrib/galileo/tests/test_llm_coverage_gaps.py

@@ -17,8 +17,29 @@
 LLM_ENV = {
     "GALILEO_API_SECRET_KEY": "test-secret",
     "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
+    "GALILEO_FEATURE_FLAG_LLM_INVOKE_RUNTIME": "enabled",
 }
 
+_EXTENSIONS = {
+    "namespace_key": "org-1",
+    "caller_id": "user-1",
+    "target_type": "log_stream",
+    "target_id": "run-1",
+    "metadata": {},
+}
+
+
+def _make_invoke_kwargs() -> dict[str, object]:
+    """Minimal required kwargs for GalileoLLMClient.invoke()."""
+    from agent_control_models import Step
+
+    from agent_control_evaluator_galileo.llm.client import GalileoExecutionContext
+
+    return {
+        "step": Step(type="llm", name="test", input="prompt"),
+        "execution_context": GalileoExecutionContext(organization_id="org-1"),
+    }
+
 
 def _decode_jwt_payload(token: str) -> dict[str, object]:
     payload_segment = token.split(".")[1]
@@ -289,12 +310,15 @@ async def test_evaluator_handles_non_success_status(monkeypatch):
     """A non-success status from the scorer must surface as an error result."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
+    from agent_control_models import Step
+
     from agent_control_evaluator_galileo.llm import LlmEvaluator, ScorerInvokeResponse
     from agent_control_evaluator_galileo.llm.client import GalileoLLMClient
 
     evaluator = LlmEvaluator.from_dict(
         {"scorer_id": "scorer-123", "threshold": 0.5, "operator": "gte"}
     )
+    step = Step(type="llm", name="test", input="prompt")
 
     with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
         mock_invoke.return_value = ScorerInvokeResponse(
@@ -304,7 +328,7 @@ async def test_evaluator_handles_non_success_status(monkeypatch):
             error_message="upstream timeout",
         )
 
-        result = await evaluator.evaluate("hello")
+        result = await evaluator.evaluate_with_extensions("hello", step, _EXTENSIONS)
 
     assert result.matched is False
     assert result.error is not None
@@ -316,13 +340,16 @@ async def test_evaluator_skips_empty_data(monkeypatch):
     """Evaluator skips invocation when there is no text to score."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
+    from agent_control_models import Step
+
     from agent_control_evaluator_galileo.llm import LlmEvaluator
     from agent_control_evaluator_galileo.llm.client import GalileoLLMClient
 
     evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123", "threshold": 0.5})
+    step = Step(type="llm", name="test", input="prompt")
 
     with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
-        result = await evaluator.evaluate(None)
+        result = await evaluator.evaluate_with_extensions(None, step, _EXTENSIONS)
 
     assert result.matched is False
     assert result.error is None
@@ -335,10 +362,13 @@ async def test_evaluator_returns_error_result_on_http_error(monkeypatch):
     """HTTP errors must surface as a non-matched EvaluatorResult with metadata."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
+    from agent_control_models import Step
+
     from agent_control_evaluator_galileo.llm import LlmEvaluator
     from agent_control_evaluator_galileo.llm.client import GalileoLLMClient
 
     evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123", "threshold": 0.5})
+    step = Step(type="llm", name="test", input="prompt")
 
     fake_request = httpx.Request("POST", "http://luna-invoke:8090/api/v1/scorers/invoke")
     fake_response = httpx.Response(500, text="internal server error", request=fake_request)
@@ -349,7 +379,7 @@ async def test_evaluator_returns_error_result_on_http_error(monkeypatch):
         new_callable=AsyncMock,
         side_effect=httpx.HTTPStatusError("500", request=fake_request, response=fake_response),
     ):
-        result = await evaluator.evaluate("hello")
+        result = await evaluator.evaluate_with_extensions("hello", step, _EXTENSIONS)
 
     assert result.matched is False
     assert result.error is not None
@@ -755,7 +785,7 @@ async def test_invoke_raises_when_response_is_not_a_json_object(monkeypatch):
 
     try:
         with pytest.raises(RuntimeError, match="not a JSON object"):
-            await client.invoke(scorer_id="scorer-123", input="hello")
+            await client.invoke(scorer_id="scorer-123", input="hello", **_make_invoke_kwargs())
     finally:
         await client.close()
 
@@ -785,7 +815,7 @@ async def test_invoke_propagates_http_status_error(monkeypatch):
 
     try:
         with pytest.raises(httpx.HTTPStatusError):
-            await client.invoke(scorer_id="scorer-123", input="hello")
+            await client.invoke(scorer_id="scorer-123", input="hello", **_make_invoke_kwargs())
     finally:
         await client.close()
 
@@ -806,7 +836,7 @@ async def test_invoke_propagates_request_error(monkeypatch):
 
     try:
         with pytest.raises(httpx.RequestError):
-            await client.invoke(scorer_id="scorer-123", input="hello")
+            await client.invoke(scorer_id="scorer-123", input="hello", **_make_invoke_kwargs())
     finally:
         await client.close()
 
@@ -846,6 +876,7 @@ def handler(request: httpx.Request) -> httpx.Response:
             scorer_id="scorer-123",
             input="hello",
             headers={"Galileo-API-Key": "should-be-stripped", "X-Custom": "keep-me"},
+            **_make_invoke_kwargs(),
         )
     finally:
         await client.close()
@@ -873,7 +904,7 @@ def handler(request: httpx.Request) -> httpx.Response:
     client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
 
     try:
-        await client.invoke(scorer_id="scorer-123", input="hello")
+        await client.invoke(scorer_id="scorer-123", input="hello", **_make_invoke_kwargs())
     finally:
         await client.close()
 
@@ -951,7 +982,7 @@ def test_coerce_number_parses_numeric_string(self):
 
 @pytest.mark.asyncio
 async def test_evaluator_with_context_uses_step(monkeypatch):
-    """evaluate_with_context passes the step to _evaluate."""
+    """evaluate_with_extensions passes the step and execution_context to invoke."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
     from agent_control_models import Step
@@ -964,29 +995,33 @@ async def test_evaluator_with_context_uses_step(monkeypatch):
 
     with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
         mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
-        result = await evaluator.evaluate_with_context("selected input", step)
+        result = await evaluator.evaluate_with_extensions("selected input", step, _EXTENSIONS)
 
     assert result.matched is True
     assert mock_invoke.call_args.kwargs["step"] == step
+    assert mock_invoke.call_args.kwargs["execution_context"] is not None
 
 
 @pytest.mark.asyncio
 async def test_evaluator_scorer_label_echoed_in_metadata(monkeypatch):
     """scorer_label echoed from the response is included in result metadata."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
+    from agent_control_models import Step
+
     from agent_control_evaluator_galileo.llm import LlmEvaluator
     from agent_control_evaluator_galileo.llm.client import GalileoLLMClient, ScorerInvokeResponse
 
     evaluator = LlmEvaluator.from_dict(
         {"scorer_id": "scorer-123", "threshold": 0.5, "scorer_label": "my-label"}
     )
+    step = Step(type="llm", name="test", input="prompt")
 
     with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
         mock_invoke.return_value = ScorerInvokeResponse(
             scorer_label="echoed-label", score=0.8, status="success"
         )
-        result = await evaluator.evaluate("hello")
+        result = await evaluator.evaluate_with_extensions("hello", step, _EXTENSIONS)
 
     assert result.metadata.get("scorer_label") == "echoed-label"
 
@@ -996,6 +1031,8 @@ async def test_evaluator_scorer_version_id_in_metadata(monkeypatch):
     """scorer_version_id is included in metadata when configured."""
     for key, value in LLM_ENV.items():
         monkeypatch.setenv(key, value)
+    from agent_control_models import Step
+
     from agent_control_evaluator_galileo.llm import LlmEvaluator
     from agent_control_evaluator_galileo.llm.client import GalileoLLMClient, ScorerInvokeResponse
 
@@ -1006,10 +1043,11 @@ async def test_evaluator_scorer_version_id_in_metadata(monkeypatch):
             "threshold": 0.5,
         }
     )
+    step = Step(type="llm", name="test", input="prompt")
 
     with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke:
         mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success")
-        result = await evaluator.evaluate("hello")
+        result = await evaluator.evaluate_with_extensions("hello", step, _EXTENSIONS)
 
     assert result.metadata.get("requested_scorer_version_id") == "ver-456"
     assert mock_invoke.call_args.kwargs["scorer_version_id"] == "ver-456"

evaluators/contrib/galileo/tests/test_llm_evaluator.py

@@ -11,6 +11,7 @@
 LLM_ENV = {
     "GALILEO_API_SECRET_KEY": "test-secret",
     "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
+    "GALILEO_FEATURE_FLAG_LLM_INVOKE_RUNTIME": "enabled",
 }