Skip to content

Commit 59b2ec5

Browse files
feat(evaluators): put runtime records behind a flag (#288)
## Summary - Gate structured Galileo Luna scorer-invoke data behind `GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME`. - Only the exact value `enabled` sends the serialized Galileo record and runtime execution context; an unset or other value preserves the legacy request and its inputs. ## Scope - User-facing/API changes: None. - Internal changes: Read the feature flag within the Galileo evaluator package and gate record generation and execution-context serialization in its Luna client. Existing scorer identity, authentication, timeout, and response handling remain intact. - Out of scope: Generic Agent Control code, Orbit calls, and authentication-flow changes. ## Risk and Rollout - Risk level: low. - Rollback plan: Unset the flag or set it to a value other than `enabled` to restore legacy request behavior. ## Testing - [x] Added or updated automated tests for missing, disabled, non-exact, and enabled flag behavior. - [x] Ran Galileo package tests (`make test`): 205 passed. - [x] Ran Galileo package lint (`make lint`) and typecheck (`make typecheck`). - [ ] Ran `make check` (individual package test, lint, and typecheck targets were run instead). - [ ] Manually verified behavior. ## Checklist - [ ] Linked issue/spec (if applicable). - [ ] Updated docs/examples for user-facing changes. - [ ] Included any required follow-up tasks.
1 parent bc1e15e commit 59b2ec5

3 files changed

Lines changed: 72 additions & 3 deletions

File tree

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
2020

2121
from ..records import UnsupportedStepTypeError, record_from_step
22-
from .config import ScorerInvokeConfig
22+
from .config import ScorerInvokeConfig, scorer_invoke_runtime_enabled
2323

2424
logger = logging.getLogger(__name__)
2525

@@ -367,6 +367,8 @@ class GalileoLunaClient:
367367
Environment Variables:
368368
GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth.
369369
GALILEO_LUNA_INVOKE_URL: Luna scorer invoke URL or service root (required).
370+
GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME: Set to ``enabled`` to send
371+
structured runtime data.
370372
GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify Luna invoke TLS.
371373
AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback.
372374
GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry.
@@ -414,6 +416,7 @@ def __init__(
414416
)
415417

416418
self.api_secret = resolved_api_secret
419+
self.runtime_enabled = scorer_invoke_runtime_enabled()
417420
self.luna_invoke_url = _normalize_luna_invoke_url(resolved_luna_invoke_url)
418421
self.luna_invoke_ca_file = (
419422
luna_invoke_ca_file
@@ -559,7 +562,7 @@ async def invoke(
559562
selected_data=selected_data,
560563
payload_field=selected_data_payload_field,
561564
)
562-
if step is not None and scorer_version_id is not None
565+
if self.runtime_enabled and step is not None and scorer_version_id is not None
563566
else None
564567
)
565568
request_body = ScorerInvokeRequest(
@@ -573,7 +576,7 @@ async def invoke(
573576
tools=step.tools if step is not None else None,
574577
),
575578
record=record,
576-
execution_context=execution_context,
579+
execution_context=execution_context if self.runtime_enabled else None,
577580
config=invoke_config,
578581
).to_dict()
579582

‎evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import os
56
from typing import Literal
67

78
from agent_control_evaluators import EvaluatorConfig
@@ -12,6 +13,12 @@
1213
LunaPayloadField = Literal["input", "output"]
1314

1415
_NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"})
16+
SCORER_INVOKE_RUNTIME_FLAG_ENV = "GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME"
17+
18+
19+
def scorer_invoke_runtime_enabled() -> bool:
20+
"""Return whether structured runtime data is enabled for Luna requests."""
21+
return os.getenv(SCORER_INVOKE_RUNTIME_FLAG_ENV) == "enabled"
1522

1623

1724
class ScorerInvokeConfig(BaseModel):

‎evaluators/contrib/galileo/tests/test_luna_evaluator.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
LUNA_ENV = {
1818
"GALILEO_API_SECRET_KEY": "test-secret",
1919
"GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
20+
"GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME": "enabled",
21+
}
22+
LEGACY_LUNA_ENV = {
23+
"GALILEO_API_SECRET_KEY": "test-secret",
24+
"GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090",
2025
}
2126
SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4"
2227
SCORER_VERSION_ID = "07fb9c96-9752-4cf5-a253-1a396100e9d2"
@@ -885,6 +890,60 @@ def handler(request: httpx.Request) -> httpx.Response:
885890
"config": {"request_timeout_seconds": 8.0},
886891
}
887892

893+
@pytest.mark.asyncio
894+
@pytest.mark.parametrize("flag_value", [None, "disabled", "Enabled"])
895+
async def test_client_uses_legacy_request_without_runtime_flag(
896+
self, flag_value: str | None
897+
) -> None:
898+
from agent_control_evaluator_galileo.luna import GalileoLunaClient
899+
from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext
900+
901+
captured: dict[str, object] = {}
902+
903+
def handler(request: httpx.Request) -> httpx.Response:
904+
captured["body"] = json.loads(request.content.decode())
905+
return httpx.Response(200, json={"score": 0.8, "status": "success"})
906+
907+
step = Step(
908+
type="llm",
909+
name="answer",
910+
input="step input",
911+
output="step output",
912+
ground_truth={"expected": "answer"},
913+
tools=[{"name": "search", "input_schema": {}}],
914+
)
915+
env = LEGACY_LUNA_ENV.copy()
916+
if flag_value is not None:
917+
env["GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME"] = flag_value
918+
with patch.dict(os.environ, env, clear=True):
919+
client = GalileoLunaClient()
920+
client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
921+
922+
try:
923+
await client.invoke(
924+
scorer_id=SCORER_ID,
925+
scorer_version_id=SCORER_VERSION_ID,
926+
input="selected input",
927+
output="selected output",
928+
step=step,
929+
execution_context=GalileoExecutionContext(organization_id="org-1"),
930+
)
931+
finally:
932+
await client.close()
933+
934+
body = captured["body"]
935+
assert isinstance(body, dict)
936+
assert body["scorer_id"] == SCORER_ID
937+
assert body["scorer_version_id"] == SCORER_VERSION_ID
938+
assert body["inputs"] == {
939+
"query": "selected input",
940+
"response": "selected output",
941+
"ground_truth": {"expected": "answer"},
942+
"tools": [{"name": "search", "input_schema": {}}],
943+
}
944+
assert "record" not in body
945+
assert "execution_context" not in body
946+
888947
@pytest.mark.asyncio
889948
async def test_client_omits_record_when_step_has_no_pinned_version(self) -> None:
890949
from agent_control_evaluator_galileo.luna import GalileoLunaClient

0 commit comments

Comments
 (0)