From df601d2b8e0bc772b58defedb1c1dbc2cde3b0d8 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Fri, 2 Oct 2026 12:15:11 -0700 Subject: [PATCH 1/6] feat(evaluators): Add evaluator for LLM --- evaluators/contrib/README.md | 2 +- evaluators/contrib/galileo/pyproject.toml | 3 +- .../__init__.py | 16 + .../llm/__init__.py | 30 + .../llm/client.py | 613 ++++++++++++++++++ .../llm/config.py | 124 ++++ .../llm/evaluator.py | 387 +++++++++++ sdks/python/ARCHITECTURE.md | 6 +- .../src/agent_control/control_decorators.py | 2 +- .../src/agent_control/evaluators/__init__.py | 25 +- .../tests/test_evaluators_optional_imports.py | 36 +- 11 files changed, 1235 insertions(+), 9 deletions(-) create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py diff --git a/evaluators/contrib/README.md b/evaluators/contrib/README.md index 91beb9b96..4b8520700 100644 --- a/evaluators/contrib/README.md +++ b/evaluators/contrib/README.md @@ -2,7 +2,7 @@ Contributed evaluators and templates for extending Agent Control. -- `galileo/` — Luna evaluator integration +- `galileo/` — Galileo evaluator integrations (Luna, LLM) - `template/` — Starter template for adding new evaluators Full guide: https://docs.agentcontrol.dev/concepts/evaluators/custom-evaluators diff --git a/evaluators/contrib/galileo/pyproject.toml b/evaluators/contrib/galileo/pyproject.toml index 40e247ed7..e9f25b6c9 100644 --- a/evaluators/contrib/galileo/pyproject.toml +++ b/evaluators/contrib/galileo/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "agent-control-evaluator-galileo" version = "8.10.0" -description = "Galileo Luna evaluator for agent-control" +description = "Galileo evaluators (Luna, LLM-as-judge) for agent-control" readme = "README.md" requires-python = ">=3.12" license = { text = "Apache-2.0" } @@ -25,6 +25,7 @@ dev = [ [project.entry-points."agent_control.evaluators"] "galileo.luna" = "agent_control_evaluator_galileo.luna:LunaEvaluator" +"galileo.llm" = "agent_control_evaluator_galileo.llm:LlmEvaluator" [build-system] requires = ["hatchling"] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index e3bea6647..5a804e55a 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -4,6 +4,7 @@ Available evaluators: - galileo.luna: Galileo Luna direct scorer evaluation + - galileo.llm: Galileo LLM-as-judge direct scorer evaluation Installation: pip install agent-control-evaluator-galileo @@ -30,6 +31,13 @@ ScorerInvokeRequest, ScorerInvokeResponse, ) +from agent_control_evaluator_galileo.llm import ( + LLM_AVAILABLE, + GalileoLLMClient, + LlmEvaluator, + LlmEvaluatorConfig, + LlmOperator, +) from agent_control_evaluator_galileo.records import ( GalileoRecord, RecordFactoryError, @@ -41,6 +49,7 @@ ) __all__ = [ + # luna "GalileoLunaClient", "ScorerInvokeRequest", "ScorerInvokeConfig", @@ -50,6 +59,13 @@ "LunaEvaluatorConfig", "LunaOperator", "LUNA_AVAILABLE", + # llm + "GalileoLLMClient", + "LlmEvaluator", + "LlmEvaluatorConfig", + "LlmOperator", + "LLM_AVAILABLE", + # records "GalileoRecord", "RecordFactoryError", "UnsupportedStepTypeError", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py new file mode 100644 index 000000000..518febbe9 --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py @@ -0,0 +1,30 @@ +"""Galileo LLM-as-judge direct scorer evaluator.""" + +from agent_control_evaluator_galileo.llm.client import ( + GalileoLLMClient, + GalileoExecutionContext, + ScorerInvokeInputs, + ScorerInvokeRecord, + ScorerInvokeRequest, + ScorerInvokeResponse, +) +from agent_control_evaluator_galileo.llm.config import ( + LlmEvaluatorConfig, + LlmOperator, + ScorerInvokeConfig, +) +from agent_control_evaluator_galileo.llm.evaluator import LLM_AVAILABLE, LlmEvaluator + +__all__ = [ + "GalileoLLMClient", + "GalileoExecutionContext", + "ScorerInvokeInputs", + "ScorerInvokeConfig", + "ScorerInvokeRecord", + "ScorerInvokeRequest", + "ScorerInvokeResponse", + "LlmEvaluatorConfig", + "LlmOperator", + "LlmEvaluator", + "LLM_AVAILABLE", +] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py new file mode 100644 index 000000000..47b21e6ba --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py @@ -0,0 +1,613 @@ +"""Direct HTTP client for Galileo LLM-as-judge scorer invocation.""" + +from __future__ import annotations + +import logging +import os +import ssl +from asyncio import Lock +from base64 import urlsafe_b64encode +from hashlib import sha256 +from hmac import new as hmac_new +from json import dumps +from time import time +from typing import Literal, cast, get_args +from urllib.parse import urlsplit + +import httpx +from agent_control_models import JSONObject, JSONValue, Step +from pydantic import BaseModel, Field, PrivateAttr, model_validator + +from .config import ScorerInvokeConfig + +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT_SECS = 10.0 +SERVER_TIMEOUT_RATIO = 0.8 +DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 +DEFAULT_LLM_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" +LLM_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" +LLM_INVOKE_CA_FILE_ENV = "GALILEO_LUNA_INVOKE_CA_FILE" +AUTH_UPSTREAM_CA_FILE_ENV = "AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE" + +# Headers that must never be forwarded to the scorer invoke endpoint (checked case-insensitively). +_BLOCKED_REQUEST_HEADERS = frozenset({"galileo-api-key"}) + +# Keep pooled-connection reuse shorter than typical server keepalive/worker +# recycle windows so requests do not pick up sockets the server already closed. +DEFAULT_KEEPALIVE_EXPIRY_SECS = 1.0 +DEFAULT_MAX_CONNECTIONS = 100 +DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 20 +DEFAULT_CLIENT_POOL_SIZE = 1 +LLM_KEEPALIVE_EXPIRY_ENV = "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS" +LLM_MAX_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_CONNECTIONS" +LLM_MAX_KEEPALIVE_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS" +LLM_CLIENT_POOL_SIZE_ENV = "GALILEO_LUNA_CLIENT_POOL_SIZE" + +# These values mirror Orbit's StepType discriminator. Agent Control's generic +# Step remains extensible; only the Galileo transport boundary is constrained. +ScorerInvokeRecordType = Literal[ + "llm", + "retriever", + "tool", + "workflow", + "agent", + "control", + "trace", + "session", +] +SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) + + +def _b64url(data: bytes) -> str: + return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _internal_auth_token( + api_secret: str, + ttl_seconds: int = DEFAULT_INTERNAL_TOKEN_TTL_SECS, +) -> str: + """Create the internal JWT expected by LLM scorer invoke routes.""" + now = int(time()) + header = {"alg": "HS256", "typ": "JWT"} + payload = { + "internal": True, + "scope": "scorers.invoke", + "iat": now, + "exp": now + ttl_seconds, + } + signing_input = ".".join( + [ + _b64url(dumps(header, separators=(",", ":")).encode("utf-8")), + _b64url(dumps(payload, separators=(",", ":")).encode("utf-8")), + ] + ) + signature = hmac_new(api_secret.encode("utf-8"), signing_input.encode("ascii"), sha256).digest() + return f"{signing_input}.{_b64url(signature)}" + + +def _normalize_llm_invoke_url(raw_url: str) -> str: + """Use full invoke URLs as-is and append the default path to bare service roots.""" + url = raw_url.strip().rstrip("/") + parsed = urlsplit(url) + if parsed.path not in ("", "/") or parsed.query or parsed.fragment: + return url + return f"{url}{DEFAULT_LLM_SCORER_INVOKE_PATH}" + + +def _load_float_env(env_name: str, default: float) -> float: + raw = os.getenv(env_name) + if raw is None or raw.strip() == "": + return default + try: + return float(raw) + except ValueError as exc: + raise ValueError(f"{env_name}={raw!r} is not a number.") from exc + + +def _load_int_env(env_name: str, default: int) -> int: + raw = os.getenv(env_name) + if raw is None or raw.strip() == "": + return default + try: + return int(raw) + except ValueError as exc: + raise ValueError(f"{env_name}={raw!r} is not an integer.") from exc + + +def _validate_connection_config( + *, + keepalive_expiry_seconds: float, + max_connections: int, + max_keepalive_connections: int, + client_pool_size: int, +) -> None: + if keepalive_expiry_seconds < 0: + raise ValueError( + f"{LLM_KEEPALIVE_EXPIRY_ENV}={keepalive_expiry_seconds} " + "must be greater than or equal to 0." + ) + if max_connections <= 0: + raise ValueError(f"{LLM_MAX_CONNECTIONS_ENV}={max_connections} must be greater than 0.") + if max_keepalive_connections < 0: + raise ValueError( + f"{LLM_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " + "must be greater than or equal to 0." + ) + if max_keepalive_connections > max_connections: + raise ValueError( + f"{LLM_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " + f"must be less than or equal to {LLM_MAX_CONNECTIONS_ENV}={max_connections}." + ) + if client_pool_size <= 0: + raise ValueError(f"{LLM_CLIENT_POOL_SIZE_ENV}={client_pool_size} must be greater than 0.") + + +def _as_float_or_none(value: JSONValue) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def _has_value(value: JSONValue) -> bool: + if value is None: + return False + if isinstance(value, str): + return value.strip() != "" + if isinstance(value, (list, dict)): + return len(value) > 0 + return True + + +def _effective_scorer_timeout( + config: ScorerInvokeConfig, + *, + http_timeout_seconds: float, +) -> ScorerInvokeConfig: + """Resolve an Orbit execution timeout that expires before the HTTP request. + + The server execution budget defaults to 80% of the caller's HTTP deadline, + leaving time for Orbit to serialize and return the result. Explicit caller + overrides are preserved only when they maintain the same ordering. + + Args: + config: Caller-provided scorer-invoke configuration. + http_timeout_seconds: Agent Control's HTTP request deadline in seconds. + + Returns: + A scorer-invoke configuration with an effective execution timeout. + + Raises: + ValueError: If either deadline is invalid or the explicit server timeout + is not shorter than the HTTP deadline. + """ + if http_timeout_seconds <= 0: + raise ValueError("HTTP timeout must be greater than 0 seconds.") + + server_timeout = config.request_timeout_seconds + if server_timeout is None: + server_timeout = http_timeout_seconds * SERVER_TIMEOUT_RATIO + elif server_timeout >= http_timeout_seconds: + raise ValueError( + "config.request_timeout_seconds must be shorter than the HTTP " + f"timeout ({http_timeout_seconds:g} seconds)." + ) + + return config.model_copy(update={"request_timeout_seconds": server_timeout}) + + +class ScorerInvokeInputs(BaseModel): + """Input values sent to the LLM scorer invoke endpoint.""" + + query: JSONValue = "" + response: JSONValue = "" + ground_truth: JSONValue = None + tools: list[JSONObject] | None = None + + +class ScorerInvokeRecord(BaseModel): + """Caller-controlled subset of Orbit's partial runtime-record contract. + + Identity, ownership, persistence, and execution IDs are intentionally not + represented here. Orbit hydrates those fields from trusted server context. + """ + + type: ScorerInvokeRecordType + name: str | None = None + input: JSONValue = None + output: JSONValue = None + context: JSONObject | None = None + tools: list[JSONObject] | None = None + dataset_output: JSONValue = None + + +class GalileoExecutionContext(BaseModel): + """Authenticated identity fields required by Galileo scorer invocation.""" + + organization_id: str = Field(min_length=1) + user_id: str = Field(min_length=1) + project_id: str = Field(min_length=1) + run_id: str = Field(min_length=1) + + +class ScorerInvokeRequest(BaseModel): + """Request payload for LLM scorer invocation. + + Attributes: + scorer_id: Required scorer identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + inputs: Selected scorer input values. + record: Optional Orbit-compatible structured runtime record. + execution_context: Optional authenticated execution context. + config: Scorer-specific configuration, always emitted. + """ + + scorer_id: str = Field(min_length=1) + scorer_version_id: str | None = Field(default=None, min_length=1) + scorer_label: str | None = Field(default=None, min_length=1) + inputs: ScorerInvokeInputs + record: ScorerInvokeRecord | None = None + execution_context: GalileoExecutionContext | None = None + config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) + + @model_validator(mode="after") + def ensure_required_values(self) -> ScorerInvokeRequest: + if not (_has_value(self.inputs.query) or _has_value(self.inputs.response)): + raise ValueError("Either inputs.query or inputs.response must be set.") + return self + + def to_dict(self) -> JSONObject: + """Convert to the LLM scorer invoke request shape.""" + return self.model_dump(mode="json", exclude_none=True) + + +def _orbit_record_from_step( + step: Step | None, + *, + selected_input: JSONValue, + selected_output: JSONValue, +) -> ScorerInvokeRecord | None: + """Translate a generic Agent Control step into Orbit's record contract. + + Selector-selected values remain the primary scorer input. When a selector + supplies one side, that value is written to both the legacy and structured + representations so Orbit's conflict validation cannot observe two meanings. + The complete step supplies the unselected side and additional record context. + + Unknown Agent Control step types intentionally fall back to the legacy + ``inputs`` contract. This keeps the open-source Step model extensible without + sending an invalid discriminator to Orbit. + + Args: + step: Complete Agent Control step, when contextual evaluation is used. + selected_input: Selector-selected value sent as ``inputs.query``. + selected_output: Selector-selected value sent as ``inputs.response``. + + Returns: + An Orbit-compatible record, or ``None`` for absent/unsupported steps. + """ + if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: + return None + + # The membership check above narrows the runtime value to Orbit's known + # discriminator set, but static type checkers cannot infer that relationship. + record_type = cast(ScorerInvokeRecordType, step.type) + return ScorerInvokeRecord( + type=record_type, + name=step.name, + input=selected_input if selected_input is not None else step.input, + output=selected_output if selected_output is not None else step.output, + context=step.context, + tools=step.tools, + dataset_output=step.ground_truth, + ) + + +class ScorerInvokeResponse(BaseModel): + """Response from LLM scorer invocation. + + Attributes: + scorer_label: Echoed scorer label, when returned. + score: Raw scorer value. + status: Invocation status. + execution_time: Execution time in seconds, when returned. + error_message: Error detail for non-success statuses. + """ + + scorer_label: str | None = None + score: JSONValue + status: str = "unknown" + execution_time: float | None = None + error_message: str | None = None + _raw_response: JSONObject = PrivateAttr(default_factory=dict) + + @property + def raw_response(self) -> JSONObject: + return self._raw_response + + @classmethod + def from_dict(cls, data: JSONObject) -> ScorerInvokeResponse: + """Create a response model from the LLM scorer invoke JSON object.""" + response = cls.model_validate( + data | {"execution_time": _as_float_or_none(data.get("execution_time"))} + ) + response._raw_response = data + return response + + +class GalileoLLMClient: + """Thin HTTP client for Galileo LLM-as-judge scorer invocation. + + Environment Variables: + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. + GALILEO_LUNA_INVOKE_URL: LLM scorer invoke URL or service root (required). + GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify invoke TLS. + AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. + GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry. + GALILEO_LUNA_MAX_CONNECTIONS: Maximum outbound HTTP connections. + GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS: Maximum idle pooled HTTP connections. + GALILEO_LUNA_CLIENT_POOL_SIZE: Number of outbound HTTP clients to rotate across. + """ + + def __init__( + self, + api_secret: str | None = None, + llm_invoke_url: str | None = None, + llm_invoke_ca_file: str | None = None, + ) -> None: + """Initialize the Galileo LLM client. + + Args: + api_secret: Internal JWT signing secret. If not provided, reads from + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. + llm_invoke_url: LLM scorer invoke URL or service root. If not provided, + reads from GALILEO_LUNA_INVOKE_URL. + llm_invoke_ca_file: Optional CA bundle used to verify invoke TLS. If not + provided, reads from GALILEO_LUNA_INVOKE_CA_FILE, then + AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE. + + Raises: + ValueError: If the API secret, invoke URL, CA bundle, or connection + tuning configuration is invalid. + """ + resolved_api_secret = ( + api_secret or os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + ) + if not resolved_api_secret: + raise ValueError( + "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for LLM " + "scorer invocation. Set one as an environment variable or pass it " + "to the constructor." + ) + + resolved_llm_invoke_url = llm_invoke_url or os.getenv(LLM_INVOKE_URL_ENV) + if resolved_llm_invoke_url is None or resolved_llm_invoke_url.strip() == "": + raise ValueError( + "GALILEO_LUNA_INVOKE_URL is required for LLM scorer invocation. " + "Set it as an environment variable or pass it to the constructor." + ) + + self.api_secret = resolved_api_secret + self.llm_invoke_url = _normalize_llm_invoke_url(resolved_llm_invoke_url) + self.llm_invoke_ca_file = ( + llm_invoke_ca_file + or os.getenv(LLM_INVOKE_CA_FILE_ENV) + or os.getenv(AUTH_UPSTREAM_CA_FILE_ENV) + or "" + ).strip() or None + self._ssl_context = self._load_ssl_context(self.llm_invoke_ca_file) + self.keepalive_expiry_seconds = _load_float_env( + LLM_KEEPALIVE_EXPIRY_ENV, DEFAULT_KEEPALIVE_EXPIRY_SECS + ) + self.max_connections = _load_int_env(LLM_MAX_CONNECTIONS_ENV, DEFAULT_MAX_CONNECTIONS) + self.max_keepalive_connections = _load_int_env( + LLM_MAX_KEEPALIVE_CONNECTIONS_ENV, DEFAULT_MAX_KEEPALIVE_CONNECTIONS + ) + self.client_pool_size = _load_int_env(LLM_CLIENT_POOL_SIZE_ENV, DEFAULT_CLIENT_POOL_SIZE) + _validate_connection_config( + keepalive_expiry_seconds=self.keepalive_expiry_seconds, + max_connections=self.max_connections, + max_keepalive_connections=self.max_keepalive_connections, + client_pool_size=self.client_pool_size, + ) + self._client: httpx.AsyncClient | None = None + self._clients: list[httpx.AsyncClient] = [] + self._next_client_index = 0 + self._client_lock = Lock() + + @staticmethod + def _load_ssl_context(ca_file: str | None) -> ssl.SSLContext | None: + """Build a TLS verification context from a CA bundle path, if configured.""" + if ca_file is None: + return None + try: + return ssl.create_default_context(cafile=ca_file) + except (OSError, ssl.SSLError) as exc: + raise ValueError(f"Failed to load CA bundle from {ca_file!r}: {exc}") from exc + + def _create_client(self) -> httpx.AsyncClient: + """Create an HTTP client with the configured TLS and connection limits.""" + verify: ssl.SSLContext | bool = self._ssl_context if self._ssl_context is not None else True + return httpx.AsyncClient( + headers={"Content-Type": "application/json"}, + timeout=httpx.Timeout(DEFAULT_TIMEOUT_SECS), + limits=httpx.Limits( + max_connections=self.max_connections, + max_keepalive_connections=self.max_keepalive_connections, + keepalive_expiry=self.keepalive_expiry_seconds, + ), + verify=verify, + ) + + def _select_pooled_client(self) -> httpx.AsyncClient: + """Select the next pooled client while holding the client state lock.""" + client = self._clients[self._next_client_index % len(self._clients)] + self._next_client_index = (self._next_client_index + 1) % len(self._clients) + return client + + async def _get_client(self) -> httpx.AsyncClient: + """Get or create the next HTTP client.""" + async with self._client_lock: + self._clients = [client for client in self._clients if not client.is_closed] + + if self.client_pool_size == 1: + if self._client is not None and not self._client.is_closed: + return self._client + self._client = self._clients[0] if self._clients else self._create_client() + self._clients = [self._client] + return self._client + + self._client = None + while len(self._clients) < self.client_pool_size: + self._clients.append(self._create_client()) + + return self._select_pooled_client() + + def _endpoint_and_auth_header(self) -> tuple[str, str]: + token = _internal_auth_token(self.api_secret) + return self.llm_invoke_url, f"Bearer {token}" + + async def invoke( + self, + *, + scorer_id: str, + scorer_version_id: str | None = None, + scorer_label: str | None = None, + input: JSONValue = None, + output: JSONValue = None, + step: Step | None = None, + execution_context: GalileoExecutionContext | None = None, + config: ScorerInvokeConfig | JSONObject | None = None, + timeout: float = DEFAULT_TIMEOUT_SECS, + headers: dict[str, str] | None = None, + ) -> ScorerInvokeResponse: + """Invoke a Galileo LLM-as-judge scorer. + + Args: + scorer_id: Required scorer identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + input: Optional user/system prompt text. + output: Optional model response text. + step: Optional complete runtime step used for structured dual-write. + execution_context: Optional authenticated Galileo scorer context. + config: Optional Orbit-supported scorer invocation configuration. + timeout: Request timeout in seconds. + headers: Additional request headers. + + Returns: + Parsed scorer invocation response. + + Raises: + ValueError: If neither input nor output is provided, config contains + a field Orbit does not support, or the timeout ordering is invalid. + RuntimeError: If the API response is not a JSON object. + httpx.HTTPStatusError: If the invoke endpoint returns an error status code. + httpx.RequestError: If the request fails before a response is received. + """ + if not (_has_value(input) or _has_value(output)): + raise ValueError("At least one of input or output must be provided.") + + # Accept dictionaries for source compatibility with the original client, + # but validate them against Orbit's authoritative allowlist locally. + invoke_config = ( + ScorerInvokeConfig.model_validate(config) + if config is not None + else ScorerInvokeConfig() + ) + invoke_config = _effective_scorer_timeout( + invoke_config, + http_timeout_seconds=timeout, + ) + request_body = ScorerInvokeRequest( + scorer_id=scorer_id, + scorer_version_id=scorer_version_id, + scorer_label=scorer_label, + inputs=ScorerInvokeInputs( + query="" if input is None else input, + response="" if output is None else output, + ground_truth=step.ground_truth if step is not None else None, + tools=step.tools if step is not None else None, + ), + record=_orbit_record_from_step( + step, + selected_input=input, + selected_output=output, + ), + execution_context=execution_context, + config=invoke_config, + ).to_dict() + + endpoint, auth_header = self._endpoint_and_auth_header() + request_headers = { + k: v for k, v in (headers or {}).items() if k.lower() not in _BLOCKED_REQUEST_HEADERS + } + request_headers["Authorization"] = auth_header + + logger.debug("[GalileoLLMClient] POST %s", endpoint) + logger.debug("[GalileoLLMClient] Request body: %s", request_body) + + try: + client = await self._get_client() + response = await client.post( + endpoint, + json=request_body, + headers=request_headers, + timeout=timeout, + ) + response.raise_for_status() + response_data = response.json() + if not isinstance(response_data, dict): + raise RuntimeError("Invalid response payload: not a JSON object") + + parsed = ScorerInvokeResponse.from_dict(response_data) + logger.debug("[GalileoLLMClient] Response: %s", parsed.raw_response) + return parsed + except httpx.HTTPStatusError as exc: + logger.error( + "[GalileoLLMClient] API error: %s - %s", + exc.response.status_code, + exc.response.text, + ) + raise + except httpx.RequestError as exc: + logger.error("[GalileoLLMClient] Request failed: %s", exc) + raise + + async def close(self) -> None: + """Close HTTP clients and release resources.""" + async with self._client_lock: + clients: list[httpx.AsyncClient] = [] + seen_client_ids: set[int] = set() + if self._client is not None: + clients.append(self._client) + seen_client_ids.add(id(self._client)) + self._client = None + for client in self._clients: + if id(client) not in seen_client_ids: + clients.append(client) + seen_client_ids.add(id(client)) + self._clients = [] + self._next_client_index = 0 + + for client in clients: + if not client.is_closed: + await client.aclose() + + async def __aenter__(self) -> GalileoLLMClient: + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: + """Async context manager exit.""" + await self.close() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py new file mode 100644 index 000000000..1b207f3af --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py @@ -0,0 +1,124 @@ +"""Configuration model for direct Galileo LLM-as-judge scorer evaluation.""" + +from __future__ import annotations + +from typing import Literal + +from agent_control_evaluators import EvaluatorConfig +from agent_control_models import JSONValue +from pydantic import BaseModel, ConfigDict, Field, model_validator + +LlmOperator = Literal["gt", "gte", "lt", "lte", "eq", "ne", "contains", "any"] +LlmPayloadField = Literal["input", "output"] + +_NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) + + +class ScorerInvokeConfig(BaseModel): + """Orbit-supported overrides for a synchronous scorer invocation. + + Orbit owns the Galileo scorer-invoke wire contract. Keeping this model + strict makes an unsupported option fail locally instead of producing a + less actionable HTTP 422 response from Runners. + + Attributes: + threshold: Legacy threshold accepted by Orbit. Agent Control still + applies its evaluator threshold locally. + score_threshold: Legacy score threshold accepted by Orbit. + request_timeout_seconds: Optional upper bound for scorer execution in + Orbit. The Agent Control HTTP and evaluator deadlines must remain + longer than this value. + """ + + model_config = ConfigDict(extra="forbid") + + threshold: float | None = None + score_threshold: float | None = None + request_timeout_seconds: float | None = Field(default=None, gt=0) + + +def coerce_number(value: JSONValue) -> float | None: + """Return a numeric value for JSON scalars that can be compared numerically.""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +class LlmEvaluatorConfig(EvaluatorConfig): + """Configuration for direct LLM-as-judge scorer evaluation. + + Attributes: + scorer_id: Required scorer identifier for LLM scorer invocation. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + threshold: Local threshold used by the evaluator for comparison. + operator: Local comparison operator. Numeric operators use threshold as a number. + scorer_config: Optional Orbit-supported scorer invocation config sent + as ``config``. + payload_field: Explicit scorer input side for scalar selected data. + timeout_ms: Request timeout in milliseconds. + """ + + scorer_id: str = Field( + min_length=1, + description="Required scorer identifier for LLM scorer invocation.", + ) + scorer_version_id: str | None = Field( + default=None, + min_length=1, + description=( + "Deprecated optional compatibility identifier. Orbit currently invokes " + "the scorer's current default version." + ), + ) + scorer_label: str | None = Field( + default=None, + min_length=1, + description="Optional display/metadata label.", + ) + threshold: JSONValue = Field( + default=0.5, + description="Local threshold used to decide whether the control matches.", + ) + operator: LlmOperator = Field( + default="gte", + description="Local comparison operator applied to the raw LLM scorer score.", + ) + scorer_config: ScorerInvokeConfig | None = Field( + default=None, + alias="config", + serialization_alias="config", + description=( + "Optional Orbit-supported configuration sent to the LLM scorer invoke endpoint." + ), + ) + payload_field: LlmPayloadField = Field( + default="input", + description=( + "Which scorer input side to use when selector output is a scalar value. " + "Structured selected data with input/output keys overrides this setting." + ), + ) + timeout_ms: int = Field( + default=10000, + ge=1000, + le=60000, + description="Request timeout in milliseconds (1-60 seconds)", + ) + + @model_validator(mode="after") + def validate_threshold(self) -> LlmEvaluatorConfig: + """Validate threshold compatibility with the configured operator.""" + if self.operator in _NUMERIC_OPERATORS and coerce_number(self.threshold) is None: + raise ValueError(f"operator '{self.operator}' requires a numeric threshold") + if self.operator != "any" and self.threshold is None: + raise ValueError("threshold is required unless operator is 'any'") + return self diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py new file mode 100644 index 000000000..7e7c8720e --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py @@ -0,0 +1,387 @@ +"""Direct Galileo LLM-as-judge evaluator implementation.""" + +from __future__ import annotations + +import json +import logging +import os +from importlib.metadata import PackageNotFoundError, version +from typing import Any + +import httpx +from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator +from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step + +from .client import GalileoExecutionContext, GalileoLLMClient, ScorerInvokeResponse +from .config import LlmEvaluatorConfig, coerce_number + +logger = logging.getLogger(__name__) + + +def _resolve_package_version() -> str: + """Return the installed package version, or a dev fallback during local imports.""" + try: + return version("agent-control-evaluator-galileo") + except PackageNotFoundError: + return "0.0.0.dev" + + +_PACKAGE_VERSION = _resolve_package_version() +LLM_AVAILABLE = True +_HTTP_ERROR_BODY_LIMIT = 500 + + +def _coerce_payload_text(value: Any) -> str | None: + """Coerce selected data into scorer text without losing structured values.""" + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + except TypeError: + return str(value) + + +def _has_text(value: str | None) -> bool: + return value is not None and value.strip() != "" + + +def _extract_dict_text(data: dict[str, Any], key: str) -> str | None: + if key not in data: + return None + return _coerce_payload_text(data.get(key)) + + +def _contains(score: JSONValue, threshold: JSONValue) -> bool: + if threshold is None: + return False + if isinstance(score, str): + return str(threshold) in score + if isinstance(score, list): + return threshold in score + if isinstance(score, dict): + return threshold in score.values() + return False + + +def _confidence_from_score(score: JSONValue) -> float: + if isinstance(score, bool): + return 1.0 if score else 0.0 + number = coerce_number(score) + if number is not None and 0.0 <= number <= 1.0: + return number + return 1.0 + + +def _truncated_http_response_body(body: str) -> tuple[str, bool]: + if len(body) <= _HTTP_ERROR_BODY_LIMIT: + return body, False + return body[:_HTTP_ERROR_BODY_LIMIT], True + + +def _http_status_error_metadata(error: httpx.HTTPStatusError) -> dict[str, Any]: + metadata: dict[str, Any] = {} + + request = error.request + metadata["http_method"] = request.method + metadata["http_endpoint_path"] = request.url.path + + response = error.response + metadata["http_status_code"] = response.status_code + metadata["http_response_content_type"] = response.headers.get("content-type") + + body = response.text + if body: + metadata["http_response_body"], metadata["http_response_body_truncated"] = ( + _truncated_http_response_body(body) + ) + + return {key: value for key, value in metadata.items() if value is not None} + + +@register_evaluator +class LlmEvaluator(Evaluator[LlmEvaluatorConfig]): + """Galileo LLM-as-judge evaluator using the direct scorer invocation API.""" + + metadata = EvaluatorMetadata( + name="galileo.llm", + version=_PACKAGE_VERSION, + description="Galileo LLM-as-judge direct scorer evaluation", + requires_api_key=True, + timeout_ms=10000, + ) + config_model = LlmEvaluatorConfig + + @classmethod + def is_available(cls) -> bool: + """Check whether required runtime dependencies are available.""" + return LLM_AVAILABLE + + def __init__(self, config: LlmEvaluatorConfig) -> None: + """Initialize the direct LLM-as-judge evaluator. + + Args: + config: Validated LlmEvaluatorConfig instance. + + Raises: + ValueError: If neither GALILEO_API_SECRET_KEY nor GALILEO_API_SECRET is set. + """ + has_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + if not has_secret: + raise ValueError( + "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for LLM " + "scorer invocation. Set one as an environment variable before using " + "galileo.llm." + ) + + super().__init__(config) + self._client = GalileoLLMClient() + + def _get_client(self) -> GalileoLLMClient: + """Get the Galileo LLM client.""" + return self._client + + def _prepare_payload(self, data: Any) -> tuple[str | None, str | None]: + """Prepare scorer input/output fields from selected data.""" + if isinstance(data, dict): + input_text = _extract_dict_text(data, "input") + output_text = _extract_dict_text(data, "output") + if _has_text(input_text) or _has_text(output_text): + return input_text, output_text + + text = _coerce_payload_text(data) + if self.config.payload_field == "output": + return None, text + return text, None + + def _score_matches(self, score: JSONValue) -> bool: + """Apply the configured local threshold comparison to a raw LLM scorer score.""" + operator = self.config.operator + threshold = self.config.threshold + + if operator == "any": + return bool(score) + if operator == "eq": + return score == threshold + if operator == "ne": + return score != threshold + if operator == "contains": + return _contains(score, threshold) + + score_number = coerce_number(score) + threshold_number = coerce_number(threshold) + if score_number is None: + raise ValueError(f"LLM scorer score {score!r} is not numeric") + if threshold_number is None: + raise ValueError(f"LLM scorer threshold {threshold!r} is not numeric") + + if operator == "gt": + return score_number > threshold_number + if operator == "gte": + return score_number >= threshold_number + if operator == "lt": + return score_number < threshold_number + if operator == "lte": + return score_number <= threshold_number + + raise ValueError(f"Unsupported LLM scorer operator: {operator}") + + async def evaluate(self, data: Any) -> EvaluatorResult: + """Evaluate selected data with Galileo LLM-as-judge direct scorer invocation. + + Args: + data: The data selected from the runtime step. + + Returns: + EvaluatorResult with local threshold decision and scorer metadata. + """ + return await self._evaluate(data, step=None) + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + """Evaluate selected data while dual-writing the complete runtime step. + + Args: + data: Data selected by the configured control selector. + step: Complete runtime step for structured scorer context. + + Returns: + EvaluatorResult with local threshold decision and scorer metadata. + """ + return await self._evaluate(data, step=step) + + async def evaluate_with_extensions( + self, + data: Any, + step: Step, + extensions: JSONObject | None, + ) -> EvaluatorResult: + """Evaluate using opaque authenticated metadata supplied by Agent Control.""" + return await self._evaluate( + data, + step=step, + extensions=extensions, + ) + + @staticmethod + def _execution_context_from_extensions( + extensions: JSONObject | None, + ) -> GalileoExecutionContext | None: + """Translate trusted opaque auth metadata into Galileo's request context.""" + if extensions is None: + return None + + metadata = extensions.get("metadata") + metadata_obj = metadata if isinstance(metadata, dict) else {} + organization_id = extensions.get("namespace_key") + # Orbit's runtime-auth contract defines caller_id as the authenticated + # user ID for this flow. Keep that contract mapping inside Galileo. + user_id = extensions.get("caller_id") + project_id = metadata_obj.get("project_id") + target_type = extensions.get("target_type") + target_id = extensions.get("target_id") + run_id = target_id if target_type == "log_stream" else None + + missing = [ + field + for field, value in ( + ("organization_id", organization_id), + ("user_id", user_id), + ("project_id", project_id), + ("run_id", run_id), + ) + if not isinstance(value, str) or not value + ] + if missing: + raise ValueError( + "Authenticated execution metadata is missing required fields: " + + ", ".join(missing) + ) + + assert isinstance(organization_id, str) + assert isinstance(user_id, str) + assert isinstance(project_id, str) + assert isinstance(run_id, str) + return GalileoExecutionContext( + organization_id=organization_id, + user_id=user_id, + project_id=project_id, + run_id=run_id, + ) + + async def _evaluate( + self, + data: Any, + *, + step: Step | None, + extensions: JSONObject | None = None, + ) -> EvaluatorResult: + """Run an LLM scorer evaluation with optional structured runtime context.""" + input_text, output_text = self._prepare_payload(data) + if not (_has_text(input_text) or _has_text(output_text)): + return EvaluatorResult( + matched=False, + confidence=1.0, + message="No data to score with LLM scorer", + metadata=self._base_metadata(), + ) + + try: + execution_context = self._execution_context_from_extensions(extensions) + scorer_kwargs = self._scorer_kwargs() + if step is not None: + scorer_kwargs["step"] = step + if execution_context is not None: + scorer_kwargs["execution_context"] = execution_context + response = await self._get_client().invoke( + **scorer_kwargs, + input=input_text if _has_text(input_text) else None, + output=output_text if _has_text(output_text) else None, + config=self.config.scorer_config, + timeout=self.get_timeout_seconds(), + ) + + if response.status.lower() != "success": + message = response.error_message or f"LLM scorer status: {response.status}" + raise RuntimeError(message) + + matched = self._score_matches(response.score) + metadata = self._metadata(response) + operator = self.config.operator + threshold = self.config.threshold + state = "triggered" if matched else "not triggered" + return EvaluatorResult( + matched=matched, + confidence=_confidence_from_score(response.score), + message=( + f"LLM scorer score {response.score!r} {operator} threshold " + f"{threshold!r}: control {state}." + ), + metadata=metadata, + ) + except Exception as exc: + logger.error("LLM scorer evaluation error: %s", exc, exc_info=True) + return self._handle_error(exc) + + def _base_metadata(self) -> dict[str, Any]: + """Build result metadata without implying a requested version executed.""" + metadata: dict[str, Any] = {"scorer_id": self.config.scorer_id} + if self.config.scorer_version_id is not None: + metadata["requested_scorer_version_id"] = self.config.scorer_version_id + if self.config.scorer_label is not None: + metadata["scorer_label"] = self.config.scorer_label + return metadata + + def _scorer_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"scorer_id": self.config.scorer_id} + if self.config.scorer_version_id is not None: + kwargs["scorer_version_id"] = self.config.scorer_version_id + if self.config.scorer_label is not None: + kwargs["scorer_label"] = self.config.scorer_label + return kwargs + + def _metadata( + self, + response: ScorerInvokeResponse, + ) -> dict[str, Any]: + metadata: dict[str, Any] = self._base_metadata() + echoed_label = response.scorer_label or self.config.scorer_label + if echoed_label is not None: + metadata["scorer_label"] = echoed_label + metadata.update( + { + "score": response.score, + "threshold": self.config.threshold, + "operator": self.config.operator, + "status": response.status, + "execution_time_seconds": response.execution_time, + "error_message": response.error_message, + } + ) + return metadata + + def _handle_error( + self, + error: Exception, + ) -> EvaluatorResult: + error_detail = str(error) + metadata: dict[str, Any] = { + **self._base_metadata(), + "error_type": type(error).__name__, + } + if isinstance(error, httpx.HTTPStatusError): + metadata.update(_http_status_error_metadata(error)) + + return EvaluatorResult( + matched=False, + confidence=0.0, + message=f"LLM scorer evaluation error: {error_detail}", + metadata=metadata, + error=error_detail, + ) + + async def aclose(self) -> None: + """Close the underlying Galileo LLM client.""" + await self._client.close() diff --git a/sdks/python/ARCHITECTURE.md b/sdks/python/ARCHITECTURE.md index 4a48c09a0..a58fb4c7d 100644 --- a/sdks/python/ARCHITECTURE.md +++ b/sdks/python/ARCHITECTURE.md @@ -20,7 +20,7 @@ sdks/python/src/agent_control/ ├── tracing.py # Distributed tracing support ├── py.typed # PEP 561 type marker └── evaluators/ # Evaluator base classes and discovery system - ├── __init__.py # Evaluator discovery, registration, and Luna integration + ├── __init__.py # Evaluator discovery, registration, and Luna/LLM integration └── base.py # Base Evaluator and EvaluatorMetadata classes ``` @@ -213,11 +213,11 @@ async def chat(message: str) -> str: **Key Components**: - Base evaluator classes (`Evaluator`, `EvaluatorMetadata`) - Evaluator discovery via entry points -- Third-party evaluator integration (e.g., Luna, Guardrails AI) +- Third-party evaluator integration (e.g., Luna, LLM, Guardrails AI) - Registration functions for custom evaluators **Structure**: -- `__init__.py` - Evaluator discovery (`discover_evaluators()`, `list_evaluators()`), registration (`register_evaluator()`), and optional Luna integration +- `__init__.py` - Evaluator discovery (`discover_evaluators()`, `list_evaluators()`), registration (`register_evaluator()`), and optional Luna/LLM integration - `base.py` - Base `Evaluator` and `EvaluatorMetadata` classes (re-exported from `agent_control_models`) **Usage**: diff --git a/sdks/python/src/agent_control/control_decorators.py b/sdks/python/src/agent_control/control_decorators.py index fee78333d..eb565d6e3 100644 --- a/sdks/python/src/agent_control/control_decorators.py +++ b/sdks/python/src/agent_control/control_decorators.py @@ -23,7 +23,7 @@ async def chat(message: str) -> str: # Server-side controls define: # - stage: "pre" or "post" # - selector.path: "input" or "output" - # - evaluator: regex, list, Luna evaluator, etc. + # - evaluator: regex, list, Luna evaluator, LLM evaluator, etc. # - action: deny, steer, or observe """ diff --git a/sdks/python/src/agent_control/evaluators/__init__.py b/sdks/python/src/agent_control/evaluators/__init__.py index 737147177..d239821e0 100644 --- a/sdks/python/src/agent_control/evaluators/__init__.py +++ b/sdks/python/src/agent_control/evaluators/__init__.py @@ -1,7 +1,7 @@ """Evaluator system for agent_control. This module provides an evaluator architecture for extending agent_control -with external evaluation systems like Galileo Luna, Guardrails AI, etc. +with external evaluation systems like Galileo Luna, Galileo LLM, Guardrails AI, etc. Evaluator Discovery: Call `discover_evaluators()` at startup to load evaluators. This loads: @@ -14,6 +14,7 @@ When installed with galileo extras, the Galileo evaluator types are available: ```python from agent_control.evaluators import LunaEvaluator, LunaEvaluatorConfig # if galileo installed + from agent_control.evaluators import LlmEvaluator, LlmEvaluatorConfig # if galileo installed ``` """ @@ -62,3 +63,25 @@ ) except ImportError: pass + +# Optionally export LLM evaluator types when available +try: + from agent_control_evaluator_galileo.llm import ( # type: ignore[import-not-found] # noqa: F401 + LLM_AVAILABLE, + GalileoLLMClient, + LlmEvaluator, + LlmEvaluatorConfig, + LlmOperator, + ) + + __all__.extend( + [ + "GalileoLLMClient", + "LlmEvaluator", + "LlmEvaluatorConfig", + "LlmOperator", + "LLM_AVAILABLE", + ] + ) +except ImportError: + pass diff --git a/sdks/python/tests/test_evaluators_optional_imports.py b/sdks/python/tests/test_evaluators_optional_imports.py index b4560fc9f..370e7a6d1 100644 --- a/sdks/python/tests/test_evaluators_optional_imports.py +++ b/sdks/python/tests/test_evaluators_optional_imports.py @@ -28,6 +28,7 @@ def _module_available(name: str) -> bool: _GALILEO_INSTALLED = _module_available("agent_control_evaluator_galileo.luna") +_GALILEO_LLM_INSTALLED = _module_available("agent_control_evaluator_galileo.llm") def _reload_evaluators_with_blocked(prefix: str) -> object: @@ -70,21 +71,35 @@ def test_module_loads_when_galileo_luna_is_unavailable(): # Core names are always present. assert "Evaluator" in reloaded.__all__ - # Luna1 names are NOT present because the import failed. + # Luna names are NOT present because the import failed. assert "LunaEvaluator" not in reloaded.__all__ assert "GalileoLunaClient" not in reloaded.__all__ +def test_module_loads_when_galileo_llm_is_unavailable(): + """Hiding ``agent_control_evaluator_galileo.llm`` exercises its except branch.""" + reloaded = _reload_evaluators_with_blocked("agent_control_evaluator_galileo.llm") + + # Core names are always present. + assert "Evaluator" in reloaded.__all__ + # LLM names are NOT present because the import failed. + assert "LlmEvaluator" not in reloaded.__all__ + assert "GalileoLLMClient" not in reloaded.__all__ + + def test_module_loads_when_galileo_package_is_unavailable(): """Hiding the whole package exercises the ImportError fallback.""" reloaded = _reload_evaluators_with_blocked("agent_control_evaluator_galileo") assert "Evaluator" in reloaded.__all__ - # The optional luna names are absent. + # The optional luna and llm names are absent. for absent in ( "LunaEvaluator", "GalileoLunaClient", "LUNA_AVAILABLE", + "LlmEvaluator", + "GalileoLLMClient", + "LLM_AVAILABLE", ): assert absent not in reloaded.__all__ @@ -108,3 +123,20 @@ def test_module_loads_galileo_optional_imports_when_available(): finally: if saved is not None: sys.modules["agent_control.evaluators"] = saved + + +@pytest.mark.skipif( + not _GALILEO_LLM_INSTALLED, + reason="agent-control-evaluator-galileo extras not installed in this environment", +) +def test_module_loads_galileo_llm_optional_imports_when_available(): + """Sanity check: with galileo installed, the LLM optional names ARE exposed.""" + saved = sys.modules.pop("agent_control.evaluators", None) + try: + import agent_control.evaluators as reloaded + + reloaded = importlib.reload(reloaded) + assert "LlmEvaluator" in reloaded.__all__ + finally: + if saved is not None: + sys.modules["agent_control.evaluators"] = saved From 0d2596ff4d783207d22f36c0a4ce6a6857655cca Mon Sep 17 00:00:00 2001 From: Wrisa Date: Fri, 2 Oct 2026 12:17:56 -0700 Subject: [PATCH 2/6] fix(evaluators): fix import sort order in galileo evaluator __init__ files --- .../agent_control_evaluator_galileo/__init__.py | 14 +++++++------- .../llm/__init__.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index 5a804e55a..9ceddd7c2 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -20,6 +20,13 @@ except PackageNotFoundError: __version__ = "0.0.0.dev" +from agent_control_evaluator_galileo.llm import ( + LLM_AVAILABLE, + GalileoLLMClient, + LlmEvaluator, + LlmEvaluatorConfig, + LlmOperator, +) from agent_control_evaluator_galileo.luna import ( LUNA_AVAILABLE, GalileoLunaClient, @@ -31,13 +38,6 @@ ScorerInvokeRequest, ScorerInvokeResponse, ) -from agent_control_evaluator_galileo.llm import ( - LLM_AVAILABLE, - GalileoLLMClient, - LlmEvaluator, - LlmEvaluatorConfig, - LlmOperator, -) from agent_control_evaluator_galileo.records import ( GalileoRecord, RecordFactoryError, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py index 518febbe9..7d78b1621 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py @@ -1,8 +1,8 @@ """Galileo LLM-as-judge direct scorer evaluator.""" from agent_control_evaluator_galileo.llm.client import ( - GalileoLLMClient, GalileoExecutionContext, + GalileoLLMClient, ScorerInvokeInputs, ScorerInvokeRecord, ScorerInvokeRequest, From 167a8f77f55b5130dde2602fef6eff5a39f85fc1 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Sat, 3 Oct 2026 13:28:13 -0700 Subject: [PATCH 3/6] added execution context --- .../llm/client.py | 13 +- .../llm/evaluator.py | 38 ++-- .../galileo/tests/test_llm_evaluator.py | 191 ++++++++++++++++++ 3 files changed, 212 insertions(+), 30 deletions(-) create mode 100644 evaluators/contrib/galileo/tests/test_llm_evaluator.py diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py index 47b21e6ba..396b0216c 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py @@ -229,12 +229,12 @@ class ScorerInvokeRecord(BaseModel): class GalileoExecutionContext(BaseModel): - """Authenticated identity fields required by Galileo scorer invocation.""" + """Authenticated organization and optional identity fields for Galileo scorer invocation.""" organization_id: str = Field(min_length=1) - user_id: str = Field(min_length=1) - project_id: str = Field(min_length=1) - run_id: str = Field(min_length=1) + user_id: str | None = Field(default=None, min_length=1) + project_id: str | None = Field(default=None, min_length=1) + run_id: str | None = Field(default=None, min_length=1) class ScorerInvokeRequest(BaseModel): @@ -267,7 +267,10 @@ def ensure_required_values(self) -> ScorerInvokeRequest: def to_dict(self) -> JSONObject: """Convert to the LLM scorer invoke request shape.""" - return self.model_dump(mode="json", exclude_none=True) + request = self.model_dump(mode="json", exclude_none=True) + if self.execution_context is not None: + request["execution_context"] = self.execution_context.model_dump(mode="json") + return request def _orbit_record_from_step( diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py index 7e7c8720e..6c12ddd18 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py @@ -236,38 +236,26 @@ def _execution_context_from_extensions( metadata = extensions.get("metadata") metadata_obj = metadata if isinstance(metadata, dict) else {} organization_id = extensions.get("namespace_key") - # Orbit's runtime-auth contract defines caller_id as the authenticated - # user ID for this flow. Keep that contract mapping inside Galileo. + # The runtime envelope's caller_id is supplied by the authenticated + # principal and is the caller identity Galileo associates with this run. user_id = extensions.get("caller_id") project_id = metadata_obj.get("project_id") target_type = extensions.get("target_type") target_id = extensions.get("target_id") - run_id = target_id if target_type == "log_stream" else None - - missing = [ - field - for field, value in ( - ("organization_id", organization_id), - ("user_id", user_id), - ("project_id", project_id), - ("run_id", run_id), - ) - if not isinstance(value, str) or not value - ] - if missing: - raise ValueError( - "Authenticated execution metadata is missing required fields: " - + ", ".join(missing) - ) + run_id = ( + target_id + if target_type in ("log_stream", "agent_stream") + and isinstance(target_id, str) + and target_id + else None + ) + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("Authenticated execution metadata is missing organization_id") - assert isinstance(organization_id, str) - assert isinstance(user_id, str) - assert isinstance(project_id, str) - assert isinstance(run_id, str) return GalileoExecutionContext( organization_id=organization_id, - user_id=user_id, - project_id=project_id, + user_id=user_id if isinstance(user_id, str) and user_id else None, + project_id=project_id if isinstance(project_id, str) and project_id else None, run_id=run_id, ) diff --git a/evaluators/contrib/galileo/tests/test_llm_evaluator.py b/evaluators/contrib/galileo/tests/test_llm_evaluator.py new file mode 100644 index 000000000..814d9de32 --- /dev/null +++ b/evaluators/contrib/galileo/tests/test_llm_evaluator.py @@ -0,0 +1,191 @@ +"""Tests for the direct Galileo LLM-as-judge evaluator and client.""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, patch + +import pytest +from agent_control_models import Step + +LLM_ENV = { + "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", +} + + +class TestGalileoLLMClient: + """Tests for ``GalileoLLMClient`` and ``ScorerInvokeRequest``.""" + + def test_execution_context_serializes_missing_optional_claims_as_null(self) -> None: + from agent_control_evaluator_galileo.llm.client import ( + GalileoExecutionContext, + ScorerInvokeInputs, + ScorerInvokeRequest, + ) + + request = ScorerInvokeRequest( + scorer_id="scorer-123", + inputs=ScorerInvokeInputs(query="hello"), + execution_context=GalileoExecutionContext(organization_id="org-1"), + ) + + assert request.to_dict()["execution_context"] == { + "organization_id": "org-1", + "user_id": None, + "project_id": None, + "run_id": None, + } + + +class TestLlmEvaluator: + """Tests for ``LlmEvaluator`` execution context handling.""" + + @patch.dict(os.environ, LLM_ENV) + @pytest.mark.asyncio + @pytest.mark.parametrize("target_type", ["log_stream", "agent_stream"]) + async def test_evaluator_consumes_verified_opaque_extensions(self, target_type: str) -> None: + from agent_control_evaluator_galileo.llm import LlmEvaluator + from agent_control_evaluator_galileo.llm.client import ( + GalileoExecutionContext, + GalileoLLMClient, + ScorerInvokeResponse, + ) + + evaluator = LlmEvaluator.from_dict( + {"scorer_id": "scorer-123", "threshold": 0.5, "operator": "gte"} + ) + extensions = { + "namespace_key": "org-1", + "caller_id": "verified-user-2", + "target_type": target_type, + "target_id": "run-4", + "metadata": {"project_id": "project-3"}, + } + + with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate_with_extensions( + "selected input", Step(type="llm", name="answer", input="prompt"), extensions + ) + + assert result.matched is True + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=Step(type="llm", name="answer", input="prompt"), + execution_context=GalileoExecutionContext( + organization_id="org-1", + user_id="verified-user-2", + project_id="project-3", + run_id="run-4", + ), + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LLM_ENV) + @pytest.mark.asyncio + async def test_evaluator_allows_missing_caller_id(self) -> None: + from agent_control_evaluator_galileo.llm import LlmEvaluator + from agent_control_evaluator_galileo.llm.client import ( + GalileoExecutionContext, + GalileoLLMClient, + ScorerInvokeResponse, + ) + + evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"}) + extensions = { + "namespace_key": "org-1", + "target_type": "log_stream", + "target_id": "run-4", + "metadata": {}, + } + + with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate_with_extensions( + "selected input", Step(type="llm", name="answer", input="prompt"), extensions + ) + + assert result.error is None + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=Step(type="llm", name="answer", input="prompt"), + execution_context=GalileoExecutionContext( + organization_id="org-1", + user_id=None, + project_id=None, + run_id="run-4", + ), + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LLM_ENV) + @pytest.mark.asyncio + async def test_evaluator_omits_run_id_for_unsupported_target_type(self) -> None: + from agent_control_evaluator_galileo.llm import LlmEvaluator + from agent_control_evaluator_galileo.llm.client import ( + GalileoExecutionContext, + GalileoLLMClient, + ScorerInvokeResponse, + ) + + evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"}) + extensions = { + "namespace_key": "org-1", + "caller_id": "verified-user-2", + "target_type": "environment", + "target_id": "prod", + "metadata": {"project_id": "project-3"}, + } + + with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate_with_extensions( + "selected input", Step(type="llm", name="answer", input="prompt"), extensions + ) + + assert result.error is None + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=Step(type="llm", name="answer", input="prompt"), + execution_context=GalileoExecutionContext( + organization_id="org-1", + user_id="verified-user-2", + project_id="project-3", + run_id=None, + ), + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + + @patch.dict(os.environ, LLM_ENV) + @pytest.mark.asyncio + async def test_evaluator_rejects_missing_organization_id(self) -> None: + from agent_control_evaluator_galileo.llm import LlmEvaluator + from agent_control_evaluator_galileo.llm.client import GalileoLLMClient + + evaluator = LlmEvaluator.from_dict({"scorer_id": "scorer-123"}) + extensions = { + "target_type": "log_stream", + "target_id": "run-4", + "metadata": {"project_id": "project-3"}, + } + + with patch.object(GalileoLLMClient, "invoke", new_callable=AsyncMock) as mock_invoke: + result = await evaluator.evaluate_with_extensions( + "selected input", + Step(type="llm", name="answer", input="prompt"), + extensions, + ) + + assert result.error is not None + assert "organization_id" in result.error + mock_invoke.assert_not_called() From 7e7e1cabe8de0f59584c1fa111294c50a2c52095 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 5 Oct 2026 11:03:56 -0700 Subject: [PATCH 4/6] Add evaluator LLM thin --- .../_shared/__init__.py | 1 + .../_shared/client.py | 624 +++++++++++++++++ .../_shared/config.py | 124 ++++ .../_shared/evaluator.py | 356 ++++++++++ .../llm/client.py | 638 +----------------- .../llm/config.py | 136 +--- .../llm/evaluator.py | 369 +--------- .../luna/client.py | 637 +---------------- .../luna/config.py | 136 +--- .../luna/evaluator.py | 370 +--------- 10 files changed, 1222 insertions(+), 2169 deletions(-) create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/__init__.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/client.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/config.py create mode 100644 evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/__init__.py new file mode 100644 index 000000000..6db786c92 --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/__init__.py @@ -0,0 +1 @@ +"""Shared internals for Galileo evaluators.""" diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/client.py new file mode 100644 index 000000000..9b2d6b587 --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/client.py @@ -0,0 +1,624 @@ +"""Shared HTTP client for Galileo scorer invocation.""" + +from __future__ import annotations + +import logging +import os +import ssl +from asyncio import Lock +from base64 import urlsafe_b64encode +from hashlib import sha256 +from hmac import new as hmac_new +from json import dumps +from time import time +from typing import Literal, cast, get_args +from urllib.parse import urlsplit + +import httpx +from agent_control_models import JSONObject, JSONValue, Step +from pydantic import BaseModel, Field, PrivateAttr, model_validator + +from .config import ScorerInvokeConfig + +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT_SECS = 10.0 +SERVER_TIMEOUT_RATIO = 0.8 +DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 +DEFAULT_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" +SCORER_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" +SCORER_INVOKE_CA_FILE_ENV = "GALILEO_LUNA_INVOKE_CA_FILE" +AUTH_UPSTREAM_CA_FILE_ENV = "AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE" + +# Headers that must never be forwarded to the scorer invoke endpoint (checked case-insensitively). +_BLOCKED_REQUEST_HEADERS = frozenset({"galileo-api-key"}) + +# Keep pooled-connection reuse shorter than typical server keepalive/worker +# recycle windows so requests do not pick up sockets the server already closed. +DEFAULT_KEEPALIVE_EXPIRY_SECS = 1.0 +DEFAULT_MAX_CONNECTIONS = 100 +DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 20 +DEFAULT_CLIENT_POOL_SIZE = 1 +KEEPALIVE_EXPIRY_ENV = "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS" +MAX_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_CONNECTIONS" +MAX_KEEPALIVE_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS" +CLIENT_POOL_SIZE_ENV = "GALILEO_LUNA_CLIENT_POOL_SIZE" + +# These values mirror Orbit's StepType discriminator. Agent Control's generic +# Step remains extensible; only the Galileo transport boundary is constrained. +ScorerInvokeRecordType = Literal[ + "llm", + "retriever", + "tool", + "workflow", + "agent", + "control", + "trace", + "session", +] +SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) + + +def _b64url(data: bytes) -> str: + return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _internal_auth_token( + api_secret: str, + ttl_seconds: int = DEFAULT_INTERNAL_TOKEN_TTL_SECS, +) -> str: + """Create the internal JWT expected by scorer invoke routes.""" + now = int(time()) + header = {"alg": "HS256", "typ": "JWT"} + payload = { + "internal": True, + "scope": "scorers.invoke", + "iat": now, + "exp": now + ttl_seconds, + } + signing_input = ".".join( + [ + _b64url(dumps(header, separators=(",", ":")).encode("utf-8")), + _b64url(dumps(payload, separators=(",", ":")).encode("utf-8")), + ] + ) + signature = hmac_new(api_secret.encode("utf-8"), signing_input.encode("ascii"), sha256).digest() + return f"{signing_input}.{_b64url(signature)}" + + +def _normalize_scorer_invoke_url(raw_url: str) -> str: + """Use full invoke URLs as-is and append the default path to bare service roots.""" + url = raw_url.strip().rstrip("/") + parsed = urlsplit(url) + if parsed.path not in ("", "/") or parsed.query or parsed.fragment: + return url + return f"{url}{DEFAULT_SCORER_INVOKE_PATH}" + + +def _load_float_env(env_name: str, default: float) -> float: + raw = os.getenv(env_name) + if raw is None or raw.strip() == "": + return default + try: + return float(raw) + except ValueError as exc: + raise ValueError(f"{env_name}={raw!r} is not a number.") from exc + + +def _load_int_env(env_name: str, default: int) -> int: + raw = os.getenv(env_name) + if raw is None or raw.strip() == "": + return default + try: + return int(raw) + except ValueError as exc: + raise ValueError(f"{env_name}={raw!r} is not an integer.") from exc + + +def _validate_connection_config( + *, + keepalive_expiry_seconds: float, + max_connections: int, + max_keepalive_connections: int, + client_pool_size: int, +) -> None: + if keepalive_expiry_seconds < 0: + raise ValueError( + f"{KEEPALIVE_EXPIRY_ENV}={keepalive_expiry_seconds} " + "must be greater than or equal to 0." + ) + if max_connections <= 0: + raise ValueError(f"{MAX_CONNECTIONS_ENV}={max_connections} must be greater than 0.") + if max_keepalive_connections < 0: + raise ValueError( + f"{MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " + "must be greater than or equal to 0." + ) + if max_keepalive_connections > max_connections: + raise ValueError( + f"{MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " + f"must be less than or equal to {MAX_CONNECTIONS_ENV}={max_connections}." + ) + if client_pool_size <= 0: + raise ValueError(f"{CLIENT_POOL_SIZE_ENV}={client_pool_size} must be greater than 0.") + + +def _as_float_or_none(value: JSONValue) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def _has_value(value: JSONValue) -> bool: + if value is None: + return False + if isinstance(value, str): + return value.strip() != "" + if isinstance(value, (list, dict)): + return len(value) > 0 + return True + + +def _effective_scorer_timeout( + config: ScorerInvokeConfig, + *, + http_timeout_seconds: float, +) -> ScorerInvokeConfig: + """Resolve an Orbit execution timeout that expires before the HTTP request. + + The server execution budget defaults to 80% of the caller's HTTP deadline, + leaving time for Orbit to serialize and return the result. Explicit caller + overrides are preserved only when they maintain the same ordering. + + Args: + config: Caller-provided scorer-invoke configuration. + http_timeout_seconds: Agent Control's HTTP request deadline in seconds. + + Returns: + A scorer-invoke configuration with an effective execution timeout. + + Raises: + ValueError: If either deadline is invalid or the explicit server timeout + is not shorter than the HTTP deadline. + """ + if http_timeout_seconds <= 0: + raise ValueError("HTTP timeout must be greater than 0 seconds.") + + server_timeout = config.request_timeout_seconds + if server_timeout is None: + server_timeout = http_timeout_seconds * SERVER_TIMEOUT_RATIO + elif server_timeout >= http_timeout_seconds: + raise ValueError( + "config.request_timeout_seconds must be shorter than the HTTP " + f"timeout ({http_timeout_seconds:g} seconds)." + ) + + return config.model_copy(update={"request_timeout_seconds": server_timeout}) + + +class ScorerInvokeInputs(BaseModel): + """Input values sent to the scorer invoke endpoint.""" + + query: JSONValue = "" + response: JSONValue = "" + ground_truth: JSONValue = None + tools: list[JSONObject] | None = None + + +class ScorerInvokeRecord(BaseModel): + """Caller-controlled subset of Orbit's partial runtime-record contract. + + Identity, ownership, persistence, and execution IDs are intentionally not + represented here. Orbit hydrates those fields from trusted server context. + """ + + type: ScorerInvokeRecordType + name: str | None = None + input: JSONValue = None + output: JSONValue = None + context: JSONObject | None = None + tools: list[JSONObject] | None = None + dataset_output: JSONValue = None + + +class GalileoExecutionContext(BaseModel): + """Authenticated organization and optional identity fields for Galileo scorer invocation.""" + + organization_id: str = Field(min_length=1) + user_id: str | None = Field(default=None, min_length=1) + project_id: str | None = Field(default=None, min_length=1) + run_id: str | None = Field(default=None, min_length=1) + + +class ScorerInvokeRequest(BaseModel): + """Request payload for scorer invocation. + + Attributes: + scorer_id: Required scorer identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + inputs: Selected scorer input values. + record: Optional Orbit-compatible structured runtime record. + execution_context: Optional authenticated execution context. + config: Scorer-specific configuration, always emitted. + """ + + scorer_id: str = Field(min_length=1) + scorer_version_id: str | None = Field(default=None, min_length=1) + scorer_label: str | None = Field(default=None, min_length=1) + inputs: ScorerInvokeInputs + record: ScorerInvokeRecord | None = None + execution_context: GalileoExecutionContext | None = None + config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) + + @model_validator(mode="after") + def ensure_required_values(self) -> ScorerInvokeRequest: + if not (_has_value(self.inputs.query) or _has_value(self.inputs.response)): + raise ValueError("Either inputs.query or inputs.response must be set.") + return self + + def to_dict(self) -> JSONObject: + """Convert to the scorer invoke request shape.""" + request = self.model_dump(mode="json", exclude_none=True) + if self.execution_context is not None: + request["execution_context"] = self.execution_context.model_dump(mode="json") + return request + + +def _orbit_record_from_step( + step: Step | None, + *, + selected_input: JSONValue, + selected_output: JSONValue, +) -> ScorerInvokeRecord | None: + """Translate a generic Agent Control step into Orbit's record contract. + + Selector-selected values remain the primary scorer input. When a selector + supplies one side, that value is written to both the legacy and structured + representations so Orbit's conflict validation cannot observe two meanings. + The complete step supplies the unselected side and additional record context. + + Unknown Agent Control step types intentionally fall back to the legacy + ``inputs`` contract. This keeps the open-source Step model extensible without + sending an invalid discriminator to Orbit. + + Args: + step: Complete Agent Control step, when contextual evaluation is used. + selected_input: Selector-selected value sent as ``inputs.query``. + selected_output: Selector-selected value sent as ``inputs.response``. + + Returns: + An Orbit-compatible record, or ``None`` for absent/unsupported steps. + """ + if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: + return None + + # The membership check above narrows the runtime value to Orbit's known + # discriminator set, but static type checkers cannot infer that relationship. + record_type = cast(ScorerInvokeRecordType, step.type) + return ScorerInvokeRecord( + type=record_type, + name=step.name, + input=selected_input if selected_input is not None else step.input, + output=selected_output if selected_output is not None else step.output, + context=step.context, + tools=step.tools, + dataset_output=step.ground_truth, + ) + + +class ScorerInvokeResponse(BaseModel): + """Response from scorer invocation. + + Attributes: + scorer_label: Echoed scorer label, when returned. + score: Raw scorer value. + status: Invocation status. + execution_time: Execution time in seconds, when returned. + error_message: Error detail for non-success statuses. + """ + + scorer_label: str | None = None + score: JSONValue + status: str = "unknown" + execution_time: float | None = None + error_message: str | None = None + _raw_response: JSONObject = PrivateAttr(default_factory=dict) + + @property + def raw_response(self) -> JSONObject: + return self._raw_response + + @classmethod + def from_dict(cls, data: JSONObject) -> ScorerInvokeResponse: + """Create a response model from the scorer invoke JSON object.""" + response = cls.model_validate( + data | {"execution_time": _as_float_or_none(data.get("execution_time"))} + ) + response._raw_response = data + return response + + +class GalileoScorerClient: + """Thin HTTP client for Galileo scorer invocation. + + Environment Variables: + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. + GALILEO_LUNA_INVOKE_URL: Scorer invoke URL or service root (required). + GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify invoke TLS. + AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. + GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry. + GALILEO_LUNA_MAX_CONNECTIONS: Maximum outbound HTTP connections. + GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS: Maximum idle pooled HTTP connections. + GALILEO_LUNA_CLIENT_POOL_SIZE: Number of outbound HTTP clients to rotate across. + """ + + def __init__( + self, + api_secret: str | None = None, + invoke_url: str | None = None, + invoke_ca_file: str | None = None, + # Backwards-compatible keyword aliases used by existing callers. + luna_invoke_url: str | None = None, + luna_invoke_ca_file: str | None = None, + ) -> None: + """Initialize the Galileo scorer client. + + Args: + api_secret: Internal JWT signing secret. If not provided, reads from + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. + invoke_url: Scorer invoke URL or service root. If not provided, + reads from GALILEO_LUNA_INVOKE_URL. + invoke_ca_file: Optional CA bundle used to verify invoke TLS. If not + provided, reads from GALILEO_LUNA_INVOKE_CA_FILE, then + AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE. + + Raises: + ValueError: If the API secret, invoke URL, CA bundle, or connection + tuning configuration is invalid. + """ + resolved_api_secret = ( + api_secret or os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + ) + if not resolved_api_secret: + raise ValueError( + "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for scorer " + "invocation. Set one as an environment variable or pass it " + "to the constructor." + ) + + resolved_invoke_url = invoke_url or luna_invoke_url or os.getenv(SCORER_INVOKE_URL_ENV) + if resolved_invoke_url is None or resolved_invoke_url.strip() == "": + raise ValueError( + "GALILEO_LUNA_INVOKE_URL is required for scorer invocation. " + "Set it as an environment variable or pass it to the constructor." + ) + + self.api_secret = resolved_api_secret + self.invoke_url = _normalize_scorer_invoke_url(resolved_invoke_url) + # Backwards-compatible aliases used by tests and external code. + self.luna_invoke_url = self.invoke_url + self.invoke_ca_file = ( + invoke_ca_file + or luna_invoke_ca_file + or os.getenv(SCORER_INVOKE_CA_FILE_ENV) + or os.getenv(AUTH_UPSTREAM_CA_FILE_ENV) + or "" + ).strip() or None + # Backwards-compatible alias used by tests and external code. + self.luna_invoke_ca_file = self.invoke_ca_file + self._ssl_context = self._load_ssl_context(self.invoke_ca_file) + self.keepalive_expiry_seconds = _load_float_env( + KEEPALIVE_EXPIRY_ENV, DEFAULT_KEEPALIVE_EXPIRY_SECS + ) + self.max_connections = _load_int_env(MAX_CONNECTIONS_ENV, DEFAULT_MAX_CONNECTIONS) + self.max_keepalive_connections = _load_int_env( + MAX_KEEPALIVE_CONNECTIONS_ENV, DEFAULT_MAX_KEEPALIVE_CONNECTIONS + ) + self.client_pool_size = _load_int_env(CLIENT_POOL_SIZE_ENV, DEFAULT_CLIENT_POOL_SIZE) + _validate_connection_config( + keepalive_expiry_seconds=self.keepalive_expiry_seconds, + max_connections=self.max_connections, + max_keepalive_connections=self.max_keepalive_connections, + client_pool_size=self.client_pool_size, + ) + self._client: httpx.AsyncClient | None = None + self._clients: list[httpx.AsyncClient] = [] + self._next_client_index = 0 + self._client_lock = Lock() + + @staticmethod + def _load_ssl_context(ca_file: str | None) -> ssl.SSLContext | None: + """Build a TLS verification context from a CA bundle path, if configured.""" + if ca_file is None: + return None + try: + return ssl.create_default_context(cafile=ca_file) + except (OSError, ssl.SSLError) as exc: + raise ValueError(f"Failed to load CA bundle from {ca_file!r}: {exc}") from exc + + def _create_client(self) -> httpx.AsyncClient: + """Create an HTTP client with the configured TLS and connection limits.""" + verify: ssl.SSLContext | bool = self._ssl_context if self._ssl_context is not None else True + return httpx.AsyncClient( + headers={"Content-Type": "application/json"}, + timeout=httpx.Timeout(DEFAULT_TIMEOUT_SECS), + limits=httpx.Limits( + max_connections=self.max_connections, + max_keepalive_connections=self.max_keepalive_connections, + keepalive_expiry=self.keepalive_expiry_seconds, + ), + verify=verify, + ) + + def _select_pooled_client(self) -> httpx.AsyncClient: + """Select the next pooled client while holding the client state lock.""" + client = self._clients[self._next_client_index % len(self._clients)] + self._next_client_index = (self._next_client_index + 1) % len(self._clients) + return client + + async def _get_client(self) -> httpx.AsyncClient: + """Get or create the next HTTP client.""" + async with self._client_lock: + self._clients = [client for client in self._clients if not client.is_closed] + + if self.client_pool_size == 1: + if self._client is not None and not self._client.is_closed: + return self._client + self._client = self._clients[0] if self._clients else self._create_client() + self._clients = [self._client] + return self._client + + self._client = None + while len(self._clients) < self.client_pool_size: + self._clients.append(self._create_client()) + + return self._select_pooled_client() + + def _endpoint_and_auth_header(self) -> tuple[str, str]: + token = _internal_auth_token(self.api_secret) + return self.invoke_url, f"Bearer {token}" + + async def invoke( + self, + *, + scorer_id: str, + scorer_version_id: str | None = None, + scorer_label: str | None = None, + input: JSONValue = None, + output: JSONValue = None, + step: Step | None = None, + execution_context: GalileoExecutionContext | None = None, + config: ScorerInvokeConfig | JSONObject | None = None, + timeout: float = DEFAULT_TIMEOUT_SECS, + headers: dict[str, str] | None = None, + ) -> ScorerInvokeResponse: + """Invoke a Galileo scorer. + + Args: + scorer_id: Required scorer identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + input: Optional user/system prompt text. + output: Optional model response text. + step: Optional complete runtime step used for structured dual-write. + execution_context: Optional authenticated Galileo scorer context. + config: Optional Orbit-supported scorer invocation configuration. + timeout: Request timeout in seconds. + headers: Additional request headers. + + Returns: + Parsed scorer invocation response. + + Raises: + ValueError: If neither input nor output is provided, config contains + a field Orbit does not support, or the timeout ordering is invalid. + RuntimeError: If the API response is not a JSON object. + httpx.HTTPStatusError: If the invoke endpoint returns an error status code. + httpx.RequestError: If the request fails before a response is received. + """ + if not (_has_value(input) or _has_value(output)): + raise ValueError("At least one of input or output must be provided.") + + # Accept dictionaries for source compatibility with the original client, + # but validate them against Orbit's authoritative allowlist locally. + invoke_config = ( + ScorerInvokeConfig.model_validate(config) + if config is not None + else ScorerInvokeConfig() + ) + invoke_config = _effective_scorer_timeout( + invoke_config, + http_timeout_seconds=timeout, + ) + request_body = ScorerInvokeRequest( + scorer_id=scorer_id, + scorer_version_id=scorer_version_id, + scorer_label=scorer_label, + inputs=ScorerInvokeInputs( + query="" if input is None else input, + response="" if output is None else output, + ground_truth=step.ground_truth if step is not None else None, + tools=step.tools if step is not None else None, + ), + record=_orbit_record_from_step( + step, + selected_input=input, + selected_output=output, + ), + execution_context=execution_context, + config=invoke_config, + ).to_dict() + + endpoint, auth_header = self._endpoint_and_auth_header() + request_headers = { + k: v for k, v in (headers or {}).items() if k.lower() not in _BLOCKED_REQUEST_HEADERS + } + request_headers["Authorization"] = auth_header + + logger.debug("[GalileoScorerClient] POST %s", endpoint) + logger.debug("[GalileoScorerClient] Request body: %s", request_body) + + try: + client = await self._get_client() + response = await client.post( + endpoint, + json=request_body, + headers=request_headers, + timeout=timeout, + ) + response.raise_for_status() + response_data = response.json() + if not isinstance(response_data, dict): + raise RuntimeError("Invalid response payload: not a JSON object") + + parsed = ScorerInvokeResponse.from_dict(response_data) + logger.debug("[GalileoScorerClient] Response: %s", parsed.raw_response) + return parsed + except httpx.HTTPStatusError as exc: + logger.error( + "[GalileoScorerClient] API error: %s - %s", + exc.response.status_code, + exc.response.text, + ) + raise + except httpx.RequestError as exc: + logger.error("[GalileoScorerClient] Request failed: %s", exc) + raise + + async def close(self) -> None: + """Close HTTP clients and release resources.""" + async with self._client_lock: + clients: list[httpx.AsyncClient] = [] + seen_client_ids: set[int] = set() + if self._client is not None: + clients.append(self._client) + seen_client_ids.add(id(self._client)) + self._client = None + for client in self._clients: + if id(client) not in seen_client_ids: + clients.append(client) + seen_client_ids.add(id(client)) + self._clients = [] + self._next_client_index = 0 + + for client in clients: + if not client.is_closed: + await client.aclose() + + async def __aenter__(self) -> GalileoScorerClient: + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: + """Async context manager exit.""" + await self.close() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/config.py new file mode 100644 index 000000000..9d0d08f74 --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/config.py @@ -0,0 +1,124 @@ +"""Shared configuration models for Galileo scorer evaluators.""" + +from __future__ import annotations + +from typing import Literal + +from agent_control_evaluators import EvaluatorConfig +from agent_control_models import JSONValue +from pydantic import BaseModel, ConfigDict, Field, model_validator + +ScorerOperator = Literal["gt", "gte", "lt", "lte", "eq", "ne", "contains", "any"] +ScorerPayloadField = Literal["input", "output"] + +_NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) + + +class ScorerInvokeConfig(BaseModel): + """Orbit-supported overrides for a synchronous scorer invocation. + + Orbit owns the Galileo scorer-invoke wire contract. Keeping this model + strict makes an unsupported option fail locally instead of producing a + less actionable HTTP 422 response from Runners. + + Attributes: + threshold: Legacy threshold accepted by Orbit. Agent Control still + applies its evaluator threshold locally. + score_threshold: Legacy score threshold accepted by Orbit. + request_timeout_seconds: Optional upper bound for scorer execution in + Orbit. The Agent Control HTTP and evaluator deadlines must remain + longer than this value. + """ + + model_config = ConfigDict(extra="forbid") + + threshold: float | None = None + score_threshold: float | None = None + request_timeout_seconds: float | None = Field(default=None, gt=0) + + +def coerce_number(value: JSONValue) -> float | None: + """Return a numeric value for JSON scalars that can be compared numerically.""" + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +class BaseScorerEvaluatorConfig(EvaluatorConfig): + """Shared configuration for Galileo direct scorer evaluators. + + Attributes: + scorer_id: Required scorer identifier for scorer invocation. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. + scorer_label: Optional display/metadata label. + threshold: Local threshold used by the evaluator for comparison. + operator: Local comparison operator. Numeric operators use threshold as a number. + scorer_config: Optional Orbit-supported scorer invocation config sent + as ``config``. + payload_field: Explicit scorer input side for scalar selected data. + timeout_ms: Request timeout in milliseconds. + """ + + scorer_id: str = Field( + min_length=1, + description="Required scorer identifier for scorer invocation.", + ) + scorer_version_id: str | None = Field( + default=None, + min_length=1, + description=( + "Deprecated optional compatibility identifier. Orbit currently invokes " + "the scorer's current default version." + ), + ) + scorer_label: str | None = Field( + default=None, + min_length=1, + description="Optional display/metadata label.", + ) + threshold: JSONValue = Field( + default=0.5, + description="Local threshold used to decide whether the control matches.", + ) + operator: ScorerOperator = Field( + default="gte", + description="Local comparison operator applied to the raw scorer score.", + ) + scorer_config: ScorerInvokeConfig | None = Field( + default=None, + alias="config", + serialization_alias="config", + description=( + "Optional Orbit-supported configuration sent to the scorer invoke endpoint." + ), + ) + payload_field: ScorerPayloadField = Field( + default="input", + description=( + "Which scorer input side to use when selector output is a scalar value. " + "Structured selected data with input/output keys overrides this setting." + ), + ) + timeout_ms: int = Field( + default=10000, + ge=1000, + le=60000, + description="Request timeout in milliseconds (1-60 seconds)", + ) + + @model_validator(mode="after") + def validate_threshold(self) -> BaseScorerEvaluatorConfig: + """Validate threshold compatibility with the configured operator.""" + if self.operator in _NUMERIC_OPERATORS and coerce_number(self.threshold) is None: + raise ValueError(f"operator '{self.operator}' requires a numeric threshold") + if self.operator != "any" and self.threshold is None: + raise ValueError("threshold is required unless operator is 'any'") + return self diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py new file mode 100644 index 000000000..a1f27ebea --- /dev/null +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py @@ -0,0 +1,356 @@ +"""Shared base evaluator for Galileo direct scorer evaluators.""" + +from __future__ import annotations + +import json +import logging +import os +from importlib.metadata import PackageNotFoundError, version +from typing import Any + +import httpx +from agent_control_evaluators import Evaluator +from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step + +from .client import GalileoExecutionContext, GalileoScorerClient, ScorerInvokeResponse +from .config import BaseScorerEvaluatorConfig, coerce_number + +logger = logging.getLogger(__name__) + +_HTTP_ERROR_BODY_LIMIT = 500 + + +def _resolve_package_version() -> str: + """Return the installed package version, or a dev fallback during local imports.""" + try: + return version("agent-control-evaluator-galileo") + except PackageNotFoundError: + return "0.0.0.dev" + + +_PACKAGE_VERSION = _resolve_package_version() + + +def _coerce_payload_text(value: Any) -> str | None: + """Coerce selected data into scorer text without losing structured values.""" + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + except TypeError: + return str(value) + + +def _has_text(value: str | None) -> bool: + return value is not None and value.strip() != "" + + +def _extract_dict_text(data: dict[str, Any], key: str) -> str | None: + if key not in data: + return None + return _coerce_payload_text(data.get(key)) + + +def _contains(score: JSONValue, threshold: JSONValue) -> bool: + if threshold is None: + return False + if isinstance(score, str): + return str(threshold) in score + if isinstance(score, list): + return threshold in score + if isinstance(score, dict): + return threshold in score.values() + return False + + +def _confidence_from_score(score: JSONValue) -> float: + if isinstance(score, bool): + return 1.0 if score else 0.0 + number = coerce_number(score) + if number is not None and 0.0 <= number <= 1.0: + return number + return 1.0 + + +def _truncated_http_response_body(body: str) -> tuple[str, bool]: + if len(body) <= _HTTP_ERROR_BODY_LIMIT: + return body, False + return body[:_HTTP_ERROR_BODY_LIMIT], True + + +def _http_status_error_metadata(error: httpx.HTTPStatusError) -> dict[str, Any]: + metadata: dict[str, Any] = {} + + request = error.request + metadata["http_method"] = request.method + metadata["http_endpoint_path"] = request.url.path + + response = error.response + metadata["http_status_code"] = response.status_code + metadata["http_response_content_type"] = response.headers.get("content-type") + + body = response.text + if body: + metadata["http_response_body"], metadata["http_response_body_truncated"] = ( + _truncated_http_response_body(body) + ) + + return {key: value for key, value in metadata.items() if value is not None} + + +class BaseGalileoScorerEvaluator(Evaluator[BaseScorerEvaluatorConfig]): + """Shared base for Galileo direct scorer evaluators. + + Subclasses set ``metadata`` and ``config_model`` to register as distinct + evaluator entry points (e.g. ``galileo.luna``, ``galileo.llm``). + """ + + config_model = BaseScorerEvaluatorConfig + + # Human-readable label used in result messages. Subclasses may override. + _display_name: str = "Galileo scorer" + + @classmethod + def is_available(cls) -> bool: + """Check whether required runtime dependencies are available.""" + return True + + def __init__(self, config: BaseScorerEvaluatorConfig) -> None: + has_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + if not has_secret: + raise ValueError( + "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for scorer " + "invocation. Set one as an environment variable before using this evaluator." + ) + super().__init__(config) + self._client = GalileoScorerClient() + + def _get_client(self) -> GalileoScorerClient: + return self._client + + def _prepare_payload(self, data: Any) -> tuple[str | None, str | None]: + """Prepare scorer input/output fields from selected data.""" + if isinstance(data, dict): + input_text = _extract_dict_text(data, "input") + output_text = _extract_dict_text(data, "output") + if _has_text(input_text) or _has_text(output_text): + return input_text, output_text + + text = _coerce_payload_text(data) + if self.config.payload_field == "output": + return None, text + return text, None + + def _score_matches(self, score: JSONValue) -> bool: + """Apply the configured local threshold comparison to a raw scorer score.""" + operator = self.config.operator + threshold = self.config.threshold + + if operator == "any": + return bool(score) + if operator == "eq": + return score == threshold + if operator == "ne": + return score != threshold + if operator == "contains": + return _contains(score, threshold) + + score_number = coerce_number(score) + threshold_number = coerce_number(threshold) + if score_number is None: + raise ValueError(f"scorer score {score!r} is not numeric") + if threshold_number is None: + raise ValueError(f"scorer threshold {threshold!r} is not numeric") + + if operator == "gt": + return score_number > threshold_number + if operator == "gte": + return score_number >= threshold_number + if operator == "lt": + return score_number < threshold_number + if operator == "lte": + return score_number <= threshold_number + + raise ValueError(f"Unsupported scorer operator: {operator}") + + async def evaluate(self, data: Any) -> EvaluatorResult: + """Evaluate selected data with Galileo direct scorer invocation. + + Args: + data: The data selected from the runtime step. + + Returns: + EvaluatorResult with local threshold decision and scorer metadata. + """ + return await self._evaluate(data, step=None) + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + """Evaluate selected data while dual-writing the complete runtime step. + + Args: + data: Data selected by the configured control selector. + step: Complete runtime step for structured scorer context. + + Returns: + EvaluatorResult with local threshold decision and scorer metadata. + """ + return await self._evaluate(data, step=step) + + async def evaluate_with_extensions( + self, + data: Any, + step: Step, + extensions: JSONObject | None, + ) -> EvaluatorResult: + """Evaluate using opaque authenticated metadata supplied by Agent Control.""" + return await self._evaluate(data, step=step, extensions=extensions) + + @staticmethod + def _execution_context_from_extensions( + extensions: JSONObject | None, + ) -> GalileoExecutionContext | None: + """Translate trusted opaque auth metadata into Galileo's request context.""" + if extensions is None: + return None + + metadata = extensions.get("metadata") + metadata_obj = metadata if isinstance(metadata, dict) else {} + organization_id = extensions.get("namespace_key") + # The runtime envelope's caller_id is supplied by the authenticated + # principal and is the caller identity Galileo associates with this run. + user_id = extensions.get("caller_id") + project_id = metadata_obj.get("project_id") + target_type = extensions.get("target_type") + target_id = extensions.get("target_id") + run_id = ( + target_id + if target_type in ("log_stream", "agent_stream") + and isinstance(target_id, str) + and target_id + else None + ) + if not isinstance(organization_id, str) or not organization_id: + raise ValueError("Authenticated execution metadata is missing organization_id") + + return GalileoExecutionContext( + organization_id=organization_id, + user_id=user_id if isinstance(user_id, str) and user_id else None, + project_id=project_id if isinstance(project_id, str) and project_id else None, + run_id=run_id, + ) + + async def _evaluate( + self, + data: Any, + *, + step: Step | None, + extensions: JSONObject | None = None, + ) -> EvaluatorResult: + """Run a scorer evaluation with optional structured runtime context.""" + display = self._display_name + input_text, output_text = self._prepare_payload(data) + if not (_has_text(input_text) or _has_text(output_text)): + return EvaluatorResult( + matched=False, + confidence=1.0, + message=f"No data to score with {display}", + metadata=self._base_metadata(), + ) + + try: + execution_context = self._execution_context_from_extensions(extensions) + scorer_kwargs = self._scorer_kwargs() + if step is not None: + scorer_kwargs["step"] = step + if execution_context is not None: + scorer_kwargs["execution_context"] = execution_context + response = await self._get_client().invoke( + **scorer_kwargs, + input=input_text if _has_text(input_text) else None, + output=output_text if _has_text(output_text) else None, + config=self.config.scorer_config, + timeout=self.get_timeout_seconds(), + ) + + if response.status.lower() != "success": + message = ( + response.error_message or f"{display} scorer status: {response.status}" + ) + raise RuntimeError(message) + + matched = self._score_matches(response.score) + metadata = self._metadata(response) + operator = self.config.operator + threshold = self.config.threshold + state = "triggered" if matched else "not triggered" + return EvaluatorResult( + matched=matched, + confidence=_confidence_from_score(response.score), + message=( + f"{display} score {response.score!r} {operator} threshold " + f"{threshold!r}: control {state}." + ), + metadata=metadata, + ) + except Exception as exc: + logger.error("%s scorer evaluation error: %s", display, exc, exc_info=True) + return self._handle_error(exc) + + def _base_metadata(self) -> dict[str, Any]: + """Build result metadata without implying a requested version executed.""" + metadata: dict[str, Any] = {"scorer_id": self.config.scorer_id} + if self.config.scorer_version_id is not None: + metadata["requested_scorer_version_id"] = self.config.scorer_version_id + if self.config.scorer_label is not None: + metadata["scorer_label"] = self.config.scorer_label + return metadata + + def _scorer_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"scorer_id": self.config.scorer_id} + if self.config.scorer_version_id is not None: + kwargs["scorer_version_id"] = self.config.scorer_version_id + if self.config.scorer_label is not None: + kwargs["scorer_label"] = self.config.scorer_label + return kwargs + + def _metadata(self, response: ScorerInvokeResponse) -> dict[str, Any]: + metadata: dict[str, Any] = self._base_metadata() + echoed_label = response.scorer_label or self.config.scorer_label + if echoed_label is not None: + metadata["scorer_label"] = echoed_label + metadata.update( + { + "score": response.score, + "threshold": self.config.threshold, + "operator": self.config.operator, + "status": response.status, + "execution_time_seconds": response.execution_time, + "error_message": response.error_message, + } + ) + return metadata + + def _handle_error(self, error: Exception) -> EvaluatorResult: + error_detail = str(error) + metadata: dict[str, Any] = { + **self._base_metadata(), + "error_type": type(error).__name__, + } + if isinstance(error, httpx.HTTPStatusError): + metadata.update(_http_status_error_metadata(error)) + + return EvaluatorResult( + matched=False, + confidence=0.0, + message=f"{self._display_name} evaluation error: {error_detail}", + metadata=metadata, + error=error_detail, + ) + + async def aclose(self) -> None: + """Close the underlying Galileo scorer client.""" + await self._client.close() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py index 396b0216c..32fd52bfa 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/client.py @@ -1,616 +1,24 @@ -"""Direct HTTP client for Galileo LLM-as-judge scorer invocation.""" - -from __future__ import annotations - -import logging -import os -import ssl -from asyncio import Lock -from base64 import urlsafe_b64encode -from hashlib import sha256 -from hmac import new as hmac_new -from json import dumps -from time import time -from typing import Literal, cast, get_args -from urllib.parse import urlsplit - -import httpx -from agent_control_models import JSONObject, JSONValue, Step -from pydantic import BaseModel, Field, PrivateAttr, model_validator - -from .config import ScorerInvokeConfig - -logger = logging.getLogger(__name__) - -DEFAULT_TIMEOUT_SECS = 10.0 -SERVER_TIMEOUT_RATIO = 0.8 -DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 -DEFAULT_LLM_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" -LLM_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" -LLM_INVOKE_CA_FILE_ENV = "GALILEO_LUNA_INVOKE_CA_FILE" -AUTH_UPSTREAM_CA_FILE_ENV = "AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE" - -# Headers that must never be forwarded to the scorer invoke endpoint (checked case-insensitively). -_BLOCKED_REQUEST_HEADERS = frozenset({"galileo-api-key"}) - -# Keep pooled-connection reuse shorter than typical server keepalive/worker -# recycle windows so requests do not pick up sockets the server already closed. -DEFAULT_KEEPALIVE_EXPIRY_SECS = 1.0 -DEFAULT_MAX_CONNECTIONS = 100 -DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 20 -DEFAULT_CLIENT_POOL_SIZE = 1 -LLM_KEEPALIVE_EXPIRY_ENV = "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS" -LLM_MAX_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_CONNECTIONS" -LLM_MAX_KEEPALIVE_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS" -LLM_CLIENT_POOL_SIZE_ENV = "GALILEO_LUNA_CLIENT_POOL_SIZE" - -# These values mirror Orbit's StepType discriminator. Agent Control's generic -# Step remains extensible; only the Galileo transport boundary is constrained. -ScorerInvokeRecordType = Literal[ - "llm", - "retriever", - "tool", - "workflow", - "agent", - "control", - "trace", - "session", +"""Galileo LLM-as-judge scorer client — re-exported from shared implementation.""" + +from agent_control_evaluator_galileo._shared.client import ( + DEFAULT_TIMEOUT_SECS, + GalileoExecutionContext, + GalileoScorerClient, + ScorerInvokeInputs, + ScorerInvokeRecord, + ScorerInvokeRequest, + ScorerInvokeResponse, +) + +# Public alias: GalileoLLMClient is the named export users and tests reference. +GalileoLLMClient = GalileoScorerClient + +__all__ = [ + "DEFAULT_TIMEOUT_SECS", + "GalileoExecutionContext", + "GalileoLLMClient", + "ScorerInvokeInputs", + "ScorerInvokeRecord", + "ScorerInvokeRequest", + "ScorerInvokeResponse", ] -SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) - - -def _b64url(data: bytes) -> str: - return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") - - -def _internal_auth_token( - api_secret: str, - ttl_seconds: int = DEFAULT_INTERNAL_TOKEN_TTL_SECS, -) -> str: - """Create the internal JWT expected by LLM scorer invoke routes.""" - now = int(time()) - header = {"alg": "HS256", "typ": "JWT"} - payload = { - "internal": True, - "scope": "scorers.invoke", - "iat": now, - "exp": now + ttl_seconds, - } - signing_input = ".".join( - [ - _b64url(dumps(header, separators=(",", ":")).encode("utf-8")), - _b64url(dumps(payload, separators=(",", ":")).encode("utf-8")), - ] - ) - signature = hmac_new(api_secret.encode("utf-8"), signing_input.encode("ascii"), sha256).digest() - return f"{signing_input}.{_b64url(signature)}" - - -def _normalize_llm_invoke_url(raw_url: str) -> str: - """Use full invoke URLs as-is and append the default path to bare service roots.""" - url = raw_url.strip().rstrip("/") - parsed = urlsplit(url) - if parsed.path not in ("", "/") or parsed.query or parsed.fragment: - return url - return f"{url}{DEFAULT_LLM_SCORER_INVOKE_PATH}" - - -def _load_float_env(env_name: str, default: float) -> float: - raw = os.getenv(env_name) - if raw is None or raw.strip() == "": - return default - try: - return float(raw) - except ValueError as exc: - raise ValueError(f"{env_name}={raw!r} is not a number.") from exc - - -def _load_int_env(env_name: str, default: int) -> int: - raw = os.getenv(env_name) - if raw is None or raw.strip() == "": - return default - try: - return int(raw) - except ValueError as exc: - raise ValueError(f"{env_name}={raw!r} is not an integer.") from exc - - -def _validate_connection_config( - *, - keepalive_expiry_seconds: float, - max_connections: int, - max_keepalive_connections: int, - client_pool_size: int, -) -> None: - if keepalive_expiry_seconds < 0: - raise ValueError( - f"{LLM_KEEPALIVE_EXPIRY_ENV}={keepalive_expiry_seconds} " - "must be greater than or equal to 0." - ) - if max_connections <= 0: - raise ValueError(f"{LLM_MAX_CONNECTIONS_ENV}={max_connections} must be greater than 0.") - if max_keepalive_connections < 0: - raise ValueError( - f"{LLM_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " - "must be greater than or equal to 0." - ) - if max_keepalive_connections > max_connections: - raise ValueError( - f"{LLM_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " - f"must be less than or equal to {LLM_MAX_CONNECTIONS_ENV}={max_connections}." - ) - if client_pool_size <= 0: - raise ValueError(f"{LLM_CLIENT_POOL_SIZE_ENV}={client_pool_size} must be greater than 0.") - - -def _as_float_or_none(value: JSONValue) -> float | None: - if isinstance(value, bool) or value is None: - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return None - return None - - -def _has_value(value: JSONValue) -> bool: - if value is None: - return False - if isinstance(value, str): - return value.strip() != "" - if isinstance(value, (list, dict)): - return len(value) > 0 - return True - - -def _effective_scorer_timeout( - config: ScorerInvokeConfig, - *, - http_timeout_seconds: float, -) -> ScorerInvokeConfig: - """Resolve an Orbit execution timeout that expires before the HTTP request. - - The server execution budget defaults to 80% of the caller's HTTP deadline, - leaving time for Orbit to serialize and return the result. Explicit caller - overrides are preserved only when they maintain the same ordering. - - Args: - config: Caller-provided scorer-invoke configuration. - http_timeout_seconds: Agent Control's HTTP request deadline in seconds. - - Returns: - A scorer-invoke configuration with an effective execution timeout. - - Raises: - ValueError: If either deadline is invalid or the explicit server timeout - is not shorter than the HTTP deadline. - """ - if http_timeout_seconds <= 0: - raise ValueError("HTTP timeout must be greater than 0 seconds.") - - server_timeout = config.request_timeout_seconds - if server_timeout is None: - server_timeout = http_timeout_seconds * SERVER_TIMEOUT_RATIO - elif server_timeout >= http_timeout_seconds: - raise ValueError( - "config.request_timeout_seconds must be shorter than the HTTP " - f"timeout ({http_timeout_seconds:g} seconds)." - ) - - return config.model_copy(update={"request_timeout_seconds": server_timeout}) - - -class ScorerInvokeInputs(BaseModel): - """Input values sent to the LLM scorer invoke endpoint.""" - - query: JSONValue = "" - response: JSONValue = "" - ground_truth: JSONValue = None - tools: list[JSONObject] | None = None - - -class ScorerInvokeRecord(BaseModel): - """Caller-controlled subset of Orbit's partial runtime-record contract. - - Identity, ownership, persistence, and execution IDs are intentionally not - represented here. Orbit hydrates those fields from trusted server context. - """ - - type: ScorerInvokeRecordType - name: str | None = None - input: JSONValue = None - output: JSONValue = None - context: JSONObject | None = None - tools: list[JSONObject] | None = None - dataset_output: JSONValue = None - - -class GalileoExecutionContext(BaseModel): - """Authenticated organization and optional identity fields for Galileo scorer invocation.""" - - organization_id: str = Field(min_length=1) - user_id: str | None = Field(default=None, min_length=1) - project_id: str | None = Field(default=None, min_length=1) - run_id: str | None = Field(default=None, min_length=1) - - -class ScorerInvokeRequest(BaseModel): - """Request payload for LLM scorer invocation. - - Attributes: - scorer_id: Required scorer identifier. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - inputs: Selected scorer input values. - record: Optional Orbit-compatible structured runtime record. - execution_context: Optional authenticated execution context. - config: Scorer-specific configuration, always emitted. - """ - - scorer_id: str = Field(min_length=1) - scorer_version_id: str | None = Field(default=None, min_length=1) - scorer_label: str | None = Field(default=None, min_length=1) - inputs: ScorerInvokeInputs - record: ScorerInvokeRecord | None = None - execution_context: GalileoExecutionContext | None = None - config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) - - @model_validator(mode="after") - def ensure_required_values(self) -> ScorerInvokeRequest: - if not (_has_value(self.inputs.query) or _has_value(self.inputs.response)): - raise ValueError("Either inputs.query or inputs.response must be set.") - return self - - def to_dict(self) -> JSONObject: - """Convert to the LLM scorer invoke request shape.""" - request = self.model_dump(mode="json", exclude_none=True) - if self.execution_context is not None: - request["execution_context"] = self.execution_context.model_dump(mode="json") - return request - - -def _orbit_record_from_step( - step: Step | None, - *, - selected_input: JSONValue, - selected_output: JSONValue, -) -> ScorerInvokeRecord | None: - """Translate a generic Agent Control step into Orbit's record contract. - - Selector-selected values remain the primary scorer input. When a selector - supplies one side, that value is written to both the legacy and structured - representations so Orbit's conflict validation cannot observe two meanings. - The complete step supplies the unselected side and additional record context. - - Unknown Agent Control step types intentionally fall back to the legacy - ``inputs`` contract. This keeps the open-source Step model extensible without - sending an invalid discriminator to Orbit. - - Args: - step: Complete Agent Control step, when contextual evaluation is used. - selected_input: Selector-selected value sent as ``inputs.query``. - selected_output: Selector-selected value sent as ``inputs.response``. - - Returns: - An Orbit-compatible record, or ``None`` for absent/unsupported steps. - """ - if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: - return None - - # The membership check above narrows the runtime value to Orbit's known - # discriminator set, but static type checkers cannot infer that relationship. - record_type = cast(ScorerInvokeRecordType, step.type) - return ScorerInvokeRecord( - type=record_type, - name=step.name, - input=selected_input if selected_input is not None else step.input, - output=selected_output if selected_output is not None else step.output, - context=step.context, - tools=step.tools, - dataset_output=step.ground_truth, - ) - - -class ScorerInvokeResponse(BaseModel): - """Response from LLM scorer invocation. - - Attributes: - scorer_label: Echoed scorer label, when returned. - score: Raw scorer value. - status: Invocation status. - execution_time: Execution time in seconds, when returned. - error_message: Error detail for non-success statuses. - """ - - scorer_label: str | None = None - score: JSONValue - status: str = "unknown" - execution_time: float | None = None - error_message: str | None = None - _raw_response: JSONObject = PrivateAttr(default_factory=dict) - - @property - def raw_response(self) -> JSONObject: - return self._raw_response - - @classmethod - def from_dict(cls, data: JSONObject) -> ScorerInvokeResponse: - """Create a response model from the LLM scorer invoke JSON object.""" - response = cls.model_validate( - data | {"execution_time": _as_float_or_none(data.get("execution_time"))} - ) - response._raw_response = data - return response - - -class GalileoLLMClient: - """Thin HTTP client for Galileo LLM-as-judge scorer invocation. - - Environment Variables: - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. - GALILEO_LUNA_INVOKE_URL: LLM scorer invoke URL or service root (required). - GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify invoke TLS. - AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. - GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry. - GALILEO_LUNA_MAX_CONNECTIONS: Maximum outbound HTTP connections. - GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS: Maximum idle pooled HTTP connections. - GALILEO_LUNA_CLIENT_POOL_SIZE: Number of outbound HTTP clients to rotate across. - """ - - def __init__( - self, - api_secret: str | None = None, - llm_invoke_url: str | None = None, - llm_invoke_ca_file: str | None = None, - ) -> None: - """Initialize the Galileo LLM client. - - Args: - api_secret: Internal JWT signing secret. If not provided, reads from - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. - llm_invoke_url: LLM scorer invoke URL or service root. If not provided, - reads from GALILEO_LUNA_INVOKE_URL. - llm_invoke_ca_file: Optional CA bundle used to verify invoke TLS. If not - provided, reads from GALILEO_LUNA_INVOKE_CA_FILE, then - AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE. - - Raises: - ValueError: If the API secret, invoke URL, CA bundle, or connection - tuning configuration is invalid. - """ - resolved_api_secret = ( - api_secret or os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") - ) - if not resolved_api_secret: - raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for LLM " - "scorer invocation. Set one as an environment variable or pass it " - "to the constructor." - ) - - resolved_llm_invoke_url = llm_invoke_url or os.getenv(LLM_INVOKE_URL_ENV) - if resolved_llm_invoke_url is None or resolved_llm_invoke_url.strip() == "": - raise ValueError( - "GALILEO_LUNA_INVOKE_URL is required for LLM scorer invocation. " - "Set it as an environment variable or pass it to the constructor." - ) - - self.api_secret = resolved_api_secret - self.llm_invoke_url = _normalize_llm_invoke_url(resolved_llm_invoke_url) - self.llm_invoke_ca_file = ( - llm_invoke_ca_file - or os.getenv(LLM_INVOKE_CA_FILE_ENV) - or os.getenv(AUTH_UPSTREAM_CA_FILE_ENV) - or "" - ).strip() or None - self._ssl_context = self._load_ssl_context(self.llm_invoke_ca_file) - self.keepalive_expiry_seconds = _load_float_env( - LLM_KEEPALIVE_EXPIRY_ENV, DEFAULT_KEEPALIVE_EXPIRY_SECS - ) - self.max_connections = _load_int_env(LLM_MAX_CONNECTIONS_ENV, DEFAULT_MAX_CONNECTIONS) - self.max_keepalive_connections = _load_int_env( - LLM_MAX_KEEPALIVE_CONNECTIONS_ENV, DEFAULT_MAX_KEEPALIVE_CONNECTIONS - ) - self.client_pool_size = _load_int_env(LLM_CLIENT_POOL_SIZE_ENV, DEFAULT_CLIENT_POOL_SIZE) - _validate_connection_config( - keepalive_expiry_seconds=self.keepalive_expiry_seconds, - max_connections=self.max_connections, - max_keepalive_connections=self.max_keepalive_connections, - client_pool_size=self.client_pool_size, - ) - self._client: httpx.AsyncClient | None = None - self._clients: list[httpx.AsyncClient] = [] - self._next_client_index = 0 - self._client_lock = Lock() - - @staticmethod - def _load_ssl_context(ca_file: str | None) -> ssl.SSLContext | None: - """Build a TLS verification context from a CA bundle path, if configured.""" - if ca_file is None: - return None - try: - return ssl.create_default_context(cafile=ca_file) - except (OSError, ssl.SSLError) as exc: - raise ValueError(f"Failed to load CA bundle from {ca_file!r}: {exc}") from exc - - def _create_client(self) -> httpx.AsyncClient: - """Create an HTTP client with the configured TLS and connection limits.""" - verify: ssl.SSLContext | bool = self._ssl_context if self._ssl_context is not None else True - return httpx.AsyncClient( - headers={"Content-Type": "application/json"}, - timeout=httpx.Timeout(DEFAULT_TIMEOUT_SECS), - limits=httpx.Limits( - max_connections=self.max_connections, - max_keepalive_connections=self.max_keepalive_connections, - keepalive_expiry=self.keepalive_expiry_seconds, - ), - verify=verify, - ) - - def _select_pooled_client(self) -> httpx.AsyncClient: - """Select the next pooled client while holding the client state lock.""" - client = self._clients[self._next_client_index % len(self._clients)] - self._next_client_index = (self._next_client_index + 1) % len(self._clients) - return client - - async def _get_client(self) -> httpx.AsyncClient: - """Get or create the next HTTP client.""" - async with self._client_lock: - self._clients = [client for client in self._clients if not client.is_closed] - - if self.client_pool_size == 1: - if self._client is not None and not self._client.is_closed: - return self._client - self._client = self._clients[0] if self._clients else self._create_client() - self._clients = [self._client] - return self._client - - self._client = None - while len(self._clients) < self.client_pool_size: - self._clients.append(self._create_client()) - - return self._select_pooled_client() - - def _endpoint_and_auth_header(self) -> tuple[str, str]: - token = _internal_auth_token(self.api_secret) - return self.llm_invoke_url, f"Bearer {token}" - - async def invoke( - self, - *, - scorer_id: str, - scorer_version_id: str | None = None, - scorer_label: str | None = None, - input: JSONValue = None, - output: JSONValue = None, - step: Step | None = None, - execution_context: GalileoExecutionContext | None = None, - config: ScorerInvokeConfig | JSONObject | None = None, - timeout: float = DEFAULT_TIMEOUT_SECS, - headers: dict[str, str] | None = None, - ) -> ScorerInvokeResponse: - """Invoke a Galileo LLM-as-judge scorer. - - Args: - scorer_id: Required scorer identifier. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - input: Optional user/system prompt text. - output: Optional model response text. - step: Optional complete runtime step used for structured dual-write. - execution_context: Optional authenticated Galileo scorer context. - config: Optional Orbit-supported scorer invocation configuration. - timeout: Request timeout in seconds. - headers: Additional request headers. - - Returns: - Parsed scorer invocation response. - - Raises: - ValueError: If neither input nor output is provided, config contains - a field Orbit does not support, or the timeout ordering is invalid. - RuntimeError: If the API response is not a JSON object. - httpx.HTTPStatusError: If the invoke endpoint returns an error status code. - httpx.RequestError: If the request fails before a response is received. - """ - if not (_has_value(input) or _has_value(output)): - raise ValueError("At least one of input or output must be provided.") - - # Accept dictionaries for source compatibility with the original client, - # but validate them against Orbit's authoritative allowlist locally. - invoke_config = ( - ScorerInvokeConfig.model_validate(config) - if config is not None - else ScorerInvokeConfig() - ) - invoke_config = _effective_scorer_timeout( - invoke_config, - http_timeout_seconds=timeout, - ) - request_body = ScorerInvokeRequest( - scorer_id=scorer_id, - scorer_version_id=scorer_version_id, - scorer_label=scorer_label, - inputs=ScorerInvokeInputs( - query="" if input is None else input, - response="" if output is None else output, - ground_truth=step.ground_truth if step is not None else None, - tools=step.tools if step is not None else None, - ), - record=_orbit_record_from_step( - step, - selected_input=input, - selected_output=output, - ), - execution_context=execution_context, - config=invoke_config, - ).to_dict() - - endpoint, auth_header = self._endpoint_and_auth_header() - request_headers = { - k: v for k, v in (headers or {}).items() if k.lower() not in _BLOCKED_REQUEST_HEADERS - } - request_headers["Authorization"] = auth_header - - logger.debug("[GalileoLLMClient] POST %s", endpoint) - logger.debug("[GalileoLLMClient] Request body: %s", request_body) - - try: - client = await self._get_client() - response = await client.post( - endpoint, - json=request_body, - headers=request_headers, - timeout=timeout, - ) - response.raise_for_status() - response_data = response.json() - if not isinstance(response_data, dict): - raise RuntimeError("Invalid response payload: not a JSON object") - - parsed = ScorerInvokeResponse.from_dict(response_data) - logger.debug("[GalileoLLMClient] Response: %s", parsed.raw_response) - return parsed - except httpx.HTTPStatusError as exc: - logger.error( - "[GalileoLLMClient] API error: %s - %s", - exc.response.status_code, - exc.response.text, - ) - raise - except httpx.RequestError as exc: - logger.error("[GalileoLLMClient] Request failed: %s", exc) - raise - - async def close(self) -> None: - """Close HTTP clients and release resources.""" - async with self._client_lock: - clients: list[httpx.AsyncClient] = [] - seen_client_ids: set[int] = set() - if self._client is not None: - clients.append(self._client) - seen_client_ids.add(id(self._client)) - self._client = None - for client in self._clients: - if id(client) not in seen_client_ids: - clients.append(client) - seen_client_ids.add(id(client)) - self._clients = [] - self._next_client_index = 0 - - for client in clients: - if not client.is_closed: - await client.aclose() - - async def __aenter__(self) -> GalileoLLMClient: - """Async context manager entry.""" - return self - - async def __aexit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: - """Async context manager exit.""" - await self.close() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py index 1b207f3af..66f6236cb 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/config.py @@ -1,124 +1,24 @@ -"""Configuration model for direct Galileo LLM-as-judge scorer evaluation.""" +"""Configuration for Galileo LLM-as-judge scorer evaluator.""" -from __future__ import annotations +from agent_control_evaluator_galileo._shared.config import ( + BaseScorerEvaluatorConfig, + ScorerInvokeConfig, + coerce_number, +) -from typing import Literal +# Public aliases kept for consistency with luna naming convention. +LlmOperator = BaseScorerEvaluatorConfig.model_fields["operator"].annotation +LlmPayloadField = BaseScorerEvaluatorConfig.model_fields["payload_field"].annotation -from agent_control_evaluators import EvaluatorConfig -from agent_control_models import JSONValue -from pydantic import BaseModel, ConfigDict, Field, model_validator -LlmOperator = Literal["gt", "gte", "lt", "lte", "eq", "ne", "contains", "any"] -LlmPayloadField = Literal["input", "output"] +class LlmEvaluatorConfig(BaseScorerEvaluatorConfig): + """Configuration for direct LLM-as-judge scorer evaluation.""" -_NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) - -class ScorerInvokeConfig(BaseModel): - """Orbit-supported overrides for a synchronous scorer invocation. - - Orbit owns the Galileo scorer-invoke wire contract. Keeping this model - strict makes an unsupported option fail locally instead of producing a - less actionable HTTP 422 response from Runners. - - Attributes: - threshold: Legacy threshold accepted by Orbit. Agent Control still - applies its evaluator threshold locally. - score_threshold: Legacy score threshold accepted by Orbit. - request_timeout_seconds: Optional upper bound for scorer execution in - Orbit. The Agent Control HTTP and evaluator deadlines must remain - longer than this value. - """ - - model_config = ConfigDict(extra="forbid") - - threshold: float | None = None - score_threshold: float | None = None - request_timeout_seconds: float | None = Field(default=None, gt=0) - - -def coerce_number(value: JSONValue) -> float | None: - """Return a numeric value for JSON scalars that can be compared numerically.""" - if isinstance(value, bool) or value is None: - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return None - return None - - -class LlmEvaluatorConfig(EvaluatorConfig): - """Configuration for direct LLM-as-judge scorer evaluation. - - Attributes: - scorer_id: Required scorer identifier for LLM scorer invocation. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - threshold: Local threshold used by the evaluator for comparison. - operator: Local comparison operator. Numeric operators use threshold as a number. - scorer_config: Optional Orbit-supported scorer invocation config sent - as ``config``. - payload_field: Explicit scorer input side for scalar selected data. - timeout_ms: Request timeout in milliseconds. - """ - - scorer_id: str = Field( - min_length=1, - description="Required scorer identifier for LLM scorer invocation.", - ) - scorer_version_id: str | None = Field( - default=None, - min_length=1, - description=( - "Deprecated optional compatibility identifier. Orbit currently invokes " - "the scorer's current default version." - ), - ) - scorer_label: str | None = Field( - default=None, - min_length=1, - description="Optional display/metadata label.", - ) - threshold: JSONValue = Field( - default=0.5, - description="Local threshold used to decide whether the control matches.", - ) - operator: LlmOperator = Field( - default="gte", - description="Local comparison operator applied to the raw LLM scorer score.", - ) - scorer_config: ScorerInvokeConfig | None = Field( - default=None, - alias="config", - serialization_alias="config", - description=( - "Optional Orbit-supported configuration sent to the LLM scorer invoke endpoint." - ), - ) - payload_field: LlmPayloadField = Field( - default="input", - description=( - "Which scorer input side to use when selector output is a scalar value. " - "Structured selected data with input/output keys overrides this setting." - ), - ) - timeout_ms: int = Field( - default=10000, - ge=1000, - le=60000, - description="Request timeout in milliseconds (1-60 seconds)", - ) - - @model_validator(mode="after") - def validate_threshold(self) -> LlmEvaluatorConfig: - """Validate threshold compatibility with the configured operator.""" - if self.operator in _NUMERIC_OPERATORS and coerce_number(self.threshold) is None: - raise ValueError(f"operator '{self.operator}' requires a numeric threshold") - if self.operator != "any" and self.threshold is None: - raise ValueError("threshold is required unless operator is 'any'") - return self +__all__ = [ + "LlmEvaluatorConfig", + "LlmOperator", + "LlmPayloadField", + "ScorerInvokeConfig", + "coerce_number", +] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py index 6c12ddd18..5da299f0a 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/evaluator.py @@ -1,375 +1,30 @@ -"""Direct Galileo LLM-as-judge evaluator implementation.""" +"""Galileo LLM-as-judge direct scorer evaluator.""" -from __future__ import annotations +from agent_control_evaluators import EvaluatorMetadata, register_evaluator -import json -import logging -import os -from importlib.metadata import PackageNotFoundError, version -from typing import Any +from agent_control_evaluator_galileo._shared.evaluator import ( + BaseGalileoScorerEvaluator, + _resolve_package_version, +) +from agent_control_evaluator_galileo.llm.client import GalileoLLMClient +from agent_control_evaluator_galileo.llm.config import LlmEvaluatorConfig -import httpx -from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator -from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step - -from .client import GalileoExecutionContext, GalileoLLMClient, ScorerInvokeResponse -from .config import LlmEvaluatorConfig, coerce_number - -logger = logging.getLogger(__name__) - - -def _resolve_package_version() -> str: - """Return the installed package version, or a dev fallback during local imports.""" - try: - return version("agent-control-evaluator-galileo") - except PackageNotFoundError: - return "0.0.0.dev" - - -_PACKAGE_VERSION = _resolve_package_version() LLM_AVAILABLE = True -_HTTP_ERROR_BODY_LIMIT = 500 - - -def _coerce_payload_text(value: Any) -> str | None: - """Coerce selected data into scorer text without losing structured values.""" - if value is None: - return None - if isinstance(value, str): - return value - if isinstance(value, (int, float, bool)): - return str(value) - try: - return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) - except TypeError: - return str(value) - - -def _has_text(value: str | None) -> bool: - return value is not None and value.strip() != "" - - -def _extract_dict_text(data: dict[str, Any], key: str) -> str | None: - if key not in data: - return None - return _coerce_payload_text(data.get(key)) - - -def _contains(score: JSONValue, threshold: JSONValue) -> bool: - if threshold is None: - return False - if isinstance(score, str): - return str(threshold) in score - if isinstance(score, list): - return threshold in score - if isinstance(score, dict): - return threshold in score.values() - return False - - -def _confidence_from_score(score: JSONValue) -> float: - if isinstance(score, bool): - return 1.0 if score else 0.0 - number = coerce_number(score) - if number is not None and 0.0 <= number <= 1.0: - return number - return 1.0 - - -def _truncated_http_response_body(body: str) -> tuple[str, bool]: - if len(body) <= _HTTP_ERROR_BODY_LIMIT: - return body, False - return body[:_HTTP_ERROR_BODY_LIMIT], True - - -def _http_status_error_metadata(error: httpx.HTTPStatusError) -> dict[str, Any]: - metadata: dict[str, Any] = {} - - request = error.request - metadata["http_method"] = request.method - metadata["http_endpoint_path"] = request.url.path - - response = error.response - metadata["http_status_code"] = response.status_code - metadata["http_response_content_type"] = response.headers.get("content-type") - - body = response.text - if body: - metadata["http_response_body"], metadata["http_response_body_truncated"] = ( - _truncated_http_response_body(body) - ) - - return {key: value for key, value in metadata.items() if value is not None} @register_evaluator -class LlmEvaluator(Evaluator[LlmEvaluatorConfig]): +class LlmEvaluator(BaseGalileoScorerEvaluator): """Galileo LLM-as-judge evaluator using the direct scorer invocation API.""" metadata = EvaluatorMetadata( name="galileo.llm", - version=_PACKAGE_VERSION, + version=_resolve_package_version(), description="Galileo LLM-as-judge direct scorer evaluation", requires_api_key=True, timeout_ms=10000, ) config_model = LlmEvaluatorConfig - - @classmethod - def is_available(cls) -> bool: - """Check whether required runtime dependencies are available.""" - return LLM_AVAILABLE - - def __init__(self, config: LlmEvaluatorConfig) -> None: - """Initialize the direct LLM-as-judge evaluator. - - Args: - config: Validated LlmEvaluatorConfig instance. - - Raises: - ValueError: If neither GALILEO_API_SECRET_KEY nor GALILEO_API_SECRET is set. - """ - has_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") - if not has_secret: - raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for LLM " - "scorer invocation. Set one as an environment variable before using " - "galileo.llm." - ) - - super().__init__(config) - self._client = GalileoLLMClient() + _display_name = "LLM scorer" def _get_client(self) -> GalileoLLMClient: - """Get the Galileo LLM client.""" - return self._client - - def _prepare_payload(self, data: Any) -> tuple[str | None, str | None]: - """Prepare scorer input/output fields from selected data.""" - if isinstance(data, dict): - input_text = _extract_dict_text(data, "input") - output_text = _extract_dict_text(data, "output") - if _has_text(input_text) or _has_text(output_text): - return input_text, output_text - - text = _coerce_payload_text(data) - if self.config.payload_field == "output": - return None, text - return text, None - - def _score_matches(self, score: JSONValue) -> bool: - """Apply the configured local threshold comparison to a raw LLM scorer score.""" - operator = self.config.operator - threshold = self.config.threshold - - if operator == "any": - return bool(score) - if operator == "eq": - return score == threshold - if operator == "ne": - return score != threshold - if operator == "contains": - return _contains(score, threshold) - - score_number = coerce_number(score) - threshold_number = coerce_number(threshold) - if score_number is None: - raise ValueError(f"LLM scorer score {score!r} is not numeric") - if threshold_number is None: - raise ValueError(f"LLM scorer threshold {threshold!r} is not numeric") - - if operator == "gt": - return score_number > threshold_number - if operator == "gte": - return score_number >= threshold_number - if operator == "lt": - return score_number < threshold_number - if operator == "lte": - return score_number <= threshold_number - - raise ValueError(f"Unsupported LLM scorer operator: {operator}") - - async def evaluate(self, data: Any) -> EvaluatorResult: - """Evaluate selected data with Galileo LLM-as-judge direct scorer invocation. - - Args: - data: The data selected from the runtime step. - - Returns: - EvaluatorResult with local threshold decision and scorer metadata. - """ - return await self._evaluate(data, step=None) - - async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: - """Evaluate selected data while dual-writing the complete runtime step. - - Args: - data: Data selected by the configured control selector. - step: Complete runtime step for structured scorer context. - - Returns: - EvaluatorResult with local threshold decision and scorer metadata. - """ - return await self._evaluate(data, step=step) - - async def evaluate_with_extensions( - self, - data: Any, - step: Step, - extensions: JSONObject | None, - ) -> EvaluatorResult: - """Evaluate using opaque authenticated metadata supplied by Agent Control.""" - return await self._evaluate( - data, - step=step, - extensions=extensions, - ) - - @staticmethod - def _execution_context_from_extensions( - extensions: JSONObject | None, - ) -> GalileoExecutionContext | None: - """Translate trusted opaque auth metadata into Galileo's request context.""" - if extensions is None: - return None - - metadata = extensions.get("metadata") - metadata_obj = metadata if isinstance(metadata, dict) else {} - organization_id = extensions.get("namespace_key") - # The runtime envelope's caller_id is supplied by the authenticated - # principal and is the caller identity Galileo associates with this run. - user_id = extensions.get("caller_id") - project_id = metadata_obj.get("project_id") - target_type = extensions.get("target_type") - target_id = extensions.get("target_id") - run_id = ( - target_id - if target_type in ("log_stream", "agent_stream") - and isinstance(target_id, str) - and target_id - else None - ) - if not isinstance(organization_id, str) or not organization_id: - raise ValueError("Authenticated execution metadata is missing organization_id") - - return GalileoExecutionContext( - organization_id=organization_id, - user_id=user_id if isinstance(user_id, str) and user_id else None, - project_id=project_id if isinstance(project_id, str) and project_id else None, - run_id=run_id, - ) - - async def _evaluate( - self, - data: Any, - *, - step: Step | None, - extensions: JSONObject | None = None, - ) -> EvaluatorResult: - """Run an LLM scorer evaluation with optional structured runtime context.""" - input_text, output_text = self._prepare_payload(data) - if not (_has_text(input_text) or _has_text(output_text)): - return EvaluatorResult( - matched=False, - confidence=1.0, - message="No data to score with LLM scorer", - metadata=self._base_metadata(), - ) - - try: - execution_context = self._execution_context_from_extensions(extensions) - scorer_kwargs = self._scorer_kwargs() - if step is not None: - scorer_kwargs["step"] = step - if execution_context is not None: - scorer_kwargs["execution_context"] = execution_context - response = await self._get_client().invoke( - **scorer_kwargs, - input=input_text if _has_text(input_text) else None, - output=output_text if _has_text(output_text) else None, - config=self.config.scorer_config, - timeout=self.get_timeout_seconds(), - ) - - if response.status.lower() != "success": - message = response.error_message or f"LLM scorer status: {response.status}" - raise RuntimeError(message) - - matched = self._score_matches(response.score) - metadata = self._metadata(response) - operator = self.config.operator - threshold = self.config.threshold - state = "triggered" if matched else "not triggered" - return EvaluatorResult( - matched=matched, - confidence=_confidence_from_score(response.score), - message=( - f"LLM scorer score {response.score!r} {operator} threshold " - f"{threshold!r}: control {state}." - ), - metadata=metadata, - ) - except Exception as exc: - logger.error("LLM scorer evaluation error: %s", exc, exc_info=True) - return self._handle_error(exc) - - def _base_metadata(self) -> dict[str, Any]: - """Build result metadata without implying a requested version executed.""" - metadata: dict[str, Any] = {"scorer_id": self.config.scorer_id} - if self.config.scorer_version_id is not None: - metadata["requested_scorer_version_id"] = self.config.scorer_version_id - if self.config.scorer_label is not None: - metadata["scorer_label"] = self.config.scorer_label - return metadata - - def _scorer_kwargs(self) -> dict[str, Any]: - kwargs: dict[str, Any] = {"scorer_id": self.config.scorer_id} - if self.config.scorer_version_id is not None: - kwargs["scorer_version_id"] = self.config.scorer_version_id - if self.config.scorer_label is not None: - kwargs["scorer_label"] = self.config.scorer_label - return kwargs - - def _metadata( - self, - response: ScorerInvokeResponse, - ) -> dict[str, Any]: - metadata: dict[str, Any] = self._base_metadata() - echoed_label = response.scorer_label or self.config.scorer_label - if echoed_label is not None: - metadata["scorer_label"] = echoed_label - metadata.update( - { - "score": response.score, - "threshold": self.config.threshold, - "operator": self.config.operator, - "status": response.status, - "execution_time_seconds": response.execution_time, - "error_message": response.error_message, - } - ) - return metadata - - def _handle_error( - self, - error: Exception, - ) -> EvaluatorResult: - error_detail = str(error) - metadata: dict[str, Any] = { - **self._base_metadata(), - "error_type": type(error).__name__, - } - if isinstance(error, httpx.HTTPStatusError): - metadata.update(_http_status_error_metadata(error)) - - return EvaluatorResult( - matched=False, - confidence=0.0, - message=f"LLM scorer evaluation error: {error_detail}", - metadata=metadata, - error=error_detail, - ) - - async def aclose(self) -> None: - """Close the underlying Galileo LLM client.""" - await self._client.close() + return self._client # type: ignore[return-value] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index 45d65333d..ff3af000d 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -1,615 +1,28 @@ -"""Direct HTTP client for Galileo Luna scorer invocation.""" - -from __future__ import annotations - -import logging -import os -import ssl -from asyncio import Lock -from base64 import urlsafe_b64encode -from hashlib import sha256 -from hmac import new as hmac_new -from json import dumps -from time import time -from typing import Literal, cast, get_args -from urllib.parse import urlsplit +"""Galileo Luna scorer client — re-exported from shared implementation.""" +# ruff: noqa: F401 import httpx -from agent_control_models import JSONObject, JSONValue, Step -from pydantic import BaseModel, Field, PrivateAttr, model_validator - -from .config import ScorerInvokeConfig - -logger = logging.getLogger(__name__) - -DEFAULT_TIMEOUT_SECS = 10.0 -SERVER_TIMEOUT_RATIO = 0.8 -DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 -DEFAULT_LUNA_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" -LUNA_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" -LUNA_INVOKE_CA_FILE_ENV = "GALILEO_LUNA_INVOKE_CA_FILE" -AUTH_UPSTREAM_CA_FILE_ENV = "AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE" - -# Headers that must never be forwarded to the Luna invoke endpoint (checked case-insensitively). -_BLOCKED_REQUEST_HEADERS = frozenset({"galileo-api-key"}) - -# Keep pooled-connection reuse shorter than typical server keepalive/worker -# recycle windows so requests do not pick up sockets the server already closed. -DEFAULT_KEEPALIVE_EXPIRY_SECS = 1.0 -DEFAULT_MAX_CONNECTIONS = 100 -DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 20 -DEFAULT_CLIENT_POOL_SIZE = 1 -LUNA_KEEPALIVE_EXPIRY_ENV = "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS" -LUNA_MAX_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_CONNECTIONS" -LUNA_MAX_KEEPALIVE_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS" -LUNA_CLIENT_POOL_SIZE_ENV = "GALILEO_LUNA_CLIENT_POOL_SIZE" - -# These values mirror Orbit's StepType discriminator. Agent Control's generic -# Step remains extensible; only the Galileo transport boundary is constrained. -ScorerInvokeRecordType = Literal[ - "llm", - "retriever", - "tool", - "workflow", - "agent", - "control", - "trace", - "session", -] -SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) - - -def _b64url(data: bytes) -> str: - return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") - - -def _internal_auth_token( - api_secret: str, - ttl_seconds: int = DEFAULT_INTERNAL_TOKEN_TTL_SECS, -) -> str: - """Create the internal JWT expected by Luna scorer invoke routes.""" - now = int(time()) - header = {"alg": "HS256", "typ": "JWT"} - payload = { - "internal": True, - "scope": "scorers.invoke", - "iat": now, - "exp": now + ttl_seconds, - } - signing_input = ".".join( - [ - _b64url(dumps(header, separators=(",", ":")).encode("utf-8")), - _b64url(dumps(payload, separators=(",", ":")).encode("utf-8")), - ] - ) - signature = hmac_new(api_secret.encode("utf-8"), signing_input.encode("ascii"), sha256).digest() - return f"{signing_input}.{_b64url(signature)}" - - -def _normalize_luna_invoke_url(raw_url: str) -> str: - """Use full invoke URLs as-is and append the default path to bare service roots.""" - url = raw_url.strip().rstrip("/") - parsed = urlsplit(url) - if parsed.path not in ("", "/") or parsed.query or parsed.fragment: - return url - return f"{url}{DEFAULT_LUNA_SCORER_INVOKE_PATH}" - - -def _load_float_env(env_name: str, default: float) -> float: - raw = os.getenv(env_name) - if raw is None or raw.strip() == "": - return default - try: - return float(raw) - except ValueError as exc: - raise ValueError(f"{env_name}={raw!r} is not a number.") from exc - - -def _load_int_env(env_name: str, default: int) -> int: - raw = os.getenv(env_name) - if raw is None or raw.strip() == "": - return default - try: - return int(raw) - except ValueError as exc: - raise ValueError(f"{env_name}={raw!r} is not an integer.") from exc - - -def _validate_connection_config( - *, - keepalive_expiry_seconds: float, - max_connections: int, - max_keepalive_connections: int, - client_pool_size: int, -) -> None: - if keepalive_expiry_seconds < 0: - raise ValueError( - f"{LUNA_KEEPALIVE_EXPIRY_ENV}={keepalive_expiry_seconds} " - "must be greater than or equal to 0." - ) - if max_connections <= 0: - raise ValueError(f"{LUNA_MAX_CONNECTIONS_ENV}={max_connections} must be greater than 0.") - if max_keepalive_connections < 0: - raise ValueError( - f"{LUNA_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " - "must be greater than or equal to 0." - ) - if max_keepalive_connections > max_connections: - raise ValueError( - f"{LUNA_MAX_KEEPALIVE_CONNECTIONS_ENV}={max_keepalive_connections} " - f"must be less than or equal to {LUNA_MAX_CONNECTIONS_ENV}={max_connections}." - ) - if client_pool_size <= 0: - raise ValueError(f"{LUNA_CLIENT_POOL_SIZE_ENV}={client_pool_size} must be greater than 0.") - - -def _as_float_or_none(value: JSONValue) -> float | None: - if isinstance(value, bool) or value is None: - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return None - return None - - -def _has_value(value: JSONValue) -> bool: - if value is None: - return False - if isinstance(value, str): - return value.strip() != "" - if isinstance(value, (list, dict)): - return len(value) > 0 - return True - - -def _effective_scorer_timeout( - config: ScorerInvokeConfig, - *, - http_timeout_seconds: float, -) -> ScorerInvokeConfig: - """Resolve an Orbit execution timeout that expires before the HTTP request. - - The server execution budget defaults to 80% of the caller's HTTP deadline, - leaving time for Orbit to serialize and return the result. Explicit caller - overrides are preserved only when they maintain the same ordering. - - Args: - config: Caller-provided scorer-invoke configuration. - http_timeout_seconds: Agent Control's HTTP request deadline in seconds. - - Returns: - A scorer-invoke configuration with an effective execution timeout. - - Raises: - ValueError: If either deadline is invalid or the explicit server timeout - is not shorter than the HTTP deadline. - """ - if http_timeout_seconds <= 0: - raise ValueError("HTTP timeout must be greater than 0 seconds.") - - server_timeout = config.request_timeout_seconds - if server_timeout is None: - server_timeout = http_timeout_seconds * SERVER_TIMEOUT_RATIO - elif server_timeout >= http_timeout_seconds: - raise ValueError( - "config.request_timeout_seconds must be shorter than the HTTP " - f"timeout ({http_timeout_seconds:g} seconds)." - ) - - return config.model_copy(update={"request_timeout_seconds": server_timeout}) - - -class ScorerInvokeInputs(BaseModel): - """Input values sent to the Luna scorer invoke endpoint.""" - - query: JSONValue = "" - response: JSONValue = "" - ground_truth: JSONValue = None - tools: list[JSONObject] | None = None - - -class ScorerInvokeRecord(BaseModel): - """Caller-controlled subset of Orbit's partial runtime-record contract. - - Identity, ownership, persistence, and execution IDs are intentionally not - represented here. Orbit hydrates those fields from trusted server context. - """ - - type: ScorerInvokeRecordType - name: str | None = None - input: JSONValue = None - output: JSONValue = None - context: JSONObject | None = None - tools: list[JSONObject] | None = None - dataset_output: JSONValue = None - - -class GalileoExecutionContext(BaseModel): - """Authenticated organization and optional identity fields for Galileo scorer invocation.""" - - organization_id: str = Field(min_length=1) - user_id: str | None = Field(default=None, min_length=1) - project_id: str | None = Field(default=None, min_length=1) - run_id: str | None = Field(default=None, min_length=1) - - -class ScorerInvokeRequest(BaseModel): - """Request payload for Luna scorer invocation. - - Attributes: - scorer_id: Required scorer identifier. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - inputs: Selected scorer input values. - record: Optional Orbit-compatible structured runtime record. - config: Scorer-specific configuration, always emitted. - """ - - scorer_id: str = Field(min_length=1) - scorer_version_id: str | None = Field(default=None, min_length=1) - scorer_label: str | None = Field(default=None, min_length=1) - inputs: ScorerInvokeInputs - record: ScorerInvokeRecord | None = None - execution_context: GalileoExecutionContext | None = None - config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) - - @model_validator(mode="after") - def ensure_required_values(self) -> ScorerInvokeRequest: - if not (_has_value(self.inputs.query) or _has_value(self.inputs.response)): - raise ValueError("Either inputs.query or inputs.response must be set.") - return self - - def to_dict(self) -> JSONObject: - """Convert to the Luna scorer invoke request shape.""" - request = self.model_dump(mode="json", exclude_none=True) - if self.execution_context is not None: - request["execution_context"] = self.execution_context.model_dump(mode="json") - return request - - -def _orbit_record_from_step( - step: Step | None, - *, - selected_input: JSONValue, - selected_output: JSONValue, -) -> ScorerInvokeRecord | None: - """Translate a generic Agent Control step into Orbit's record contract. - - Selector-selected values remain the primary scorer input. When a selector - supplies one side, that value is written to both the legacy and structured - representations so Orbit's conflict validation cannot observe two meanings. - The complete step supplies the unselected side and additional record context. - - Unknown Agent Control step types intentionally fall back to the legacy - ``inputs`` contract. This keeps the open-source Step model extensible without - sending an invalid discriminator to Orbit. - - Args: - step: Complete Agent Control step, when contextual evaluation is used. - selected_input: Selector-selected value sent as ``inputs.query``. - selected_output: Selector-selected value sent as ``inputs.response``. - - Returns: - An Orbit-compatible record, or ``None`` for absent/unsupported steps. - """ - if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: - return None - - # The membership check above narrows the runtime value to Orbit's known - # discriminator set, but static type checkers cannot infer that relationship. - record_type = cast(ScorerInvokeRecordType, step.type) - return ScorerInvokeRecord( - type=record_type, - name=step.name, - input=selected_input if selected_input is not None else step.input, - output=selected_output if selected_output is not None else step.output, - context=step.context, - tools=step.tools, - dataset_output=step.ground_truth, - ) - - -class ScorerInvokeResponse(BaseModel): - """Response from Luna scorer invocation. - - Attributes: - scorer_label: Echoed scorer label, when returned. - score: Raw scorer value. - status: Invocation status. - execution_time: Execution time in seconds, when returned. - error_message: Error detail for non-success statuses. - """ - - scorer_label: str | None = None - score: JSONValue - status: str = "unknown" - execution_time: float | None = None - error_message: str | None = None - _raw_response: JSONObject = PrivateAttr(default_factory=dict) - - @property - def raw_response(self) -> JSONObject: - return self._raw_response - - @classmethod - def from_dict(cls, data: JSONObject) -> ScorerInvokeResponse: - """Create a response model from the Luna scorer invoke JSON object.""" - response = cls.model_validate( - data | {"execution_time": _as_float_or_none(data.get("execution_time"))} - ) - response._raw_response = data - return response - - -class GalileoLunaClient: - """Thin HTTP client for Galileo Luna scorer invocation. - - Environment Variables: - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. - GALILEO_LUNA_INVOKE_URL: Luna scorer invoke URL or service root (required). - GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify Luna invoke TLS. - AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. - GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry. - GALILEO_LUNA_MAX_CONNECTIONS: Maximum outbound HTTP connections. - GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS: Maximum idle pooled HTTP connections. - GALILEO_LUNA_CLIENT_POOL_SIZE: Number of outbound HTTP clients to rotate across. - """ - - def __init__( - self, - api_secret: str | None = None, - luna_invoke_url: str | None = None, - luna_invoke_ca_file: str | None = None, - ) -> None: - """Initialize the Galileo Luna client. - - Args: - api_secret: Internal JWT signing secret. If not provided, reads from - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. - luna_invoke_url: Luna scorer invoke URL or service root. If not provided, - reads from GALILEO_LUNA_INVOKE_URL. - luna_invoke_ca_file: Optional CA bundle used to verify Luna invoke TLS. If not - provided, reads from GALILEO_LUNA_INVOKE_CA_FILE, then - AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE. - - Raises: - ValueError: If the API secret, Luna invoke URL, CA bundle, or connection - tuning configuration is invalid. - """ - resolved_api_secret = ( - api_secret or os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") - ) - if not resolved_api_secret: - raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for Luna " - "scorer invocation. Set one as an environment variable or pass it " - "to the constructor." - ) - - resolved_luna_invoke_url = luna_invoke_url or os.getenv(LUNA_INVOKE_URL_ENV) - if resolved_luna_invoke_url is None or resolved_luna_invoke_url.strip() == "": - raise ValueError( - "GALILEO_LUNA_INVOKE_URL is required for Luna scorer invocation. " - "Set it as an environment variable or pass it to the constructor." - ) - - self.api_secret = resolved_api_secret - self.luna_invoke_url = _normalize_luna_invoke_url(resolved_luna_invoke_url) - self.luna_invoke_ca_file = ( - luna_invoke_ca_file - or os.getenv(LUNA_INVOKE_CA_FILE_ENV) - or os.getenv(AUTH_UPSTREAM_CA_FILE_ENV) - or "" - ).strip() or None - self._ssl_context = self._load_ssl_context(self.luna_invoke_ca_file) - self.keepalive_expiry_seconds = _load_float_env( - LUNA_KEEPALIVE_EXPIRY_ENV, DEFAULT_KEEPALIVE_EXPIRY_SECS - ) - self.max_connections = _load_int_env(LUNA_MAX_CONNECTIONS_ENV, DEFAULT_MAX_CONNECTIONS) - self.max_keepalive_connections = _load_int_env( - LUNA_MAX_KEEPALIVE_CONNECTIONS_ENV, DEFAULT_MAX_KEEPALIVE_CONNECTIONS - ) - self.client_pool_size = _load_int_env(LUNA_CLIENT_POOL_SIZE_ENV, DEFAULT_CLIENT_POOL_SIZE) - _validate_connection_config( - keepalive_expiry_seconds=self.keepalive_expiry_seconds, - max_connections=self.max_connections, - max_keepalive_connections=self.max_keepalive_connections, - client_pool_size=self.client_pool_size, - ) - self._client: httpx.AsyncClient | None = None - self._clients: list[httpx.AsyncClient] = [] - self._next_client_index = 0 - self._client_lock = Lock() - - @staticmethod - def _load_ssl_context(ca_file: str | None) -> ssl.SSLContext | None: - """Build a TLS verification context from a CA bundle path, if configured.""" - if ca_file is None: - return None - try: - return ssl.create_default_context(cafile=ca_file) - except (OSError, ssl.SSLError) as exc: - raise ValueError(f"Failed to load CA bundle from {ca_file!r}: {exc}") from exc - - def _create_client(self) -> httpx.AsyncClient: - """Create an HTTP client with the configured TLS and connection limits.""" - verify: ssl.SSLContext | bool = self._ssl_context if self._ssl_context is not None else True - return httpx.AsyncClient( - headers={"Content-Type": "application/json"}, - timeout=httpx.Timeout(DEFAULT_TIMEOUT_SECS), - limits=httpx.Limits( - max_connections=self.max_connections, - max_keepalive_connections=self.max_keepalive_connections, - keepalive_expiry=self.keepalive_expiry_seconds, - ), - verify=verify, - ) - - def _select_pooled_client(self) -> httpx.AsyncClient: - """Select the next pooled client while holding the client state lock.""" - client = self._clients[self._next_client_index % len(self._clients)] - self._next_client_index = (self._next_client_index + 1) % len(self._clients) - return client - - async def _get_client(self) -> httpx.AsyncClient: - """Get or create the next HTTP client.""" - async with self._client_lock: - self._clients = [client for client in self._clients if not client.is_closed] - - if self.client_pool_size == 1: - if self._client is not None and not self._client.is_closed: - return self._client - self._client = self._clients[0] if self._clients else self._create_client() - self._clients = [self._client] - return self._client - - self._client = None - while len(self._clients) < self.client_pool_size: - self._clients.append(self._create_client()) - - return self._select_pooled_client() - - def _endpoint_and_auth_header(self) -> tuple[str, str]: - token = _internal_auth_token(self.api_secret) - return self.luna_invoke_url, f"Bearer {token}" - - async def invoke( - self, - *, - scorer_id: str, - scorer_version_id: str | None = None, - scorer_label: str | None = None, - input: JSONValue = None, - output: JSONValue = None, - step: Step | None = None, - execution_context: GalileoExecutionContext | None = None, - config: ScorerInvokeConfig | JSONObject | None = None, - timeout: float = DEFAULT_TIMEOUT_SECS, - headers: dict[str, str] | None = None, - ) -> ScorerInvokeResponse: - """Invoke a Galileo Luna scorer. - - Args: - scorer_id: Required scorer identifier. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - input: Optional user/system prompt text. - output: Optional model response text. - step: Optional complete runtime step used for structured dual-write. - execution_context: Optional authenticated Galileo scorer context. - config: Optional Orbit-supported scorer invocation configuration. - timeout: Request timeout in seconds. - headers: Additional request headers. - - Returns: - Parsed scorer invocation response. - - Raises: - ValueError: If neither input nor output is provided, config contains - a field Orbit does not support, or the timeout ordering is invalid. - RuntimeError: If the API response is not a JSON object. - httpx.HTTPStatusError: If the Luna invoke endpoint returns an error status code. - httpx.RequestError: If the request fails before a response is received. - """ - if not (_has_value(input) or _has_value(output)): - raise ValueError("At least one of input or output must be provided.") - - # Accept dictionaries for source compatibility with the original client, - # but validate them against Orbit's authoritative allowlist locally. - invoke_config = ( - ScorerInvokeConfig.model_validate(config) - if config is not None - else ScorerInvokeConfig() - ) - invoke_config = _effective_scorer_timeout( - invoke_config, - http_timeout_seconds=timeout, - ) - request_body = ScorerInvokeRequest( - scorer_id=scorer_id, - scorer_version_id=scorer_version_id, - scorer_label=scorer_label, - inputs=ScorerInvokeInputs( - query="" if input is None else input, - response="" if output is None else output, - ground_truth=step.ground_truth if step is not None else None, - tools=step.tools if step is not None else None, - ), - record=_orbit_record_from_step( - step, - selected_input=input, - selected_output=output, - ), - execution_context=execution_context, - config=invoke_config, - ).to_dict() - - endpoint, auth_header = self._endpoint_and_auth_header() - request_headers = { - k: v for k, v in (headers or {}).items() if k.lower() not in _BLOCKED_REQUEST_HEADERS - } - request_headers["Authorization"] = auth_header - - logger.debug("[GalileoLunaClient] POST %s", endpoint) - logger.debug("[GalileoLunaClient] Request body: %s", request_body) - - try: - client = await self._get_client() - response = await client.post( - endpoint, - json=request_body, - headers=request_headers, - timeout=timeout, - ) - response.raise_for_status() - response_data = response.json() - if not isinstance(response_data, dict): - raise RuntimeError("Invalid response payload: not a JSON object") - - parsed = ScorerInvokeResponse.from_dict(response_data) - logger.debug("[GalileoLunaClient] Response: %s", parsed.raw_response) - return parsed - except httpx.HTTPStatusError as exc: - logger.error( - "[GalileoLunaClient] API error: %s - %s", - exc.response.status_code, - exc.response.text, - ) - raise - except httpx.RequestError as exc: - logger.error("[GalileoLunaClient] Request failed: %s", exc) - raise - - async def close(self) -> None: - """Close HTTP clients and release resources.""" - async with self._client_lock: - clients: list[httpx.AsyncClient] = [] - seen_client_ids: set[int] = set() - if self._client is not None: - clients.append(self._client) - seen_client_ids.add(id(self._client)) - self._client = None - for client in self._clients: - if id(client) not in seen_client_ids: - clients.append(client) - seen_client_ids.add(id(client)) - self._clients = [] - self._next_client_index = 0 - - for client in clients: - if not client.is_closed: - await client.aclose() - - async def __aenter__(self) -> GalileoLunaClient: - """Async context manager entry.""" - return self - async def __aexit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: - """Async context manager exit.""" - await self.close() +from agent_control_evaluator_galileo._shared.client import ( + DEFAULT_CLIENT_POOL_SIZE, + DEFAULT_KEEPALIVE_EXPIRY_SECS, + DEFAULT_MAX_CONNECTIONS, + DEFAULT_MAX_KEEPALIVE_CONNECTIONS, + DEFAULT_TIMEOUT_SECS, + GalileoExecutionContext, + GalileoScorerClient, + ScorerInvokeInputs, + ScorerInvokeRecord, + ScorerInvokeRequest, + ScorerInvokeResponse, + _as_float_or_none, + _effective_scorer_timeout, + _has_value, + _internal_auth_token, + _normalize_scorer_invoke_url, + _orbit_record_from_step, + _validate_connection_config, +) + +# Public alias: GalileoLunaClient is the named export users and tests reference. +GalileoLunaClient = GalileoScorerClient diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py index 0bc0fa37e..e3c0b6bcf 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py @@ -1,124 +1,24 @@ -"""Configuration model for direct Galileo Luna scorer evaluation.""" +"""Configuration for Galileo Luna scorer evaluator.""" -from __future__ import annotations +from agent_control_evaluator_galileo._shared.config import ( + BaseScorerEvaluatorConfig, + ScorerInvokeConfig, + coerce_number, +) -from typing import Literal +# Public aliases kept for backwards compatibility. +LunaOperator = BaseScorerEvaluatorConfig.model_fields["operator"].annotation +LunaPayloadField = BaseScorerEvaluatorConfig.model_fields["payload_field"].annotation -from agent_control_evaluators import EvaluatorConfig -from agent_control_models import JSONValue -from pydantic import BaseModel, ConfigDict, Field, model_validator -LunaOperator = Literal["gt", "gte", "lt", "lte", "eq", "ne", "contains", "any"] -LunaPayloadField = Literal["input", "output"] +class LunaEvaluatorConfig(BaseScorerEvaluatorConfig): + """Configuration for direct Luna scorer evaluation.""" -_NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) - -class ScorerInvokeConfig(BaseModel): - """Orbit-supported overrides for a synchronous scorer invocation. - - Orbit owns the Galileo scorer-invoke wire contract. Keeping this model - strict makes an unsupported option fail locally instead of producing a - less actionable HTTP 422 response from Runners. - - Attributes: - threshold: Legacy threshold accepted by Orbit. Agent Control still - applies its evaluator threshold locally. - score_threshold: Legacy score threshold accepted by Orbit. - request_timeout_seconds: Optional upper bound for scorer execution in - Orbit. The Agent Control HTTP and evaluator deadlines must remain - longer than this value. - """ - - model_config = ConfigDict(extra="forbid") - - threshold: float | None = None - score_threshold: float | None = None - request_timeout_seconds: float | None = Field(default=None, gt=0) - - -def coerce_number(value: JSONValue) -> float | None: - """Return a numeric value for JSON scalars that can be compared numerically.""" - if isinstance(value, bool) or value is None: - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return None - return None - - -class LunaEvaluatorConfig(EvaluatorConfig): - """Configuration for direct Luna scorer evaluation. - - Attributes: - scorer_id: Required scorer identifier for Luna scorer invocation. - scorer_version_id: Deprecated optional compatibility identifier. Orbit - currently invokes the scorer's current default version. - scorer_label: Optional display/metadata label. - threshold: Local threshold used by the evaluator for comparison. - operator: Local comparison operator. Numeric operators use threshold as a number. - scorer_config: Optional Orbit-supported scorer invocation config sent - as ``config``. - payload_field: Explicit scorer input side for scalar selected data. - timeout_ms: Request timeout in milliseconds. - """ - - scorer_id: str = Field( - min_length=1, - description="Required scorer identifier for Luna scorer invocation.", - ) - scorer_version_id: str | None = Field( - default=None, - min_length=1, - description=( - "Deprecated optional compatibility identifier. Orbit currently invokes " - "the scorer's current default version." - ), - ) - scorer_label: str | None = Field( - default=None, - min_length=1, - description="Optional display/metadata label.", - ) - threshold: JSONValue = Field( - default=0.5, - description="Local threshold used to decide whether the control matches.", - ) - operator: LunaOperator = Field( - default="gte", - description="Local comparison operator applied to the raw Luna score.", - ) - scorer_config: ScorerInvokeConfig | None = Field( - default=None, - alias="config", - serialization_alias="config", - description=( - "Optional Orbit-supported configuration sent to the Luna scorer invoke endpoint." - ), - ) - payload_field: LunaPayloadField = Field( - default="input", - description=( - "Which scorer input side to use when selector output is a scalar value. " - "Structured selected data with input/output keys overrides this setting." - ), - ) - timeout_ms: int = Field( - default=10000, - ge=1000, - le=60000, - description="Request timeout in milliseconds (1-60 seconds)", - ) - - @model_validator(mode="after") - def validate_threshold(self) -> LunaEvaluatorConfig: - """Validate threshold compatibility with the configured operator.""" - if self.operator in _NUMERIC_OPERATORS and coerce_number(self.threshold) is None: - raise ValueError(f"operator '{self.operator}' requires a numeric threshold") - if self.operator != "any" and self.threshold is None: - raise ValueError("threshold is required unless operator is 'any'") - return self +__all__ = [ + "LunaEvaluatorConfig", + "LunaOperator", + "LunaPayloadField", + "ScorerInvokeConfig", + "coerce_number", +] diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 14b58d884..c4ebc3d6f 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -1,21 +1,24 @@ -"""Direct Galileo Luna evaluator implementation.""" +"""Galileo Luna direct scorer evaluator.""" -from __future__ import annotations - -import json -import logging -import os +# ruff: noqa: F401 from importlib.metadata import PackageNotFoundError, version -from typing import Any - -import httpx -from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator -from agent_control_models import EvaluatorResult, JSONObject, JSONValue, Step -from .client import GalileoExecutionContext, GalileoLunaClient, ScorerInvokeResponse -from .config import LunaEvaluatorConfig, coerce_number +from agent_control_evaluators import EvaluatorMetadata, register_evaluator + +from agent_control_evaluator_galileo._shared.evaluator import ( + BaseGalileoScorerEvaluator, + _coerce_payload_text, + _confidence_from_score, + _contains, + _extract_dict_text, + _has_text, + _http_status_error_metadata, + _truncated_http_response_body, +) +from agent_control_evaluator_galileo.luna.client import GalileoLunaClient +from agent_control_evaluator_galileo.luna.config import LunaEvaluatorConfig -logger = logging.getLogger(__name__) +LUNA_AVAILABLE = True def _resolve_package_version() -> str: @@ -26,350 +29,19 @@ def _resolve_package_version() -> str: return "0.0.0.dev" -_PACKAGE_VERSION = _resolve_package_version() -LUNA_AVAILABLE = True -_HTTP_ERROR_BODY_LIMIT = 500 - - -def _coerce_payload_text(value: Any) -> str | None: - """Coerce selected data into scorer text without losing structured values.""" - if value is None: - return None - if isinstance(value, str): - return value - if isinstance(value, (int, float, bool)): - return str(value) - try: - return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) - except TypeError: - return str(value) - - -def _has_text(value: str | None) -> bool: - return value is not None and value.strip() != "" - - -def _extract_dict_text(data: dict[str, Any], key: str) -> str | None: - if key not in data: - return None - return _coerce_payload_text(data.get(key)) - - -def _contains(score: JSONValue, threshold: JSONValue) -> bool: - if threshold is None: - return False - if isinstance(score, str): - return str(threshold) in score - if isinstance(score, list): - return threshold in score - if isinstance(score, dict): - return threshold in score.values() - return False - - -def _confidence_from_score(score: JSONValue) -> float: - if isinstance(score, bool): - return 1.0 if score else 0.0 - number = coerce_number(score) - if number is not None and 0.0 <= number <= 1.0: - return number - return 1.0 - - -def _truncated_http_response_body(body: str) -> tuple[str, bool]: - if len(body) <= _HTTP_ERROR_BODY_LIMIT: - return body, False - return body[:_HTTP_ERROR_BODY_LIMIT], True - - -def _http_status_error_metadata(error: httpx.HTTPStatusError) -> dict[str, Any]: - metadata: dict[str, Any] = {} - - request = error.request - metadata["http_method"] = request.method - metadata["http_endpoint_path"] = request.url.path - - response = error.response - metadata["http_status_code"] = response.status_code - metadata["http_response_content_type"] = response.headers.get("content-type") - - body = response.text - if body: - metadata["http_response_body"], metadata["http_response_body_truncated"] = ( - _truncated_http_response_body(body) - ) - - return {key: value for key, value in metadata.items() if value is not None} - - @register_evaluator -class LunaEvaluator(Evaluator[LunaEvaluatorConfig]): +class LunaEvaluator(BaseGalileoScorerEvaluator): """Galileo Luna evaluator using the direct scorer invocation API.""" metadata = EvaluatorMetadata( name="galileo.luna", - version=_PACKAGE_VERSION, + version=_resolve_package_version(), description="Galileo Luna direct scorer evaluation", requires_api_key=True, timeout_ms=10000, ) config_model = LunaEvaluatorConfig - - @classmethod - def is_available(cls) -> bool: - """Check whether required runtime dependencies are available.""" - return LUNA_AVAILABLE - - def __init__(self, config: LunaEvaluatorConfig) -> None: - """Initialize the direct Luna evaluator. - - Args: - config: Validated LunaEvaluatorConfig instance. - - Raises: - ValueError: If neither GALILEO_API_SECRET_KEY nor GALILEO_API_SECRET is set. - """ - has_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") - if not has_secret: - raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for Luna " - "scorer invocation. Set one as an environment variable before using " - "galileo.luna." - ) - - super().__init__(config) - self._client = GalileoLunaClient() + _display_name = "Luna" def _get_client(self) -> GalileoLunaClient: - """Get the Galileo Luna client.""" - return self._client - - def _prepare_payload(self, data: Any) -> tuple[str | None, str | None]: - """Prepare scorer input/output fields from selected data.""" - if isinstance(data, dict): - input_text = _extract_dict_text(data, "input") - output_text = _extract_dict_text(data, "output") - if _has_text(input_text) or _has_text(output_text): - return input_text, output_text - - text = _coerce_payload_text(data) - if self.config.payload_field == "output": - return None, text - return text, None - - def _score_matches(self, score: JSONValue) -> bool: - """Apply the configured local threshold comparison to a raw Luna score.""" - operator = self.config.operator - threshold = self.config.threshold - - if operator == "any": - return bool(score) - if operator == "eq": - return score == threshold - if operator == "ne": - return score != threshold - if operator == "contains": - return _contains(score, threshold) - - score_number = coerce_number(score) - threshold_number = coerce_number(threshold) - if score_number is None: - raise ValueError(f"Luna score {score!r} is not numeric") - if threshold_number is None: - raise ValueError(f"Luna threshold {threshold!r} is not numeric") - - if operator == "gt": - return score_number > threshold_number - if operator == "gte": - return score_number >= threshold_number - if operator == "lt": - return score_number < threshold_number - if operator == "lte": - return score_number <= threshold_number - - raise ValueError(f"Unsupported Luna operator: {operator}") - - async def evaluate(self, data: Any) -> EvaluatorResult: - """Evaluate selected data with Galileo Luna direct scorer invocation. - - Args: - data: The data selected from the runtime step. - - Returns: - EvaluatorResult with local threshold decision and scorer metadata. - """ - return await self._evaluate(data, step=None) - - async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: - """Evaluate selected data while dual-writing the complete runtime step. - - Args: - data: Data selected by the configured control selector. - step: Complete runtime step for structured scorer context. - - Returns: - EvaluatorResult with local threshold decision and scorer metadata. - """ - return await self._evaluate(data, step=step) - - async def evaluate_with_extensions( - self, - data: Any, - step: Step, - extensions: JSONObject | None, - ) -> EvaluatorResult: - """Evaluate using opaque authenticated metadata supplied by Agent Control.""" - return await self._evaluate( - data, - step=step, - extensions=extensions, - ) - - @staticmethod - def _execution_context_from_extensions( - extensions: JSONObject | None, - ) -> GalileoExecutionContext | None: - """Translate trusted opaque auth metadata into Galileo's request context.""" - if extensions is None: - return None - - metadata = extensions.get("metadata") - metadata_obj = metadata if isinstance(metadata, dict) else {} - organization_id = extensions.get("namespace_key") - # The runtime envelope's caller_id is supplied by the authenticated - # principal and is the caller identity Galileo associates with this run. - user_id = extensions.get("caller_id") - project_id = metadata_obj.get("project_id") - target_type = extensions.get("target_type") - target_id = extensions.get("target_id") - run_id = ( - target_id - if target_type in ("log_stream", "agent_stream") - and isinstance(target_id, str) - and target_id - else None - ) - if not isinstance(organization_id, str) or not organization_id: - raise ValueError("Authenticated execution metadata is missing organization_id") - - return GalileoExecutionContext( - organization_id=organization_id, - user_id=user_id if isinstance(user_id, str) and user_id else None, - project_id=project_id if isinstance(project_id, str) and project_id else None, - run_id=run_id, - ) - - async def _evaluate( - self, - data: Any, - *, - step: Step | None, - extensions: JSONObject | None = None, - ) -> EvaluatorResult: - """Run a Luna evaluation with optional structured runtime context.""" - input_text, output_text = self._prepare_payload(data) - if not (_has_text(input_text) or _has_text(output_text)): - return EvaluatorResult( - matched=False, - confidence=1.0, - message="No data to score with Luna", - metadata=self._base_metadata(), - ) - - try: - execution_context = self._execution_context_from_extensions(extensions) - scorer_kwargs = self._scorer_kwargs() - if step is not None: - scorer_kwargs["step"] = step - if execution_context is not None: - scorer_kwargs["execution_context"] = execution_context - response = await self._get_client().invoke( - **scorer_kwargs, - input=input_text if _has_text(input_text) else None, - output=output_text if _has_text(output_text) else None, - config=self.config.scorer_config, - timeout=self.get_timeout_seconds(), - ) - - if response.status.lower() != "success": - message = response.error_message or f"Luna scorer status: {response.status}" - raise RuntimeError(message) - - matched = self._score_matches(response.score) - metadata = self._metadata(response) - operator = self.config.operator - threshold = self.config.threshold - state = "triggered" if matched else "not triggered" - return EvaluatorResult( - matched=matched, - confidence=_confidence_from_score(response.score), - message=( - f"Luna score {response.score!r} {operator} threshold " - f"{threshold!r}: control {state}." - ), - metadata=metadata, - ) - except Exception as exc: - logger.error("Luna evaluation error: %s", exc, exc_info=True) - return self._handle_error(exc) - - def _base_metadata(self) -> dict[str, Any]: - """Build result metadata without implying a requested version executed.""" - metadata: dict[str, Any] = {"scorer_id": self.config.scorer_id} - if self.config.scorer_version_id is not None: - metadata["requested_scorer_version_id"] = self.config.scorer_version_id - if self.config.scorer_label is not None: - metadata["scorer_label"] = self.config.scorer_label - return metadata - - def _scorer_kwargs(self) -> dict[str, Any]: - kwargs: dict[str, Any] = {"scorer_id": self.config.scorer_id} - if self.config.scorer_version_id is not None: - kwargs["scorer_version_id"] = self.config.scorer_version_id - if self.config.scorer_label is not None: - kwargs["scorer_label"] = self.config.scorer_label - return kwargs - - def _metadata( - self, - response: ScorerInvokeResponse, - ) -> dict[str, Any]: - metadata: dict[str, Any] = self._base_metadata() - echoed_label = response.scorer_label or self.config.scorer_label - if echoed_label is not None: - metadata["scorer_label"] = echoed_label - metadata.update( - { - "score": response.score, - "threshold": self.config.threshold, - "operator": self.config.operator, - "status": response.status, - "execution_time_seconds": response.execution_time, - "error_message": response.error_message, - } - ) - return metadata - - def _handle_error( - self, - error: Exception, - ) -> EvaluatorResult: - error_detail = str(error) - metadata: dict[str, Any] = { - **self._base_metadata(), - "error_type": type(error).__name__, - } - if isinstance(error, httpx.HTTPStatusError): - metadata.update(_http_status_error_metadata(error)) - - return EvaluatorResult( - matched=False, - confidence=0.0, - message=f"Luna evaluation error: {error_detail}", - metadata=metadata, - error=error_detail, - ) - - async def aclose(self) -> None: - """Close the underlying Galileo Luna client.""" - await self._client.close() + return self._client # type: ignore[return-value] From 2853024d41bc6df51b332ecff3941dc30b108621 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 5 Oct 2026 12:29:12 -0700 Subject: [PATCH 5/6] added logs --- .../_shared/evaluator.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py index a1f27ebea..7bc4f37b9 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py @@ -252,8 +252,11 @@ async def _evaluate( ) -> EvaluatorResult: """Run a scorer evaluation with optional structured runtime context.""" display = self._display_name + scorer_id = self.config.scorer_id + logger.info("[%s] Dispatching scorer evaluation: scorer_id=%s", display, scorer_id) input_text, output_text = self._prepare_payload(data) if not (_has_text(input_text) or _has_text(output_text)): + logger.info("[%s] Skipping scorer invocation: no data to score", display) return EvaluatorResult( matched=False, confidence=1.0, @@ -268,6 +271,12 @@ async def _evaluate( scorer_kwargs["step"] = step if execution_context is not None: scorer_kwargs["execution_context"] = execution_context + logger.info( + "[%s] Invoking scorer: scorer_id=%s timeout=%.1fs", + display, + scorer_id, + self.get_timeout_seconds(), + ) response = await self._get_client().invoke( **scorer_kwargs, input=input_text if _has_text(input_text) else None, @@ -287,6 +296,15 @@ async def _evaluate( operator = self.config.operator threshold = self.config.threshold state = "triggered" if matched else "not triggered" + logger.info( + "[%s] Scorer result: scorer_id=%s score=%r %s threshold=%r → %s", + display, + scorer_id, + response.score, + operator, + threshold, + state, + ) return EvaluatorResult( matched=matched, confidence=_confidence_from_score(response.score), @@ -297,7 +315,13 @@ async def _evaluate( metadata=metadata, ) except Exception as exc: - logger.error("%s scorer evaluation error: %s", display, exc, exc_info=True) + logger.error( + "[%s] Scorer evaluation error: scorer_id=%s error=%s", + display, + scorer_id, + exc, + exc_info=True, + ) return self._handle_error(exc) def _base_metadata(self) -> dict[str, Any]: From 2451af3b924801824b28c436a147f0baf3875f84 Mon Sep 17 00:00:00 2001 From: Wrisa Date: Mon, 5 Oct 2026 14:23:24 -0700 Subject: [PATCH 6/6] added identity log --- .../_shared/evaluator.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py index 7bc4f37b9..c260a07fc 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/_shared/evaluator.py @@ -236,10 +236,23 @@ def _execution_context_from_extensions( if not isinstance(organization_id, str) or not organization_id: raise ValueError("Authenticated execution metadata is missing organization_id") + resolved_user_id = user_id if isinstance(user_id, str) and user_id else None + resolved_project_id = project_id if isinstance(project_id, str) and project_id else None + raw_api_key = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") or "" + api_key_hint = f"...{raw_api_key[-4:]}" if len(raw_api_key) >= 4 else "***" + logger.info( + "[execution_context] caller_id=%r → user_id=%r org=%r project=%r run=%r api_key=%s", + user_id, + resolved_user_id, + organization_id, + resolved_project_id, + run_id, + api_key_hint, + ) return GalileoExecutionContext( organization_id=organization_id, - user_id=user_id if isinstance(user_id, str) and user_id else None, - project_id=project_id if isinstance(project_id, str) and project_id else None, + user_id=resolved_user_id, + project_id=resolved_project_id, run_id=run_id, )