Add LLM evaluator

Author: wrisa

evaluators/contrib/README.md

@@ -2,7 +2,7 @@
 
 Contributed evaluators and templates for extending Agent Control.
 
-- `galileo/` — Luna evaluator integration
+- `galileo/` — Galileo evaluator integrations (Luna, LLM)
 - `template/` — Starter template for adding new evaluators
 
 Full guide: https://docs.agentcontrol.dev/concepts/evaluators/custom-evaluators

evaluators/contrib/galileo/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "agent-control-evaluator-galileo"
 version = "8.11.0"
-description = "Galileo Luna evaluator for agent-control"
+description = "Galileo evaluators (Luna, LLM-as-judge) for agent-control"
 readme = "README.md"
 requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
@@ -25,6 +25,7 @@ dev = [
 
 [project.entry-points."agent_control.evaluators"]
 "galileo.luna" = "agent_control_evaluator_galileo.luna:LunaEvaluator"
+"galileo.llm" = "agent_control_evaluator_galileo.llm:LlmEvaluator"
 
 [build-system]
 requires = ["hatchling"]

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py

@@ -4,6 +4,7 @@
 
 Available evaluators:
     - galileo.luna: Galileo Luna direct scorer evaluation
+    - galileo.llm:  Galileo LLM-as-judge direct scorer evaluation
 
 Installation:
     pip install agent-control-evaluator-galileo
@@ -19,6 +20,13 @@
 except PackageNotFoundError:
     __version__ = "0.0.0.dev"
 
+from agent_control_evaluator_galileo.llm import (
+    LLM_AVAILABLE,
+    GalileoLLMClient,
+    LlmEvaluator,
+    LlmEvaluatorConfig,
+    LlmOperator,
+)
 from agent_control_evaluator_galileo.luna import (
     LUNA_AVAILABLE,
     GalileoLunaClient,
@@ -41,6 +49,7 @@
 )
 
 __all__ = [
+    # luna
     "GalileoLunaClient",
     "ScorerInvokeRequest",
     "ScorerInvokeConfig",
@@ -50,6 +59,13 @@
     "LunaEvaluatorConfig",
     "LunaOperator",
     "LUNA_AVAILABLE",
+    # llm
+    "GalileoLLMClient",
+    "LlmEvaluator",
+    "LlmEvaluatorConfig",
+    "LlmOperator",
+    "LLM_AVAILABLE",
+    # records
     "GalileoRecord",
     "GalileoRecordNormalizer",
     "RecordFactoryError",

evaluators/contrib/galileo/src/agent_control_evaluator_galileo/llm/__init__.py

@@ -0,0 +1,30 @@
+"""Galileo LLM-as-judge direct scorer evaluator."""
+
+from agent_control_evaluator_galileo.llm.client import (
+    GalileoExecutionContext,
+    GalileoLLMClient,
+    ScorerInvokeInputs,
+    ScorerInvokeRecord,
+    ScorerInvokeRequest,
+    ScorerInvokeResponse,
+)
+from agent_control_evaluator_galileo.llm.config import (
+    LlmEvaluatorConfig,
+    LlmOperator,
+    ScorerInvokeConfig,
+)
+from agent_control_evaluator_galileo.llm.evaluator import LLM_AVAILABLE, LlmEvaluator
+
+__all__ = [
+    "GalileoLLMClient",
+    "GalileoExecutionContext",
+    "ScorerInvokeInputs",
+    "ScorerInvokeConfig",
+    "ScorerInvokeRecord",
+    "ScorerInvokeRequest",
+    "ScorerInvokeResponse",
+    "LlmEvaluatorConfig",
+    "LlmOperator",
+    "LlmEvaluator",
+    "LLM_AVAILABLE",
+]