Skip to content

Commit f33df58

Browse files
abhinav-galileoabhinav-galileo
committed
refactor(evaluators)!: standardize naming conventions and remove legacy API
BREAKING CHANGE: `parse_evaluator_ref()` removed, use `parse_evaluator_ref_full()` or `is_agent_scoped()` instead. Evaluator naming conventions: - Built-in: "regex", "list", "json", "sql" (no namespace) - External: "galileo/luna2" (slash separator) - Agent-scoped: "my-agent:custom" (colon separator) Changes: - Rename galileo-luna2 → galileo/luna2 throughout codebase - Add ParsedEvaluatorRef dataclass with type detection - Remove deprecated parse_evaluator_ref() tuple API - Migrate endpoints to use parse_evaluator_ref_full() and is_agent_scoped() - Standardize XEvaluator + XEvaluatorConfig naming in docs - Fix pre-existing lint issues (import sorting, Union syntax, unused imports)
1 parent 8eeaf28 commit f33df58

23 files changed

Lines changed: 572 additions & 251 deletions

File tree

‎CONTRIBUTING.md‎

Lines changed: 302 additions & 104 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ Controls are defined via the API or dashboard. Each control specifies what to ch
185185
"scope": { "step_types": ["llm"], "stages": ["pre"] },
186186
"selector": { "path": "input" },
187187
"evaluator": {
188-
"name": "galileo-luna2",
188+
"name": "galileo/luna2",
189189
"config": {
190190
"metric": "input_toxicity",
191191
"operator": "gt",

‎docs/OVERVIEW.md‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ Flexible value matching with multiple modes and logic options.
236236

237237
---
238238

239-
### 3. Luna-2 Evaluator (`galileo-luna2`)
239+
### 3. Luna-2 Evaluator (`galileo/luna2`)
240240

241241
AI-powered detection using Galileo's Luna-2 small language models. Provides real-time, low-latency evaluation for complex patterns that can't be caught with regex or lists.
242242

@@ -267,7 +267,7 @@ AI-powered detection using Galileo's Luna-2 small language models. Provides real
267267
```json
268268
// Block toxic inputs (score > 0.5)
269269
{
270-
"name": "galileo-luna2",
270+
"name": "galileo/luna2",
271271
"config": {
272272
"metric": "input_toxicity",
273273
"operator": "gt",
@@ -278,7 +278,7 @@ AI-powered detection using Galileo's Luna-2 small language models. Provides real
278278

279279
// Block prompt injection attempts
280280
{
281-
"name": "galileo-luna2",
281+
"name": "galileo/luna2",
282282
"config": {
283283
"metric": "prompt_injection",
284284
"operator": "gt",
@@ -289,7 +289,7 @@ AI-powered detection using Galileo's Luna-2 small language models. Provides real
289289

290290
// Flag potential hallucinations (warn but allow)
291291
{
292-
"name": "galileo-luna2",
292+
"name": "galileo/luna2",
293293
"config": {
294294
"metric": "hallucination",
295295
"operator": "gt",
@@ -299,7 +299,7 @@ AI-powered detection using Galileo's Luna-2 small language models. Provides real
299299

300300
// Using a central stage (pre-defined server-side rules)
301301
{
302-
"name": "galileo-luna2",
302+
"name": "galileo/luna2",
303303
"config": {
304304
"stage_type": "central",
305305
"stage_name": "production-safety",
@@ -434,23 +434,28 @@ Every evaluator implements the `Evaluator` base class:
434434

435435
```python
436436
from typing import Any
437-
from pydantic import BaseModel
437+
438438
from agent_control_models import EvaluatorResult
439-
from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator
439+
from agent_control_evaluators import (
440+
Evaluator,
441+
EvaluatorConfig,
442+
EvaluatorMetadata,
443+
register_evaluator,
444+
)
440445

441446

442-
class MyEvaluatorConfig(BaseModel):
447+
class MyEvaluatorConfig(EvaluatorConfig):
443448
"""Configuration schema for your evaluator."""
444449
threshold: float = 0.5
445450
custom_option: str = "default"
446451

447452

448453
@register_evaluator
449-
class MyCustomEvaluator(Evaluator[MyEvaluatorConfig]):
454+
class MyEvaluator(Evaluator[MyEvaluatorConfig]):
450455
"""Your custom evaluator."""
451456

452457
metadata = EvaluatorMetadata(
453-
name="my-custom-evaluator",
458+
name="my-evaluator",
454459
version="1.0.0",
455460
description="Detects custom patterns using proprietary logic",
456461
requires_api_key=True, # Set to True if you need credentials
@@ -527,7 +532,7 @@ except ImportError:
527532
AVAILABLE = False
528533

529534
@register_evaluator
530-
class MyEvaluator(Evaluator[MyConfig]):
535+
class MyEvaluator(Evaluator[MyEvaluatorConfig]):
531536
@classmethod
532537
def is_available(cls) -> bool:
533538
return AVAILABLE
@@ -552,7 +557,7 @@ Here's how a partner might integrate their content moderation API:
552557

553558
```python
554559
@register_evaluator
555-
class ContentModerationEvaluator(Evaluator[ContentModConfig]):
560+
class ContentModerationEvaluator(Evaluator[ContentModerationEvaluatorConfig]):
556561
"""Integration with Acme Content Moderation API."""
557562

558563
metadata = EvaluatorMetadata(
@@ -562,9 +567,9 @@ class ContentModerationEvaluator(Evaluator[ContentModConfig]):
562567
requires_api_key=True,
563568
timeout_ms=3000,
564569
)
565-
config_model = ContentModConfig
570+
config_model = ContentModerationEvaluatorConfig
566571

567-
def __init__(self, config: ContentModConfig) -> None:
572+
def __init__(self, config: ContentModerationEvaluatorConfig) -> None:
568573
super().__init__(config)
569574
self.client = AcmeClient(api_key=os.getenv("ACME_API_KEY"))
570575

‎docs/REFERENCE.md‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ Flexible value matching with multiple modes and logic options.
304304

305305
AI-powered detection using Galileo's Luna-2 small language models. Provides real-time, low-latency evaluation for complex patterns that can't be caught with regex or lists.
306306

307-
**Evaluator name**: `galileo-luna2`
307+
**Evaluator name**: `galileo/luna2`
308308

309309
**Installation**: Luna-2 requires an optional dependency:
310310

@@ -348,7 +348,7 @@ pip install agent-control-evaluators[luna2]
348348
```json
349349
// Block toxic inputs (score > 0.5)
350350
{
351-
"name": "galileo-luna2",
351+
"name": "galileo/luna2",
352352
"config": {
353353
"metric": "input_toxicity",
354354
"operator": "gt",
@@ -359,7 +359,7 @@ pip install agent-control-evaluators[luna2]
359359

360360
// Block prompt injection attempts
361361
{
362-
"name": "galileo-luna2",
362+
"name": "galileo/luna2",
363363
"config": {
364364
"metric": "prompt_injection",
365365
"operator": "gt",
@@ -370,7 +370,7 @@ pip install agent-control-evaluators[luna2]
370370

371371
// Flag potential hallucinations (warn but allow)
372372
{
373-
"name": "galileo-luna2",
373+
"name": "galileo/luna2",
374374
"config": {
375375
"metric": "hallucination",
376376
"operator": "gt",
@@ -380,7 +380,7 @@ pip install agent-control-evaluators[luna2]
380380

381381
// Using central stage (pre-defined in Galileo)
382382
{
383-
"name": "galileo-luna2",
383+
"name": "galileo/luna2",
384384
"config": {
385385
"stage_type": "central",
386386
"stage_name": "production-safety",
@@ -401,23 +401,28 @@ You can create custom evaluators to extend Agent Control with your own detection
401401

402402
```python
403403
from typing import Any
404-
from pydantic import BaseModel
404+
405405
from agent_control_models import EvaluatorResult
406-
from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator
406+
from agent_control_evaluators import (
407+
Evaluator,
408+
EvaluatorConfig,
409+
EvaluatorMetadata,
410+
register_evaluator,
411+
)
407412

408413

409-
class MyEvaluatorConfig(BaseModel):
414+
class MyEvaluatorConfig(EvaluatorConfig):
410415
"""Configuration schema for your evaluator."""
411416
threshold: float = 0.5
412417
custom_option: str = "default"
413418

414419

415420
@register_evaluator
416-
class MyCustomEvaluator(Evaluator[MyEvaluatorConfig]):
421+
class MyEvaluator(Evaluator[MyEvaluatorConfig]):
417422
"""Your custom evaluator."""
418423

419424
metadata = EvaluatorMetadata(
420-
name="my-custom-evaluator",
425+
name="my-evaluator",
421426
version="1.0.0",
422427
description="Detects custom patterns using proprietary logic",
423428
requires_api_key=True,
@@ -454,14 +459,14 @@ class MyCustomEvaluator(Evaluator[MyEvaluatorConfig]):
454459

455460
```toml
456461
[project.entry-points."agent_control.evaluators"]
457-
my-evaluator = "my_package.evaluator:MyCustomEvaluator"
462+
my-evaluator = "my_package.evaluator:MyEvaluator"
458463
```
459464

460465
**Optional Dependencies**: Override `is_available()` if your evaluator has optional dependencies:
461466

462467
```python
463468
@register_evaluator
464-
class MyEvaluator(Evaluator[MyConfig]):
469+
class MyEvaluator(Evaluator[MyEvaluatorConfig]):
465470
@classmethod
466471
def is_available(cls) -> bool:
467472
try:
@@ -881,6 +886,6 @@ make alembic-upgrade
881886
4. Verify the metric name is valid
882887
5. Check `on_error` setting if failures are silently allowed
883888

884-
**Evaluator Not Found**: If `galileo-luna2` doesn't appear in `list_evaluators()`:
889+
**Evaluator Not Found**: If `galileo/luna2` doesn't appear in `list_evaluators()`:
885890
- Verify `httpx` is installed (Luna-2's `is_available()` returns `False` without it)
886891
- Check server logs for evaluator discovery messages

‎evaluators/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ Evaluator implementations for agent-control.
99
- **json** - JSON schema validation
1010
- **sql** - SQL query validation using sqlglot
1111

12-
## Optional Evaluators
12+
## External Evaluators
1313

14-
- **galileo-luna2** - Galileo Luna-2 integration (requires `luna2` extra)
14+
- **galileo/luna2** - Galileo Luna-2 integration (requires `luna2` extra)
1515

1616
## Installation
1717

‎evaluators/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ regex = "agent_control_evaluators.regex:RegexEvaluator"
2424
list = "agent_control_evaluators.list:ListEvaluator"
2525
json = "agent_control_evaluators.json:JSONEvaluator"
2626
sql = "agent_control_evaluators.sql:SQLEvaluator"
27-
galileo-luna2 = "agent_control_evaluators.galileo_luna2:Luna2Evaluator"
27+
"galileo/luna2" = "agent_control_evaluators.galileo_luna2:Luna2Evaluator"
2828

2929
[build-system]
3030
requires = ["hatchling"]

‎evaluators/src/agent_control_evaluators/__init__.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,20 @@
44
Built-in evaluators (regex, list, json, sql) are registered automatically on import.
55
66
Available evaluators:
7-
- regex: Regular expression matching (built-in)
8-
- list: List-based value matching (built-in)
9-
- json: JSON validation (built-in)
10-
- sql: SQL query validation (built-in)
11-
- galileo-luna2: Galileo Luna-2 runtime protection (pip install agent-control-evaluators[luna2])
7+
Built-in (no namespace):
8+
- regex: Regular expression matching
9+
- list: List-based value matching
10+
- json: JSON validation
11+
- sql: SQL query validation
12+
13+
External (provider/name format):
14+
- galileo/luna2: Galileo Luna-2 runtime protection
15+
(pip install agent-control-evaluators[luna2])
16+
17+
Naming convention:
18+
- Built-in: "regex", "list", "json", "sql"
19+
- External: "provider/name" (e.g., "galileo/luna2")
20+
- Agent-scoped: "agent:name" (custom code deployed with agent)
1221
1322
Custom evaluators are Evaluator classes deployed with the engine.
1423
Their schemas are registered via initAgent for validation purposes.

‎evaluators/src/agent_control_evaluators/_base.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class EvaluatorMetadata:
4343
"""Metadata about an evaluator.
4444
4545
Attributes:
46-
name: Unique evaluator name (e.g., "regex", "galileo_luna2")
46+
name: Unique evaluator name (e.g., "regex", "galileo/luna2")
4747
version: Evaluator version string
4848
description: Human-readable description
4949
requires_api_key: Whether the evaluator requires an API key
@@ -57,7 +57,7 @@ class EvaluatorMetadata:
5757
timeout_ms: int = 10000
5858

5959

60-
class Evaluator(ABC, Generic[ConfigT]):
60+
class Evaluator(ABC, Generic[ConfigT]): # noqa: UP046 - need Python 3.10 compat
6161
"""Base class for all evaluators (built-in, external, or custom).
6262
6363
All evaluators follow the same pattern:

‎evaluators/src/agent_control_evaluators/_factory.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@
1111
from agent_control_evaluators._discovery import list_evaluators
1212

1313
if TYPE_CHECKING:
14-
from agent_control_evaluators._base import Evaluator
1514
from agent_control_models import EvaluatorSpec
1615

16+
from agent_control_evaluators._base import Evaluator
17+
1718
logger = logging.getLogger(__name__)
1819

1920
# Configuration

‎evaluators/src/agent_control_evaluators/galileo_luna2/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@
2626
"LUNA2_AVAILABLE",
2727
]
2828

29-
# Export client classes when available
29+
# Export client classes when available (added to __all__ below)
3030
if LUNA2_AVAILABLE:
31-
from .client import (
31+
from .client import ( # noqa: F401
3232
GalileoProtectClient,
3333
PassthroughAction,
3434
Payload,

0 commit comments

Comments
 (0)