Skip to content

Commit 5a892ce

Browse files
miguelgrcmiguelgrc
authored
[OPIK-6741] [BE][FE] feat: add EvalTriggerScope to evaluation rules (#7439)
* [OPIK-6741] [BE][FE] feat: add EvalTriggerScope to evaluation rules Add a trigger_scope field (PRODUCTION / EXPERIMENT / BOTH) to online evaluation rules so they can fire on experiment traces, production traces, or both. Existing rules default to PRODUCTION (backward compatible). Clients that omit trigger_scope get PRODUCTION silently. BE: New EvalTriggerScope enum, DB migration 000090, field threaded through all model/DAO/service/mapper layers, OnlineScoringSampler now lets experiment traces through and filters per-evaluator based on trigger scope. Null-safe defaulting in service and sampler. FE: ToggleGroup control on the v2 rule create/edit dialog for selecting trigger scope, wired into form schema and API payload. Tests: 3 dedicated TriggerScope unit tests, TracesUpdated path coverage for EXPERIMENT source, updated existing sampler and integration tests. * fix(evaluators): null-safe triggerScope default + review fixes - Add withTriggerScope to AutomationRuleEvaluatorModel interface + 6 records - Default null triggerScope to PRODUCTION in service save path - Add --comment to migration 000090 - Fix DisabledRulesTest by setting explicit triggerScope on test rules - Add triggerScope to update test expected builder * Revision 2: hide trigger scope toggle for thread/span rules Thread and span evaluators have separate scoring paths that don't check triggerScope, so the toggle is a no-op for those rule types. Hide it in the FE to avoid user confusion.
1 parent ddeb1e1 commit 5a892ce

38 files changed

Lines changed: 355 additions & 51 deletions

File tree

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public sealed interface AutomationRule permits AutomationRuleEvaluator {
3535
AutomationRuleAction getAction();
3636
float getSamplingRate();
3737
boolean isEnabled();
38+
EvalTriggerScope getTriggerScope();
3839

3940
Instant getCreatedAt();
4041
String getCreatedBy();

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluator.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ public abstract sealed class AutomationRuleEvaluator<T, E extends Filter> implem
8383
@Builder.Default
8484
private final boolean enabled = true;
8585

86+
@JsonView({View.Public.class, View.Write.class})
87+
@Schema(description = "Controls whether the rule fires on production traces, experiment traces, or both. Defaults to 'production' if omitted.")
88+
@Builder.Default
89+
private final EvalTriggerScope triggerScope = EvalTriggerScope.PRODUCTION;
90+
8691
@JsonIgnore
8792
@Builder.Default
8893
private final List<E> filters = List.of();

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorLlmAsJudge.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public LlmAsJudgeCode(LlmAsJudgeModelParameters model, List<LlmAsJudgeMessage> m
5151
}
5252

5353
@ConstructorProperties({"id", "projectId", "projectName", "projects", "projectIds", "name", "samplingRate",
54-
"enabled", "filters", "code",
54+
"enabled", "triggerScope", "filters", "code",
5555
"createdAt",
5656
"createdBy",
5757
"lastUpdatedAt", "lastUpdatedBy"})
@@ -61,10 +61,12 @@ public AutomationRuleEvaluatorLlmAsJudge(UUID id, UUID projectId, String project
6161
@NotBlank String name,
6262
float samplingRate,
6363
boolean enabled,
64+
EvalTriggerScope triggerScope,
6465
List<TraceFilter> filters,
6566
@NotNull LlmAsJudgeCode code, Instant createdAt, String createdBy, Instant lastUpdatedAt,
6667
String lastUpdatedBy) {
67-
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, filters, code,
68+
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, triggerScope, filters,
69+
code,
6870
createdAt, createdBy,
6971
lastUpdatedAt,
7072
lastUpdatedBy);

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorSpanLlmAsJudge.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public record SpanLlmAsJudgeCode(
4343
}
4444

4545
@ConstructorProperties({"id", "projectId", "projectName", "projects", "projectIds", "name", "samplingRate",
46-
"enabled", "filters", "code",
46+
"enabled", "triggerScope", "filters", "code",
4747
"createdAt",
4848
"createdBy",
4949
"lastUpdatedAt", "lastUpdatedBy"})
@@ -53,10 +53,12 @@ public AutomationRuleEvaluatorSpanLlmAsJudge(UUID id, UUID projectId, String pro
5353
@NotBlank String name,
5454
float samplingRate,
5555
boolean enabled,
56+
EvalTriggerScope triggerScope,
5657
List<SpanFilter> filters,
5758
@NotNull SpanLlmAsJudgeCode code, Instant createdAt, String createdBy, Instant lastUpdatedAt,
5859
String lastUpdatedBy) {
59-
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, filters, code,
60+
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, triggerScope, filters,
61+
code,
6062
createdAt, createdBy,
6163
lastUpdatedAt,
6264
lastUpdatedBy);

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorSpanUserDefinedMetricPython.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,20 @@ public record SpanUserDefinedMetricPythonCode(
4242
}
4343

4444
@ConstructorProperties({"id", "projectId", "projectName", "projects", "projectIds", "name", "samplingRate",
45-
"enabled", "filters", "code",
45+
"enabled", "triggerScope", "filters", "code",
4646
"createdAt",
4747
"createdBy",
4848
"lastUpdatedAt", "lastUpdatedBy"})
4949
public AutomationRuleEvaluatorSpanUserDefinedMetricPython(UUID id, UUID projectId, String projectName,
5050
SortedSet<ProjectReference> projects,
5151
Set<UUID> projectIds,
52-
@NotBlank String name, float samplingRate, boolean enabled, List<SpanFilter> filters,
52+
@NotBlank String name, float samplingRate, boolean enabled, EvalTriggerScope triggerScope,
53+
List<SpanFilter> filters,
5354
@NotNull SpanUserDefinedMetricPythonCode code,
5455
Instant createdAt,
5556
String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
56-
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, filters, code,
57+
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, triggerScope, filters,
58+
code,
5759
createdAt, createdBy,
5860
lastUpdatedAt,
5961
lastUpdatedBy);

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorTraceThreadLlmAsJudge.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,20 +54,22 @@ public TraceThreadLlmAsJudgeCode(LlmAsJudgeModelParameters model, List<LlmAsJudg
5454
}
5555

5656
@ConstructorProperties({"id", "projectId", "projectName", "projects", "projectIds", "name", "samplingRate",
57-
"enabled", "filters", "code",
57+
"enabled", "triggerScope", "filters", "code",
5858
"createdAt",
5959
"createdBy",
6060
"lastUpdatedAt", "lastUpdatedBy"})
6161
public AutomationRuleEvaluatorTraceThreadLlmAsJudge(UUID id, UUID projectId, String projectName,
6262
SortedSet<ProjectReference> projects,
6363
Set<UUID> projectIds,
6464
@NotBlank String name,
65-
float samplingRate, boolean enabled, List<TraceThreadFilter> filters,
65+
float samplingRate, boolean enabled, EvalTriggerScope triggerScope,
66+
List<TraceThreadFilter> filters,
6667
@NotNull TraceThreadLlmAsJudgeCode code,
6768
Instant createdAt,
6869
String createdBy,
6970
Instant lastUpdatedAt, String lastUpdatedBy) {
70-
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, filters, code,
71+
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, triggerScope, filters,
72+
code,
7173
createdAt, createdBy,
7274
lastUpdatedAt,
7375
lastUpdatedBy);

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorTraceThreadUserDefinedMetricPython.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,19 @@ public record TraceThreadUserDefinedMetricPythonCode(
4141
}
4242

4343
@ConstructorProperties({"id", "projectId", "projectName", "projects", "projectIds", "name", "samplingRate",
44-
"enabled", "filters", "code",
44+
"enabled", "triggerScope", "filters", "code",
4545
"createdAt",
4646
"createdBy",
4747
"lastUpdatedAt", "lastUpdatedBy"})
4848
public AutomationRuleEvaluatorTraceThreadUserDefinedMetricPython(UUID id, UUID projectId, String projectName,
4949
SortedSet<ProjectReference> projects,
5050
Set<UUID> projectIds,
51-
@NotBlank String name, float samplingRate, boolean enabled, List<TraceThreadFilter> filters,
51+
@NotBlank String name, float samplingRate, boolean enabled, EvalTriggerScope triggerScope,
52+
List<TraceThreadFilter> filters,
5253
@NotNull TraceThreadUserDefinedMetricPythonCode code,
5354
Instant createdAt, String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
54-
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, filters, code,
55+
super(id, projectId, projectName, projects, projectIds, name, samplingRate, enabled, triggerScope, filters,
56+
code,
5557
createdAt, createdBy,
5658
lastUpdatedAt,
5759
lastUpdatedBy);

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorUpdate.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ public abstract sealed class AutomationRuleEvaluatorUpdate<T, E extends Filter>
5656
@Builder.Default
5757
private final boolean enabled = true;
5858

59+
@Builder.Default
60+
private final EvalTriggerScope triggerScope = EvalTriggerScope.PRODUCTION;
61+
5962
@JsonIgnore
6063
@Builder.Default
6164
private final List<E> filters = List.of();

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorUpdateLlmAsJudge.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ public final class AutomationRuleEvaluatorUpdateLlmAsJudge
2727
extends
2828
AutomationRuleEvaluatorUpdate<LlmAsJudgeCode, TraceFilter> {
2929

30-
@ConstructorProperties({"name", "samplingRate", "enabled", "filters", "code", "projectId", "projectIds"})
30+
@ConstructorProperties({"name", "samplingRate", "enabled", "triggerScope", "filters", "code", "projectId",
31+
"projectIds"})
3132
public AutomationRuleEvaluatorUpdateLlmAsJudge(
32-
@NotBlank String name, float samplingRate, boolean enabled, List<TraceFilter> filters,
33+
@NotBlank String name, float samplingRate, boolean enabled, EvalTriggerScope triggerScope,
34+
List<TraceFilter> filters,
3335
@NotNull LlmAsJudgeCode code,
3436
UUID projectId,
3537
Set<UUID> projectIds) {
36-
super(name, samplingRate, enabled, filters, code, projectId, projectIds);
38+
super(name, samplingRate, enabled, triggerScope, filters, code, projectId, projectIds);
3739
}
3840

3941
/**

‎apps/opik-backend/src/main/java/com/comet/opik/api/evaluators/AutomationRuleEvaluatorUpdateSpanLlmAsJudge.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ public final class AutomationRuleEvaluatorUpdateSpanLlmAsJudge
2727
extends
2828
AutomationRuleEvaluatorUpdate<SpanLlmAsJudgeCode, SpanFilter> {
2929

30-
@ConstructorProperties({"name", "samplingRate", "enabled", "filters", "code", "projectId", "projectIds"})
30+
@ConstructorProperties({"name", "samplingRate", "enabled", "triggerScope", "filters", "code", "projectId",
31+
"projectIds"})
3132
public AutomationRuleEvaluatorUpdateSpanLlmAsJudge(
32-
@NotBlank String name, float samplingRate, boolean enabled, List<SpanFilter> filters,
33+
@NotBlank String name, float samplingRate, boolean enabled, EvalTriggerScope triggerScope,
34+
List<SpanFilter> filters,
3335
@NotNull SpanLlmAsJudgeCode code,
3436
UUID projectId,
3537
Set<UUID> projectIds) {
36-
super(name, samplingRate, enabled, filters, code, projectId, projectIds);
38+
super(name, samplingRate, enabled, triggerScope, filters, code, projectId, projectIds);
3739
}
3840

3941
/**

0 commit comments

Comments
 (0)