comet-ml
diff --git a/‎apps/opik-backend/config.yml‎
Lines changed: 7 additions & 0 deletions b/‎apps/opik-backend/config.yml‎
Lines changed: 7 additions & 0 deletions
diff --git a/‎apps/opik-backend/src/main/java/com/comet/opik/api/error/InvalidUUIDExceptionMapper.java‎
Lines changed: 7 additions & 17 deletions b/‎apps/opik-backend/src/main/java/com/comet/opik/api/error/InvalidUUIDExceptionMapper.java‎
Lines changed: 7 additions & 17 deletions
diff --git a/‎apps/opik-backend/src/main/java/com/comet/opik/domain/IdGenerator.java‎
Lines changed: 49 additions & 13 deletions b/‎apps/opik-backend/src/main/java/com/comet/opik/domain/IdGenerator.java‎
Lines changed: 49 additions & 13 deletions
diff --git a/‎apps/opik-backend/src/main/java/com/comet/opik/domain/SpanService.java‎
Lines changed: 14 additions & 12 deletions b/‎apps/opik-backend/src/main/java/com/comet/opik/domain/SpanService.java‎
Lines changed: 14 additions & 12 deletions
diff --git a/‎apps/opik-backend/src/main/java/com/comet/opik/domain/TraceService.java‎
Lines changed: 13 additions & 2 deletions b/‎apps/opik-backend/src/main/java/com/comet/opik/domain/TraceService.java‎
Lines changed: 13 additions & 2 deletions
diff --git a/‎apps/opik-backend/src/main/java/com/comet/opik/infrastructure/UuidValidationConfig.java‎
Lines changed: 8 additions & 0 deletions b/‎apps/opik-backend/src/main/java/com/comet/opik/infrastructure/UuidValidationConfig.java‎
Lines changed: 8 additions & 0 deletions
@@ -145,6 +145,13 @@ uuidValidation:
   #  Description: Operational kill-switch. When false, ids are not checked against the window (the
   #   window is still validated to be a sane value).
   enabled: ${UUID_VALIDATION_ENABLED:-false}
+  #  Default: false
+  #  Description: Audit (shadow / log-only) mode. Only takes effect when enabled is true. When true,
+  #   out-of-window ids are counted (opik.ingestion.uuid_v7.rejected, tagged by workspace) and logged
+  #   but NOT rejected, so offenders surface without breaking ingestion. When false, out-of-window ids
+  #   are rejected with HTTP 400. Effective mode: enabled=false -> disabled; enabled=true & auditOnly=true
+  #   -> audit; enabled=true & auditOnly=false -> reject.
+  auditOnly: ${UUID_VALIDATION_AUDIT_ONLY:-false}
   #  Default: 24h
   #  Description: Validation window (between 12h and 45d). Writes whose `id` is a UUIDv7 with an
   #   embedded timestamp more than this far in the past or future are rejected.
 
@@ -1,10 +1,7 @@
 package com.comet.opik.api.error;
 
+import com.comet.opik.infrastructure.metrics.UuidValidationMetrics;
 import io.dropwizard.jersey.errors.ErrorMessage;
-import io.opentelemetry.api.GlobalOpenTelemetry;
-import io.opentelemetry.api.common.AttributeKey;
-import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.metrics.LongCounter;
 import jakarta.inject.Inject;
 import jakarta.inject.Provider;
 import jakarta.ws.rs.Path;
@@ -21,8 +18,10 @@
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 
 /**
- * Maps {@link InvalidUUIDException} to HTTP 400 and records the reject-rate metric.
- * Tags each rejection with the matched route.
+ * Maps {@link InvalidUUIDException} to HTTP 400 and records the reject-rate metric, tagged with the
+ * matched route and {@code mode=reject}. The counter itself lives in {@link UuidValidationMetrics}
+ * (the central owner of {@code opik.ingestion.uuid_v7.rejected}); this mapper only supplies the route
+ * and delegates.
  *
  * <p>{@link ResourceInfo} is request-scoped, so it is obtained lazily through a {@link Provider}
  * (injecting it directly would fail to construct this singleton outside a request). The lookup resolves
@@ -32,26 +31,17 @@
 @RequiredArgsConstructor(onConstructor_ = @Inject)
 public class InvalidUUIDExceptionMapper implements ExceptionMapper<InvalidUUIDException> {
 
-    private static final String METRIC_NAMESPACE = "opik.ingestion";
     private static final String UNKNOWN_ROUTE = "unknown";
 
-    private static final AttributeKey<String> HTTP_ROUTE_KEY = AttributeKey.stringKey("http_route");
-    private static final AttributeKey<String> REASON_KEY = AttributeKey.stringKey("reason");
-
-    private static final LongCounter REJECTED_COUNTER = GlobalOpenTelemetry.get().getMeter(METRIC_NAMESPACE)
-            .counterBuilder("%s.uuid_v7.rejected".formatted(METRIC_NAMESPACE))
-            .setDescription("Number of writes rejected because the id failed UUIDv7 ingestion validation")
-            .build();
-
     private final Provider<ResourceInfo> resourceInfo;
+    private final UuidValidationMetrics uuidValidationMetrics;
 
     @Override
     public Response toResponse(@NonNull InvalidUUIDException exception) {
         var httpRoute = getHttpRoute();
         log.info("Rejected ingestion id, httpRoute: '{}', reason: '{}', error message: '{}'",
                 httpRoute, exception.getReason().getValue(), exception.getMessage());
-        REJECTED_COUNTER.add(1,
-                Attributes.of(HTTP_ROUTE_KEY, httpRoute, REASON_KEY, exception.getReason().getValue()));
+        uuidValidationMetrics.recordReject(exception.getReason().getValue(), httpRoute);
         // Force JSON: ingestion endpoints negotiate other content types (e.g. the OTel endpoint uses protobuf),
         // which have no writer for the error entity — without this the 400 fails to serialize and surfaces as a 500
         return Response.status(BAD_REQUEST)
 
@@ -2,7 +2,9 @@
 
 import com.comet.opik.api.error.InvalidUUIDException;
 import com.comet.opik.api.error.InvalidUUIDException.Reason;
+import com.comet.opik.infrastructure.auth.RequestContext;
 import com.comet.opik.infrastructure.db.UuidV7TimestampValidator;
+import com.comet.opik.infrastructure.metrics.ErrorMetricsResolver;
 import com.fasterxml.uuid.Generators;
 import com.fasterxml.uuid.impl.TimeBasedEpochGenerator;
 import com.google.inject.ImplementedBy;
@@ -11,6 +13,7 @@
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import reactor.core.publisher.Mono;
+import reactor.util.context.ContextView;
 
 import java.time.Instant;
 import java.util.UUID;
@@ -25,11 +28,12 @@ public interface IdGenerator {
     UUID getTimeOrderedEpoch(long epochMilli);
 
     /**
-     * Validates an ingested {@code id}: it must be a version 7 UUID ({@link #validateVersion(UUID, String)})
-     * whose embedded timestamp is within the configured ingestion window.
-     * Rejects with HTTP 400. Encapsulates both data-quality checks behind one call.
+     * Validates an ingested {@code id}: it must be a version 7 UUID
+     * ({@link #validateVersion(UUID, String)}) whose embedded timestamp is within the configured
+     * ingestion window, throwing {@link InvalidUUIDException} otherwise. {@code workspaceId} attributes
+     * the source workspace for observability; pass {@link ErrorMetricsResolver#UNKNOWN} when unavailable.
      */
-    void validateId(UUID id, String resource);
+    void validateId(UUID id, String resource, String workspaceId);
 
     Mono<UUID> validateIdAsync(UUID id, String resource);
 
@@ -46,6 +50,13 @@ public interface IdGenerator {
      */
     void validateIdNotInFuture(UUID id, String resource);
 
+    /**
+     * Workspace-attributed overload of {@link #validateIdNotInFuture(UUID, String)}: callers that know the
+     * request workspace pass it so the audit metric is attributed. The 2-arg form defaults to
+     * {@link ErrorMetricsResolver#UNKNOWN} for callers that don't carry a workspace.
+     */
+    void validateIdNotInFuture(UUID id, String resource, String workspaceId);
+
     Mono<UUID> validateIdNotInFutureAsync(UUID id, String resource);
 
     /**
@@ -54,6 +65,9 @@ public interface IdGenerator {
      */
     void validateIdNotInFutureIfPresent(UUID id, String resource);
 
+    /** Workspace-attributed overload of {@link #validateIdNotInFutureIfPresent(UUID, String)}. */
+    void validateIdNotInFutureIfPresent(UUID id, String resource, String workspaceId);
+
     Mono<UUID> validateIdNotInFutureIfPresentAsync(UUID id, String resource);
 
     static Mono<UUID> validateVersionAsync(@NonNull UUID id, String resource) {
@@ -93,40 +107,62 @@ public UUID getTimeOrderedEpoch(long epochMilli) {
     }
 
     @Override
-    public void validateId(@NonNull UUID id, String resource) {
+    public void validateId(@NonNull UUID id, String resource, String workspaceId) {
         IdGenerator.validateVersion(id, resource);
-        uuidV7TimestampValidator.validate(id);
+        uuidV7TimestampValidator.validate(id, resource, workspaceId);
     }
 
     @Override
     public Mono<UUID> validateIdAsync(@NonNull UUID id, String resource) {
-        return Mono.fromCallable(() -> {
-            validateId(id, resource);
-            return id;
+        return Mono.deferContextual(ctx -> {
+            validateId(id, resource, workspaceId(ctx));
+            return Mono.just(id);
         });
     }
 
     @Override
     public void validateIdNotInFuture(@NonNull UUID id, String resource) {
+        // Callers that don't carry a workspace (most config-entity references) default to UNKNOWN.
+        validateIdNotInFuture(id, resource, ErrorMetricsResolver.UNKNOWN);
+    }
+
+    @Override
+    public void validateIdNotInFuture(@NonNull UUID id, String resource, String workspaceId) {
         IdGenerator.validateVersion(id, resource);
-        uuidV7TimestampValidator.validateNotInFuture(id);
+        uuidV7TimestampValidator.validateNotInFuture(id, resource, workspaceId);
     }
 
     @Override
     public Mono<UUID> validateIdNotInFutureAsync(@NonNull UUID id, String resource) {
-        return Mono.fromCallable(() -> {
-            validateIdNotInFuture(id, resource);
-            return id;
+        return Mono.deferContextual(ctx -> {
+            validateIdNotInFuture(id, resource, workspaceId(ctx));
+            return Mono.just(id);
         });
     }
 
+    /**
+     * Reads the {@code workspace_id} from the reactive context (the async ingestion paths carry it
+     * there, not in a request-scoped thread-local), falling back to {@link ErrorMetricsResolver#UNKNOWN}
+     * so the audit metric always has a value.
+     */
+    private static String workspaceId(ContextView ctx) {
+        return ctx.getOrDefault(RequestContext.WORKSPACE_ID, ErrorMetricsResolver.UNKNOWN);
+    }
+
     @Override
     public void validateIdNotInFutureIfPresent(UUID id, String resource) {
         if (id != null) {
             validateIdNotInFuture(id, resource);
         }
     }
 
+    @Override
+    public void validateIdNotInFutureIfPresent(UUID id, String resource, String workspaceId) {
+        if (id != null) {
+            validateIdNotInFuture(id, resource, workspaceId);
+        }
+    }
+
     @Override
     public Mono<UUID> validateIdNotInFutureIfPresentAsync(UUID id, String resource) {
         return id == null ? Mono.empty() : validateIdNotInFutureAsync(id, resource);
 
@@ -393,15 +393,6 @@ public Mono<Long> create(@NonNull SpanBatch batch) {
 
         List<Span> dedupedSpans = dedupSpans(batch.spans());
 
-        // Fail fast on invalid ids BEFORE any side effect below (auto-stripped attachment deletion, project
-        // creation), so a rejected batch never mutates state.
-        dedupedSpans.forEach(span -> {
-            if (span.id() != null) {
-                idGenerator.validateId(span.id(), SPAN_KEY);
-            }
-            validateSpanReferences(span.traceId(), span.parentSpanId());
-        });
-
         List<String> projectNames = dedupedSpans
                 .stream()
                 .map(Span::projectName)
@@ -419,7 +410,19 @@ public Mono<Long> create(@NonNull SpanBatch batch) {
                 .filter(Objects::nonNull)
                 .collect(Collectors.toSet());
 
-        return attachmentService.deleteAutoStrippedAttachments(SPAN, spanIds)
+        // Fail fast on invalid ids BEFORE any side effect below (auto-stripped attachment deletion, project
+        // creation), so a rejected batch never mutates state. Runs inside deferContextual so the audit
+        // metric can attribute the batch's own ids to the request workspace.
+        return Mono.deferContextual(validationCtx -> {
+            String validationWorkspaceId = validationCtx.get(RequestContext.WORKSPACE_ID);
+            dedupedSpans.forEach(span -> {
+                if (span.id() != null) {
+                    idGenerator.validateId(span.id(), SPAN_KEY, validationWorkspaceId);
+                }
+                validateSpanReferences(span.traceId(), span.parentSpanId());
+            });
+            return attachmentService.deleteAutoStrippedAttachments(SPAN, spanIds);
+        })
                 .then(Mono.deferContextual(ctx -> {
                     String workspaceId = ctx.get(RequestContext.WORKSPACE_ID);
                     String workspaceName = ctx.getOrDefault(RequestContext.WORKSPACE_NAME, "");
@@ -500,9 +503,8 @@ private List<Span> bindSpanToProjectAndId(List<Span> spans, List<Project> projec
                         throw new IllegalStateException("Project not found: %s".formatted(span.projectName()));
                     }
 
+                    // Ids are already validated up-front in create(SpanBatch); generated ids are inherently valid.
                     UUID id = span.id() == null ? idGenerator.generateId() : span.id();
-                    idGenerator.validateId(id, SPAN_KEY);
-                    // trace/parent references are validated up front in create(SpanBatch) before side effects.
 
                     return span.toBuilder().id(id).projectId(project.id()).build();
                 })
 
@@ -191,7 +191,18 @@ public Mono<Long> create(TraceBatch batch) {
                 .filter(Objects::nonNull)
                 .collect(Collectors.toSet());
 
-        return attachmentService.deleteAutoStrippedAttachments(EntityType.TRACE, traceIds)
+        // Fail fast on invalid ids BEFORE any side effect below (auto-stripped attachment deletion, project
+        // creation), so a rejected batch never mutates state. Runs inside deferContextual so the audit
+        // metric can attribute the batch's own ids to the request workspace.
+        return Mono.deferContextual(validationCtx -> {
+            String validationWorkspaceId = validationCtx.get(RequestContext.WORKSPACE_ID);
+            dedupedTraces.forEach(trace -> {
+                if (trace.id() != null) {
+                    idGenerator.validateId(trace.id(), TRACE_KEY, validationWorkspaceId);
+                }
+            });
+            return attachmentService.deleteAutoStrippedAttachments(EntityType.TRACE, traceIds);
+        })
                 .then(Mono.deferContextual(ctx -> {
                     String workspaceId = ctx.get(RequestContext.WORKSPACE_ID);
                     String workspaceName = ctx.getOrDefault(RequestContext.WORKSPACE_NAME, "");
@@ -251,8 +262,8 @@ private List<Trace> bindTraceToProjectAndId(List<Trace> traces, List<Project> pr
                     String projectName = WorkspaceUtils.getProjectName(trace.projectName());
                     Project project = projectPerName.get(projectName);
 
+                    // Ids are already validated up-front in create(TraceBatch); generated ids are inherently valid.
                     UUID id = trace.id() == null ? idGenerator.generateId() : trace.id();
-                    idGenerator.validateId(id, TRACE_KEY);
 
                     return trace.toBuilder().id(id).projectId(project.id()).projectName(project.name()).build();
                 })
 
@@ -14,9 +14,17 @@
  * <p>{@code enabled} is an operational kill-switch: when {@code false}, ids are not checked against
  * {@code window}. {@code window} bounds the embedded-timestamp distance from now, so a misbehaving
  * client can't land a row in a far-future partition.
+ *
+ * <p>{@code auditOnly} adds a third, shadow state on top of the {@code enabled} switch. It only takes
+ * effect when {@code enabled} is {@code true}: instead of rejecting out-of-window ids, the validator
+ * records the reject-rate metric (tagged by workspace) and logs, but lets the write through. This
+ * surfaces offending clients in real time without breaking ingestion. The effective mode is:
+ * {@code enabled=false} → disabled (no-op); {@code enabled=true, auditOnly=true} → audit (count + log,
+ * no reject); {@code enabled=true, auditOnly=false} → reject (HTTP 400).
  */
 @Builder(toBuilder = true)
 public record UuidValidationConfig(
         boolean enabled,
+        boolean auditOnly,
         @NotNull @MinDuration(value = 12, unit = TimeUnit.HOURS) @MaxDuration(value = 45, unit = TimeUnit.DAYS) Duration window) {
 }