fix(cutover): cluster-aware finalize detection + assert parked-backup contents in rollback tests

Addresses two later Baz review comments on PR #7645.

finalize.sh (severity high) — production is multi-replica, and finalize is the one
irreversible step, so resolve the parked-backup detection CLUSTER-WIDE via clusterAllReplicas
(mirroring exchange_and_wrap.sh's settle gate) instead of the connected node only: a name
present on some-but-not-all replicas means an ON CLUSTER DDL has not finished propagating, so
acting on one node's partial view could recycle/drop mid-transition — refuse loudly instead of
a silent "Nothing to finalize". Also guard the recycle destination: refuse (cluster-wide) if
`traces_local_v2` already exists before the TRUNCATE, since recycle renames the backup INTO
that name and RENAME will not overwrite — a stray shadow means a retry cutover started before
the rollback was finalized. Declined the broader schema/replica-path validation Baz suggested:
`traces_post_rollback_backup` is created only by the rollback RENAME (it IS the 000101 object
by construction), so validating engine/columns guards a manufactured state the code-style rule
says to avoid.

TracesLocalV2CutoverTest (severity low) — the stage-B/C rollback tests asserted the parked
backup only EXISTS, so a rename that produced an empty or wrong table would pass while losing
the successor data the parking exists to preserve. Assert the backup actually holds the
successor data (survivors present, post-cutover delete stays masked) and carries the successor
(non-Nullable end_time) schema. 10/10 green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: andrescrz

apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh

@@ -14,9 +14,14 @@
 #                                   the exact 000101 shadow — schema, codecs (000106/000107) and replica path — so the
 #                                   estate matches the applied Liquibase state and a retry starts from a clean shadow.
 # Both are `*_backup` names — retained until this script runs; the working `traces_local_v2` shadow is never detected as a
-# backup. This detects whichever parked table is present and never touches the live `traces` / `traces_local` shard. It
-# refuses if the live `traces` is empty while the backup is not (the live table may be unhealthy and the "backup" the only
-# copy), and if BOTH parked names exist (an ambiguous, unexpected state that a human must resolve).
+# backup. This detects whichever parked table is present and never touches the live `traces` / `traces_local` shard.
+# Detection is CLUSTER-WIDE (via clusterAllReplicas, like exchange_and_wrap.sh's settle gate): because finalize is the one
+# irreversible step and production is multi-replica, a name present on only SOME replicas means an ON CLUSTER DDL has not
+# finished propagating, so acting on the connected node's partial view could recycle/drop mid-transition — it refuses
+# loudly instead. It also refuses if the live `traces` is empty while the backup is not (the live table may be unhealthy
+# and the "backup" the only copy), if BOTH parked names exist (an ambiguous state a human must resolve), and — before a
+# recycle — if `traces_local_v2` already exists (recycle renames the backup INTO that name; a stray shadow means a retry
+# cutover started before the rollback was finalized).
 #
 # Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD).
 #
@@ -45,16 +50,39 @@ ch() {
     clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:finalize' --query "$1"
 }
 
-exists() {
-    ch "SELECT count() FROM system.tables WHERE database = '$DATABASE' AND name = '$1'"
+# Cluster-wide detection. finalize is the one irreversible step and production is multi-replica, so a table's presence is
+# resolved across ALL replicas (clusterAllReplicas, mirroring exchange_and_wrap.sh's settle gate), not just the connected
+# node. Resolve the cluster and its replica count once; a down replica makes clusterAllReplicas throw — correct here,
+# since finalizing against an estate we cannot fully see would be unsafe.
+CLUSTER="$(ch "SELECT getMacro('cluster')")"
+[[ -n "$CLUSTER" ]] || { echo "ERROR: could not resolve the '{cluster}' macro (getMacro('cluster') was empty)." >&2; exit 1; }
+REPLICAS="$(ch "SELECT count() FROM clusterAllReplicas('$CLUSTER', system.one)")"
+
+# Classify a table across the cluster: sets CLUSTER_HAS=1 if present on ALL replicas, 0 if on none, and refuses loudly on
+# a mixed (present on some) state — an unfinished ON CLUSTER propagation the connected-node view would hide. Call it
+# directly (NOT in "$(...)"), so its refuse-exit stops the whole script rather than only a subshell.
+CLUSTER_HAS=0
+classify() {
+    local n
+    n="$(ch "SELECT count() FROM clusterAllReplicas('$CLUSTER', system.tables) WHERE database = '$DATABASE' AND name = '$1'")"
+    if [[ "$n" == "0" ]]; then
+        CLUSTER_HAS=0
+    elif [[ "$n" == "$REPLICAS" ]]; then
+        CLUSTER_HAS=1
+    else
+        echo "ERROR: '$1' exists on $n of $REPLICAS replicas — an ON CLUSTER DDL has not finished propagating." >&2
+        echo "       Refusing to finalize a mid-transition cluster; let it settle (or fix the unfinished host), then re-run." >&2
+        exit 1
+    fi
 }
 
-[[ "$(exists traces)" != "0" ]] || { echo "ERROR: live 'traces' table not found in '$DATABASE'." >&2; exit 1; }
+classify traces
+[[ "$CLUSTER_HAS" == "1" ]] || { echo "ERROR: live 'traces' table not found on all replicas in '$DATABASE'." >&2; exit 1; }
 
 # Detect the parked backup by name: traces_pre_cutover_backup (post-successful-cutover) or traces_post_rollback_backup
 # (post-rollback). They never co-exist in a clean flow; if both are present the estate is ambiguous — refuse.
-HAS_PRECUTOVER="$([[ "$(exists traces_pre_cutover_backup)" != "0" ]] && echo 1 || echo 0)"
-HAS_POST_ROLLBACK="$([[ "$(exists traces_post_rollback_backup)" != "0" ]] && echo 1 || echo 0)"
+classify traces_pre_cutover_backup;   HAS_PRECUTOVER="$CLUSTER_HAS"
+classify traces_post_rollback_backup; HAS_POST_ROLLBACK="$CLUSTER_HAS"
 
 if [[ "$HAS_PRECUTOVER" == "1" && "$HAS_POST_ROLLBACK" == "1" ]]; then
     echo "ERROR: both 'traces_pre_cutover_backup' and 'traces_post_rollback_backup' exist — ambiguous state." >&2
@@ -94,6 +122,17 @@ if [[ "$BACKUP" == "traces_post_rollback_backup" ]]; then
     # naming a laggard that then converges via the DDL queue), NOT globally atomic. Both statements touch only the parked
     # backup / disposable shadow — never the live `traces` — so unlike the rollback promote and the wrap (which rename live
     # `traces`) the brief cross-replica skew is invisible to readers, and finalize needs no maintenance window.
+    #
+    # Guard the destination first: recycle renames the backup INTO `traces_local_v2`, and ClickHouse RENAME fails on an
+    # existing target. A stray `traces_local_v2` here means a retry cutover started before this rollback was finalized —
+    # refuse (cluster-wide) BEFORE truncating, so we fail early with a clear message instead of after the TRUNCATE.
+    classify traces_local_v2
+    if [[ "$CLUSTER_HAS" != "0" ]]; then
+        echo "ERROR: 'traces_local_v2' already exists — cannot recycle '$BACKUP' into it (RENAME will not overwrite)." >&2
+        echo "       This usually means a retry cutover began before the rollback was finalized. Resolve the estate" >&2
+        echo "       (inspect/drop 'traces_local_v2') before recycling." >&2
+        exit 1
+    fi
     if [[ "$CONFIRM" != "1" ]]; then
         echo "DRY RUN: would recycle $DATABASE.$BACKUP into an empty $DATABASE.traces_local_v2 (TRUNCATE + RENAME)."
         echo "         Re-run with --confirm."

apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java

@@ -496,6 +496,16 @@ void bufferedCutoverPreservesEveryDeletionAcrossExchange() {
         assertThat(tableExists("traces_post_rollback_backup"))
                 .as("rollback ends in the canonical state: successor data parked as traces_post_rollback_backup")
                 .isTrue();
+        assertThat(liveCount("traces_post_rollback_backup", postWrapDeleted, workspaceId))
+                .as("parked backup keeps the successor's post-wrap delete masked (not resurrected in the backup)")
+                .isZero();
+        assertThat(liveCount("traces_post_rollback_backup", idStrings(survivors.subList(1, survivors.size())),
+                workspaceId))
+                .as("parked backup actually holds the successor data (survivors), not an empty or wrong table")
+                .isEqualTo(survivors.size() - 1);
+        assertThat(columnType("traces_post_rollback_backup", "end_time"))
+                .as("parked backup carries the successor's non-Nullable schema, confirming the right table was parked")
+                .doesNotStartWith("Nullable");
         assertThat(tableExists("traces_local_v2"))
                 .as("the disposable shadow name is free after rollback (so stage A cannot truncate the backup)")
                 .isFalse();
@@ -591,6 +601,19 @@ void rollbackAfterExchangeSwapsBackWithoutResurrectingDeletes() {
         assertThat(tableExists("traces_post_rollback_backup"))
                 .as("canonical state: successor parked as traces_post_rollback_backup")
                 .isTrue();
+        assertThat(liveCount("traces_post_rollback_backup", postCutoverDeleted, workspaceId))
+                .as("parked backup keeps the post-cutover delete masked (not resurrected in the backup)")
+                .isZero();
+        assertThat(liveCount("traces_post_rollback_backup", idStrings(windowDeleted), workspaceId))
+                .as("parked backup does not resurrect window deletes")
+                .isZero();
+        assertThat(liveCount("traces_post_rollback_backup", idStrings(survivors.subList(1, survivors.size())),
+                workspaceId))
+                .as("parked backup actually holds the successor data (survivors), not an empty or wrong table")
+                .isEqualTo(survivors.size() - 1);
+        assertThat(columnType("traces_post_rollback_backup", "end_time"))
+                .as("parked backup carries the successor's non-Nullable schema")
+                .doesNotStartWith("Nullable");
         assertThat(tableExists("traces_local_v2"))
                 .as("the disposable shadow name is free after rollback (so stage A cannot truncate the backup)")
                 .isFalse();