fix(cli/scoring): sort speaker ids so diarization speaker mapping is deterministic (#923)

## Summary

Fixes #922 — same nondeterminism class as #911/#921, in the diarization
scorer.

`computeSpeakerMapping` built its speaker index orders with
`Array(predicted.keys)` / `Array(groundTruth.keys)`. Swift dictionary
iteration order is per-instance random (the hash seed incorporates the
storage allocation — re-confirmed empirically on Swift 6.2.3), and both
assignment paths (the subset DP and the greedy fallback) tie-break by
first-encountered index via strict `>` comparisons. The confusion matrix
rounds overlaps to integer milliseconds, so exact ties are realistic —
and whenever two pairings tie, the winning speaker mapping (and JER)
depended on random key enumeration order.

## Evidence

Standalone harness on the scenario in the new test (two predicted
speakers each overlapping one ground-truth speaker by exactly 4.75 s
after the 0.25 s collar): the pre-fix code produced **2 distinct speaker
mappings within a single 200-call process** (3/3 runs), and fails the
new test's predicate in **20/20 processes**. With the fix: 1 mapping,
always (`right → spk_a`, pinned in the test).

## Fixes (commit 2 adds the full sweep from adversarial review)

1. `computeSpeakerMapping`: sort both key arrays — ties resolve by the
solver's fixed index-order rule over lexicographically sorted speaker
ids; unambiguous assignments unchanged.
2. `offlineMetrics`: the `correctlyAssigned`, `jaccardScores`, and
unmapped-pred loops accumulated Doubles in dictionary order. Addition is
not associative, and both DER and JER pass through near-zero
cancellation (`max(0, overlapSpeech - correctlyAssigned)`, `1.0 -
averageJaccard`) where a ulp-level reorder survives the Float cast —
with 3+ speakers, DER/JER were still not bit-deterministic. All three
loops now iterate in sorted key order.
3. Streaming path (`DiarizationBenchmark.calculateStreamingMetrics`):
`overlapsByGtSpeaker.max(by: { $0.value < $1.value })` kept the
first-encountered element among tied overlaps in random dictionary order
— same class. Ties now break by smaller speaker id.

## Tests

`DiarizationSpeakerMappingTests`: tied-overlap stability across 50 calls
(each call regroups segments into fresh dictionaries, each with an
independent per-allocation key order — the unfixed code fails this 20/20
processes), the exact tied winner pinned so silent tie-break changes
fail loudly, plus an unambiguous-mapping correctness check.

Author: Alex-Wengg

Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift

@@ -939,6 +939,18 @@ enum StreamDiarizationBenchmark {
         }
     }
 
+    /// Deterministic winner among per-speaker overlap totals: largest
+    /// overlap, ties broken by smaller speaker id (dictionary iteration
+    /// order is per-instance random; issue #922).
+    static func bestOverlapMatch(
+        _ overlapsBySpeaker: [String: Float]
+    ) -> (speakerId: String, overlap: Float)? {
+        overlapsBySpeaker.max(by: {
+            if $0.value != $1.value { return $0.value < $1.value }
+            return $0.key > $1.key
+        }).map { ($0.key, $0.value) }
+    }
+
     /// Calculate DER metrics with first-occurrence mapping for streaming evaluation
     private static func calculateStreamingMetrics(
         predicted: [TimedSpeakerSegment],
@@ -985,7 +997,7 @@ enum StreamDiarizationBenchmark {
             }
 
             // Find the GT speaker with most overlap
-            if let (bestMatch, bestOverlap) = overlapsByGtSpeaker.max(by: { $0.value < $1.value }),
+            if let (bestMatch, bestOverlap) = bestOverlapMatch(overlapsByGtSpeaker),
                 bestOverlap > 0.5
             {  // Require at least 0.5s total overlap
                 firstOccurrenceMap[predSegment.speakerId] = bestMatch

Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift

@@ -186,8 +186,11 @@ enum DiarizationMetricsCalculator {
             groundTruth: groundTruthBySpeaker
         )
 
+        // Dictionary iteration order is per-instance random, so accumulate in
+        // sorted order — Double addition is not associative, and near-zero
+        // confusion/JER residues are sensitive to the summation order.
         var correctlyAssigned = 0.0
-        for (predId, truthId) in speakerMapping {
+        for (predId, truthId) in speakerMapping.sorted(by: { $0.key < $1.key }) {
             if let predSegments = predictedBySpeaker[predId],
                 let truthSegments = groundTruthBySpeaker[truthId]
             {
@@ -205,7 +208,7 @@ enum DiarizationMetricsCalculator {
         var jaccardScores: [Double] = []
         let inverseMapping = Dictionary(uniqueKeysWithValues: speakerMapping.map { ($0.value, $0.key) })
 
-        for (truthId, truthSegments) in groundTruthBySpeaker {
+        for (truthId, truthSegments) in groundTruthBySpeaker.sorted(by: { $0.key < $1.key }) {
             let matchedPred = inverseMapping[truthId]
             let predictedSegmentsForSpeaker = matchedPred.flatMap { predictedBySpeaker[$0] } ?? []
             let intersection = overlapDuration(predictedSegmentsForSpeaker, truthSegments)
@@ -215,7 +218,8 @@ enum DiarizationMetricsCalculator {
             }
         }
 
-        for (predId, predSegments) in predictedBySpeaker where speakerMapping[predId] == nil {
+        for (predId, predSegments) in predictedBySpeaker.sorted(by: { $0.key < $1.key })
+        where speakerMapping[predId] == nil {
             if unionDuration(predSegments) > 0 {
                 jaccardScores.append(0.0)
             }
@@ -230,7 +234,7 @@ enum DiarizationMetricsCalculator {
         }
 
         if let logger = logger {
-            logger.debug("🎯 Offline mapping: \(speakerMapping)")
+            logger.debug("🎯 Offline mapping: \(speakerMapping.sorted(by: { $0.key < $1.key }))")
             let formattedDer = String(format: "%.1f", der)
             let formattedMiss = String(format: "%.1f", missRate)
             let formattedFalseAlarm = String(format: "%.1f", falseAlarmRate)
@@ -593,8 +597,11 @@ enum DiarizationMetricsCalculator {
     ) -> [String: String] {
         guard !predicted.isEmpty, !groundTruth.isEmpty else { return [:] }
 
-        let predictedIds = Array(predicted.keys)
-        let groundTruthIds = Array(groundTruth.keys)
+        // Sorted so assignment tie-breaks are deterministic: dictionary key
+        // order is per-instance random, and the solvers keep the
+        // first-encountered winner among tied overlaps (issue #922).
+        let predictedIds = predicted.keys.sorted()
+        let groundTruthIds = groundTruth.keys.sorted()
 
         var confusionMatrix = Array(
             repeating: Array(repeating: 0, count: predictedIds.count),

Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift

@@ -0,0 +1,112 @@
+#if os(macOS)
+import XCTest
+
+@testable import FluidAudio
+@testable import FluidAudioCLI
+
+/// Tests for the diarization scorer's speaker mapping (issue #922).
+///
+/// `computeSpeakerMapping` builds its speaker index orders from dictionary
+/// keys, and the assignment solvers keep the first-encountered winner among
+/// tied overlaps. Before the key arrays were sorted, per-instance random
+/// dictionary order made the winning mapping — and with it JER — drift
+/// between runs on tied confusion-matrix entries.
+final class DiarizationSpeakerMappingTests: XCTestCase {
+
+    private func segment(_ speaker: String, _ start: Float, _ end: Float) -> TimedSpeakerSegment {
+        TimedSpeakerSegment(
+            speakerId: speaker, embedding: [], startTimeSeconds: start, endTimeSeconds: end,
+            qualityScore: 1.0)
+    }
+
+    /// Two predicted speakers tie exactly on overlap with one ground-truth
+    /// speaker (4.75 s each after the 0.25 s collar). Only one can win the
+    /// assignment; the winner must not depend on dictionary key order.
+    func testTiedOverlapsMapDeterministically() {
+        let groundTruth = [segment("spk_a", 0, 10)]
+        let predicted = [
+            segment("left", 0, 5),
+            segment("right", 5, 10),
+        ]
+
+        let first = DiarizationMetricsCalculator.offlineMetrics(
+            predicted: predicted, groundTruth: groundTruth)
+        // Pinned winner: the assignment DP keeps the skip-branch result on
+        // ties, so with sorted ids "right" wins. A silent tie-break change
+        // would shift recorded benchmark numbers; this catches it.
+        XCTAssertEqual(first.speakerMapping, ["right": "spk_a"])
+
+        // Every call regroups segments into fresh dictionaries, so each
+        // iteration samples a new per-instance key order.
+        for _ in 0..<50 {
+            let metrics = DiarizationMetricsCalculator.offlineMetrics(
+                predicted: predicted, groundTruth: groundTruth)
+            XCTAssertEqual(metrics.speakerMapping, first.speakerMapping)
+            XCTAssertEqual(metrics.jer, first.jer)
+            XCTAssertEqual(metrics.der, first.der)
+        }
+    }
+
+    /// With 3+ speakers, der/jer accumulate Doubles across several speakers;
+    /// the sums must be byte-stable call to call now that the accumulation
+    /// loops iterate in sorted key order.
+    func testMultiSpeakerMetricsAreStableAcrossCalls() {
+        let groundTruth = [
+            segment("gt_a", 0, 10),
+            segment("gt_b", 11, 20),
+            segment("gt_c", 21, 30),
+        ]
+        let predicted = [
+            segment("p1", 0, 9),
+            segment("p2", 11.5, 19),
+            segment("p3", 21.5, 28),
+        ]
+
+        let first = DiarizationMetricsCalculator.offlineMetrics(
+            predicted: predicted, groundTruth: groundTruth)
+        XCTAssertEqual(first.speakerMapping, ["p1": "gt_a", "p2": "gt_b", "p3": "gt_c"])
+
+        for _ in 0..<50 {
+            let metrics = DiarizationMetricsCalculator.offlineMetrics(
+                predicted: predicted, groundTruth: groundTruth)
+            XCTAssertEqual(metrics.speakerMapping, first.speakerMapping)
+            XCTAssertEqual(metrics.der, first.der)
+            XCTAssertEqual(metrics.jer, first.jer)
+            XCTAssertEqual(metrics.speakerErrorRate, first.speakerErrorRate)
+        }
+    }
+
+    /// The streaming scorer's overlap winner must break ties by smaller
+    /// speaker id, not by whichever tied entry a fresh dictionary happens to
+    /// enumerate first.
+    func testStreamingTieBreakPicksSmallestSpeakerId() {
+        for _ in 0..<100 {
+            let overlaps: [String: Float] = ["spk_b": 2.0, "spk_a": 2.0, "spk_c": 1.5]
+            let best = StreamDiarizationBenchmark.bestOverlapMatch(overlaps)
+            XCTAssertEqual(best?.speakerId, "spk_a")
+            XCTAssertEqual(best?.overlap, 2.0)
+        }
+
+        let unambiguous = StreamDiarizationBenchmark.bestOverlapMatch(["spk_a": 1.0, "spk_b": 3.0])
+        XCTAssertEqual(unambiguous?.speakerId, "spk_b")
+        XCTAssertNil(StreamDiarizationBenchmark.bestOverlapMatch([:]))
+    }
+
+    /// Unambiguous overlaps must still produce the correct mapping after the
+    /// key arrays are sorted.
+    func testUnambiguousMappingIsCorrect() {
+        let groundTruth = [
+            segment("alice", 0, 10),
+            segment("bob", 11, 20),
+        ]
+        let predicted = [
+            segment("s1", 0, 9),
+            segment("s2", 11.5, 20),
+        ]
+
+        let metrics = DiarizationMetricsCalculator.offlineMetrics(
+            predicted: predicted, groundTruth: groundTruth)
+        XCTAssertEqual(metrics.speakerMapping, ["s1": "alice", "s2": "bob"])
+    }
+}
+#endif