From 2623a88be3b8e98b1ba1075b1d044321b252b8d8 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Mon, 14 Sep 2026 11:16:52 -0400 Subject: [PATCH 1/3] fix(cli/scoring): sort speaker ids so diarization mapping is deterministic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit computeSpeakerMapping enumerated predicted/ground-truth speakers via Array(dict.keys) — a per-instance random order (see #911/#921) — and both assignment solvers keep the first-encountered winner among tied confusion-matrix entries, so the reported speaker mapping (and JER in asymmetric cases) could differ between runs whenever two pairings tied in millisecond-rounded overlap. Reproduced in a standalone harness: two predicted speakers tying at 4.75 s overlap against one ground-truth speaker produced 2 distinct mappings within a single 200-call process on the old code; sorted key arrays yield 1. Ties now resolve by the assignment solver's fixed index-order rule over lexicographically sorted speaker ids. Fixes #922 --- .../Utils/DiarizationMetrics.swift | 7 +- .../CLI/DiarizationSpeakerMappingTests.swift | 65 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift diff --git a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift index 2654d49b..e28e1899 100644 --- a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift +++ b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift @@ -593,8 +593,11 @@ enum DiarizationMetricsCalculator { ) -> [String: String] { guard !predicted.isEmpty, !groundTruth.isEmpty else { return [:] } - let predictedIds = Array(predicted.keys) - let groundTruthIds = Array(groundTruth.keys) + // Sorted so assignment tie-breaks are deterministic: dictionary key + // order is per-instance random, and the solvers keep the + // first-encountered winner among tied overlaps (issue #922). + let predictedIds = predicted.keys.sorted() + let groundTruthIds = groundTruth.keys.sorted() var confusionMatrix = Array( repeating: Array(repeating: 0, count: predictedIds.count), diff --git a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift new file mode 100644 index 00000000..55650dd3 --- /dev/null +++ b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift @@ -0,0 +1,65 @@ +#if os(macOS) +import XCTest + +@testable import FluidAudio +@testable import FluidAudioCLI + +/// Tests for the diarization scorer's speaker mapping (issue #922). +/// +/// `computeSpeakerMapping` builds its speaker index orders from dictionary +/// keys, and the assignment solvers keep the first-encountered winner among +/// tied overlaps. Before the key arrays were sorted, per-instance random +/// dictionary order made the winning mapping — and with it JER — drift +/// between runs on tied confusion-matrix entries. +final class DiarizationSpeakerMappingTests: XCTestCase { + + private func segment(_ speaker: String, _ start: Float, _ end: Float) -> TimedSpeakerSegment { + TimedSpeakerSegment( + speakerId: speaker, embedding: [], startTimeSeconds: start, endTimeSeconds: end, + qualityScore: 1.0) + } + + /// Two predicted speakers tie exactly on overlap with one ground-truth + /// speaker (4.75 s each after the 0.25 s collar). Only one can win the + /// assignment; the winner must not depend on dictionary key order. + func testTiedOverlapsMapDeterministically() { + let groundTruth = [segment("spk_a", 0, 10)] + let predicted = [ + segment("left", 0, 5), + segment("right", 5, 10), + ] + + let first = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(first.speakerMapping.count, 1) + XCTAssertEqual(Set(first.speakerMapping.values), ["spk_a"]) + + // Every call regroups segments into fresh dictionaries, so each + // iteration samples a new per-instance key order. + for _ in 0..<50 { + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, first.speakerMapping) + XCTAssertEqual(metrics.jer, first.jer) + XCTAssertEqual(metrics.der, first.der) + } + } + + /// Unambiguous overlaps must still produce the correct mapping after the + /// key arrays are sorted. + func testUnambiguousMappingIsCorrect() { + let groundTruth = [ + segment("alice", 0, 10), + segment("bob", 11, 20), + ] + let predicted = [ + segment("s1", 0, 9), + segment("s2", 11.5, 20), + ] + + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, ["s1": "alice", "s2": "bob"]) + } +} +#endif From 59d355cf4e43348df677d21a066381367dfdc4ca Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Mon, 14 Sep 2026 11:26:56 -0400 Subject: [PATCH 2/3] fix(cli/scoring): finish diarization determinism sweep; review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-review fixes on the #922 branch: - offlineMetrics still iterated speakerMapping / groundTruthBySpeaker / predictedBySpeaker dictionaries while accumulating Doubles. Addition is not associative and both der and jer pass through near-zero cancellation (max(0, overlapSpeech - correctlyAssigned) and 1.0 - averageJaccard), where a ulp-level reorder survives the Float cast — so with 3+ speakers der/jer were still not run-to-run deterministic after the mapping fix. All three loops now iterate in sorted key order; debug mapping log also sorted. - Streaming path had the same class: overlapsByGtSpeaker.max(by: { $0.value < $1.value }) keeps the first-encountered element among tied overlaps, in random dictionary order. Ties now break by smaller speaker id. - Test pins the exact tied winner (["right": "spk_a"]) instead of count/values-set asserts, so a silent tie-break change that would shift recorded benchmark numbers fails the test. Winner re-verified via the standalone harness (200-call x 3-process: 1 distinct mapping; unambiguous scenario unchanged). Review also empirically re-confirmed per-ALLOCATION dictionary seeding (Swift 6.2.3): the unfixed code fails the new test's predicate in 20/20 processes, so the in-process 50-iteration loop is a real regression guard. --- .../Commands/DiarizationBenchmark.swift | 8 ++++++-- Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift | 12 ++++++++---- .../CLI/DiarizationSpeakerMappingTests.swift | 6 ++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift index 8e070b4c..d34b2418 100644 --- a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift @@ -984,8 +984,12 @@ enum StreamDiarizationBenchmark { } } - // Find the GT speaker with most overlap - if let (bestMatch, bestOverlap) = overlapsByGtSpeaker.max(by: { $0.value < $1.value }), + // Find the GT speaker with most overlap; break ties by speaker id + // so the mapping doesn't depend on random dictionary order. + if let (bestMatch, bestOverlap) = overlapsByGtSpeaker.max(by: { + if $0.value != $1.value { return $0.value < $1.value } + return $0.key > $1.key + }), bestOverlap > 0.5 { // Require at least 0.5s total overlap firstOccurrenceMap[predSegment.speakerId] = bestMatch diff --git a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift index e28e1899..eb061692 100644 --- a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift +++ b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift @@ -186,8 +186,11 @@ enum DiarizationMetricsCalculator { groundTruth: groundTruthBySpeaker ) + // Dictionary iteration order is per-instance random, so accumulate in + // sorted order — Double addition is not associative, and near-zero + // confusion/JER residues are sensitive to the summation order. var correctlyAssigned = 0.0 - for (predId, truthId) in speakerMapping { + for (predId, truthId) in speakerMapping.sorted(by: { $0.key < $1.key }) { if let predSegments = predictedBySpeaker[predId], let truthSegments = groundTruthBySpeaker[truthId] { @@ -205,7 +208,7 @@ enum DiarizationMetricsCalculator { var jaccardScores: [Double] = [] let inverseMapping = Dictionary(uniqueKeysWithValues: speakerMapping.map { ($0.value, $0.key) }) - for (truthId, truthSegments) in groundTruthBySpeaker { + for (truthId, truthSegments) in groundTruthBySpeaker.sorted(by: { $0.key < $1.key }) { let matchedPred = inverseMapping[truthId] let predictedSegmentsForSpeaker = matchedPred.flatMap { predictedBySpeaker[$0] } ?? [] let intersection = overlapDuration(predictedSegmentsForSpeaker, truthSegments) @@ -215,7 +218,8 @@ enum DiarizationMetricsCalculator { } } - for (predId, predSegments) in predictedBySpeaker where speakerMapping[predId] == nil { + for (predId, predSegments) in predictedBySpeaker.sorted(by: { $0.key < $1.key }) + where speakerMapping[predId] == nil { if unionDuration(predSegments) > 0 { jaccardScores.append(0.0) } @@ -230,7 +234,7 @@ enum DiarizationMetricsCalculator { } if let logger = logger { - logger.debug("🎯 Offline mapping: \(speakerMapping)") + logger.debug("🎯 Offline mapping: \(speakerMapping.sorted(by: { $0.key < $1.key }))") let formattedDer = String(format: "%.1f", der) let formattedMiss = String(format: "%.1f", missRate) let formattedFalseAlarm = String(format: "%.1f", falseAlarmRate) diff --git a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift index 55650dd3..af75a6ec 100644 --- a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift +++ b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift @@ -31,8 +31,10 @@ final class DiarizationSpeakerMappingTests: XCTestCase { let first = DiarizationMetricsCalculator.offlineMetrics( predicted: predicted, groundTruth: groundTruth) - XCTAssertEqual(first.speakerMapping.count, 1) - XCTAssertEqual(Set(first.speakerMapping.values), ["spk_a"]) + // Pinned winner: the assignment DP keeps the skip-branch result on + // ties, so with sorted ids "right" wins. A silent tie-break change + // would shift recorded benchmark numbers; this catches it. + XCTAssertEqual(first.speakerMapping, ["right": "spk_a"]) // Every call regroups segments into fresh dictionaries, so each // iteration samples a new per-instance key order. From e7b44808d3c2d0091a54324adaef9a85af61507c Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Mon, 14 Sep 2026 11:45:34 -0400 Subject: [PATCH 3/3] test(cli/scoring): cover streaming tie-break and multi-speaker accumulation Closes the review's remaining test gap: - Extract the streaming overlap-winner selection into StreamDiarizationBenchmark.bestOverlapMatch (internal, testable) and test it directly: 100 fresh tied dictionaries always yield the smaller speaker id; unambiguous max and empty-dict cases covered. - Add a 3-speaker offlineMetrics test asserting mapping plus exact der/jer/speakerErrorRate stability across 50 calls, exercising the sorted Double-accumulation loops. Expected values verified with the standalone harness: mapping p1/p2/p3 -> gt_a/gt_b/gt_c, metrics bitwise identical within and across 5 processes. --- .../Commands/DiarizationBenchmark.swift | 20 ++++++--- .../CLI/DiarizationSpeakerMappingTests.swift | 45 +++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift index d34b2418..a2951892 100644 --- a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift @@ -939,6 +939,18 @@ enum StreamDiarizationBenchmark { } } + /// Deterministic winner among per-speaker overlap totals: largest + /// overlap, ties broken by smaller speaker id (dictionary iteration + /// order is per-instance random; issue #922). + static func bestOverlapMatch( + _ overlapsBySpeaker: [String: Float] + ) -> (speakerId: String, overlap: Float)? { + overlapsBySpeaker.max(by: { + if $0.value != $1.value { return $0.value < $1.value } + return $0.key > $1.key + }).map { ($0.key, $0.value) } + } + /// Calculate DER metrics with first-occurrence mapping for streaming evaluation private static func calculateStreamingMetrics( predicted: [TimedSpeakerSegment], @@ -984,12 +996,8 @@ enum StreamDiarizationBenchmark { } } - // Find the GT speaker with most overlap; break ties by speaker id - // so the mapping doesn't depend on random dictionary order. - if let (bestMatch, bestOverlap) = overlapsByGtSpeaker.max(by: { - if $0.value != $1.value { return $0.value < $1.value } - return $0.key > $1.key - }), + // Find the GT speaker with most overlap + if let (bestMatch, bestOverlap) = bestOverlapMatch(overlapsByGtSpeaker), bestOverlap > 0.5 { // Require at least 0.5s total overlap firstOccurrenceMap[predSegment.speakerId] = bestMatch diff --git a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift index af75a6ec..fc8ed73f 100644 --- a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift +++ b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift @@ -47,6 +47,51 @@ final class DiarizationSpeakerMappingTests: XCTestCase { } } + /// With 3+ speakers, der/jer accumulate Doubles across several speakers; + /// the sums must be byte-stable call to call now that the accumulation + /// loops iterate in sorted key order. + func testMultiSpeakerMetricsAreStableAcrossCalls() { + let groundTruth = [ + segment("gt_a", 0, 10), + segment("gt_b", 11, 20), + segment("gt_c", 21, 30), + ] + let predicted = [ + segment("p1", 0, 9), + segment("p2", 11.5, 19), + segment("p3", 21.5, 28), + ] + + let first = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(first.speakerMapping, ["p1": "gt_a", "p2": "gt_b", "p3": "gt_c"]) + + for _ in 0..<50 { + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, first.speakerMapping) + XCTAssertEqual(metrics.der, first.der) + XCTAssertEqual(metrics.jer, first.jer) + XCTAssertEqual(metrics.speakerErrorRate, first.speakerErrorRate) + } + } + + /// The streaming scorer's overlap winner must break ties by smaller + /// speaker id, not by whichever tied entry a fresh dictionary happens to + /// enumerate first. + func testStreamingTieBreakPicksSmallestSpeakerId() { + for _ in 0..<100 { + let overlaps: [String: Float] = ["spk_b": 2.0, "spk_a": 2.0, "spk_c": 1.5] + let best = StreamDiarizationBenchmark.bestOverlapMatch(overlaps) + XCTAssertEqual(best?.speakerId, "spk_a") + XCTAssertEqual(best?.overlap, 2.0) + } + + let unambiguous = StreamDiarizationBenchmark.bestOverlapMatch(["spk_a": 1.0, "spk_b": 3.0]) + XCTAssertEqual(unambiguous?.speakerId, "spk_b") + XCTAssertNil(StreamDiarizationBenchmark.bestOverlapMatch([:])) + } + /// Unambiguous overlaps must still produce the correct mapping after the /// key arrays are sorted. func testUnambiguousMappingIsCorrect() {