diff --git a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift index 8e070b4c..a2951892 100644 --- a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift @@ -939,6 +939,18 @@ enum StreamDiarizationBenchmark { } } + /// Deterministic winner among per-speaker overlap totals: largest + /// overlap, ties broken by smaller speaker id (dictionary iteration + /// order is per-instance random; issue #922). + static func bestOverlapMatch( + _ overlapsBySpeaker: [String: Float] + ) -> (speakerId: String, overlap: Float)? { + overlapsBySpeaker.max(by: { + if $0.value != $1.value { return $0.value < $1.value } + return $0.key > $1.key + }).map { ($0.key, $0.value) } + } + /// Calculate DER metrics with first-occurrence mapping for streaming evaluation private static func calculateStreamingMetrics( predicted: [TimedSpeakerSegment], @@ -985,7 +997,7 @@ enum StreamDiarizationBenchmark { } // Find the GT speaker with most overlap - if let (bestMatch, bestOverlap) = overlapsByGtSpeaker.max(by: { $0.value < $1.value }), + if let (bestMatch, bestOverlap) = bestOverlapMatch(overlapsByGtSpeaker), bestOverlap > 0.5 { // Require at least 0.5s total overlap firstOccurrenceMap[predSegment.speakerId] = bestMatch diff --git a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift index 2654d49b..eb061692 100644 --- a/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift +++ b/Sources/FluidAudioCLI/Utils/DiarizationMetrics.swift @@ -186,8 +186,11 @@ enum DiarizationMetricsCalculator { groundTruth: groundTruthBySpeaker ) + // Dictionary iteration order is per-instance random, so accumulate in + // sorted order — Double addition is not associative, and near-zero + // confusion/JER residues are sensitive to the summation order. var correctlyAssigned = 0.0 - for (predId, truthId) in speakerMapping { + for (predId, truthId) in speakerMapping.sorted(by: { $0.key < $1.key }) { if let predSegments = predictedBySpeaker[predId], let truthSegments = groundTruthBySpeaker[truthId] { @@ -205,7 +208,7 @@ enum DiarizationMetricsCalculator { var jaccardScores: [Double] = [] let inverseMapping = Dictionary(uniqueKeysWithValues: speakerMapping.map { ($0.value, $0.key) }) - for (truthId, truthSegments) in groundTruthBySpeaker { + for (truthId, truthSegments) in groundTruthBySpeaker.sorted(by: { $0.key < $1.key }) { let matchedPred = inverseMapping[truthId] let predictedSegmentsForSpeaker = matchedPred.flatMap { predictedBySpeaker[$0] } ?? [] let intersection = overlapDuration(predictedSegmentsForSpeaker, truthSegments) @@ -215,7 +218,8 @@ enum DiarizationMetricsCalculator { } } - for (predId, predSegments) in predictedBySpeaker where speakerMapping[predId] == nil { + for (predId, predSegments) in predictedBySpeaker.sorted(by: { $0.key < $1.key }) + where speakerMapping[predId] == nil { if unionDuration(predSegments) > 0 { jaccardScores.append(0.0) } @@ -230,7 +234,7 @@ enum DiarizationMetricsCalculator { } if let logger = logger { - logger.debug("🎯 Offline mapping: \(speakerMapping)") + logger.debug("🎯 Offline mapping: \(speakerMapping.sorted(by: { $0.key < $1.key }))") let formattedDer = String(format: "%.1f", der) let formattedMiss = String(format: "%.1f", missRate) let formattedFalseAlarm = String(format: "%.1f", falseAlarmRate) @@ -593,8 +597,11 @@ enum DiarizationMetricsCalculator { ) -> [String: String] { guard !predicted.isEmpty, !groundTruth.isEmpty else { return [:] } - let predictedIds = Array(predicted.keys) - let groundTruthIds = Array(groundTruth.keys) + // Sorted so assignment tie-breaks are deterministic: dictionary key + // order is per-instance random, and the solvers keep the + // first-encountered winner among tied overlaps (issue #922). + let predictedIds = predicted.keys.sorted() + let groundTruthIds = groundTruth.keys.sorted() var confusionMatrix = Array( repeating: Array(repeating: 0, count: predictedIds.count), diff --git a/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift new file mode 100644 index 00000000..fc8ed73f --- /dev/null +++ b/Tests/FluidAudioTests/CLI/DiarizationSpeakerMappingTests.swift @@ -0,0 +1,112 @@ +#if os(macOS) +import XCTest + +@testable import FluidAudio +@testable import FluidAudioCLI + +/// Tests for the diarization scorer's speaker mapping (issue #922). +/// +/// `computeSpeakerMapping` builds its speaker index orders from dictionary +/// keys, and the assignment solvers keep the first-encountered winner among +/// tied overlaps. Before the key arrays were sorted, per-instance random +/// dictionary order made the winning mapping — and with it JER — drift +/// between runs on tied confusion-matrix entries. +final class DiarizationSpeakerMappingTests: XCTestCase { + + private func segment(_ speaker: String, _ start: Float, _ end: Float) -> TimedSpeakerSegment { + TimedSpeakerSegment( + speakerId: speaker, embedding: [], startTimeSeconds: start, endTimeSeconds: end, + qualityScore: 1.0) + } + + /// Two predicted speakers tie exactly on overlap with one ground-truth + /// speaker (4.75 s each after the 0.25 s collar). Only one can win the + /// assignment; the winner must not depend on dictionary key order. + func testTiedOverlapsMapDeterministically() { + let groundTruth = [segment("spk_a", 0, 10)] + let predicted = [ + segment("left", 0, 5), + segment("right", 5, 10), + ] + + let first = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + // Pinned winner: the assignment DP keeps the skip-branch result on + // ties, so with sorted ids "right" wins. A silent tie-break change + // would shift recorded benchmark numbers; this catches it. + XCTAssertEqual(first.speakerMapping, ["right": "spk_a"]) + + // Every call regroups segments into fresh dictionaries, so each + // iteration samples a new per-instance key order. + for _ in 0..<50 { + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, first.speakerMapping) + XCTAssertEqual(metrics.jer, first.jer) + XCTAssertEqual(metrics.der, first.der) + } + } + + /// With 3+ speakers, der/jer accumulate Doubles across several speakers; + /// the sums must be byte-stable call to call now that the accumulation + /// loops iterate in sorted key order. + func testMultiSpeakerMetricsAreStableAcrossCalls() { + let groundTruth = [ + segment("gt_a", 0, 10), + segment("gt_b", 11, 20), + segment("gt_c", 21, 30), + ] + let predicted = [ + segment("p1", 0, 9), + segment("p2", 11.5, 19), + segment("p3", 21.5, 28), + ] + + let first = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(first.speakerMapping, ["p1": "gt_a", "p2": "gt_b", "p3": "gt_c"]) + + for _ in 0..<50 { + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, first.speakerMapping) + XCTAssertEqual(metrics.der, first.der) + XCTAssertEqual(metrics.jer, first.jer) + XCTAssertEqual(metrics.speakerErrorRate, first.speakerErrorRate) + } + } + + /// The streaming scorer's overlap winner must break ties by smaller + /// speaker id, not by whichever tied entry a fresh dictionary happens to + /// enumerate first. + func testStreamingTieBreakPicksSmallestSpeakerId() { + for _ in 0..<100 { + let overlaps: [String: Float] = ["spk_b": 2.0, "spk_a": 2.0, "spk_c": 1.5] + let best = StreamDiarizationBenchmark.bestOverlapMatch(overlaps) + XCTAssertEqual(best?.speakerId, "spk_a") + XCTAssertEqual(best?.overlap, 2.0) + } + + let unambiguous = StreamDiarizationBenchmark.bestOverlapMatch(["spk_a": 1.0, "spk_b": 3.0]) + XCTAssertEqual(unambiguous?.speakerId, "spk_b") + XCTAssertNil(StreamDiarizationBenchmark.bestOverlapMatch([:])) + } + + /// Unambiguous overlaps must still produce the correct mapping after the + /// key arrays are sorted. + func testUnambiguousMappingIsCorrect() { + let groundTruth = [ + segment("alice", 0, 10), + segment("bob", 11, 20), + ] + let predicted = [ + segment("s1", 0, 9), + segment("s2", 11.5, 20), + ] + + let metrics = DiarizationMetricsCalculator.offlineMetrics( + predicted: predicted, groundTruth: groundTruth) + XCTAssertEqual(metrics.speakerMapping, ["s1": "alice", "s2": "bob"]) + } +} +#endif