diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift index 18895cb663..85331e7c7e 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift @@ -86,6 +86,10 @@ public final class AudioTrackMixer { /// jitter never trips it, and the file is written offline, so the latency it costs at a /// transition into silence costs the recording nothing. static let emissionGraceFrames = sampleRate / 4 + /// A gap between two passes of the sample queue longer than this means the queue itself + /// was blocked, not that the sources went quiet. The ticker fires every 10 ms, so ten + /// times that is never ordinary scheduling jitter. See `discountQueueStall(until:)`. + static let queueStallFrames = sampleRate / 10 /// Longest hole a source may silence-pad across. Past this the source has been dead long /// enough that padding would materialize however long the outage lasted — the cursor /// carries the track across the gap instead. @@ -121,6 +125,8 @@ public final class AudioTrackMixer { private var deliveredThroughFrame = [Int64?](repeating: nil, count: Source.allCases.count) private var undeliveredFrames = [Int64](repeating: 0, count: Source.allCases.count) private var longestHoleFrames = [Int64](repeating: 0, count: Source.allCases.count) + /// Where the clock stood at the previous pass of the sample queue, in absolute frames. + private var lastPassFrame: Int64? private var pending: [CMSampleBuffer] = [] private var didWarnAboutBacklog = false private var didWarnAboutDecode: Set = [] @@ -178,6 +184,7 @@ public final class AudioTrackMixer { } let now = clock() + discountQueueStall(until: now) let startFrame = frameIndex(of: presentationTime, from: anchor) noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount)) sources[source.rawValue].ingest(frames, atFrame: startFrame) @@ -194,7 +201,33 @@ public final class AudioTrackMixer { guard anchor != nil else { return } - drain(upTo: clock(), grace: Int64(MixFormat.emissionGraceFrames)) + let now = clock() + discountQueueStall(until: now) + drain(upTo: now, grace: Int64(MixFormat.emissionGraceFrames)) + } + + /// Keeps a blocked sample queue from reading as silent sources (#1117). + /// + /// The grace is measured on the clock, but the clock keeps running while the queue cannot: + /// a writer or a busy queue that holds it past `emissionGraceFrames` would otherwise make + /// the first pass after it — a tick, or the first of the buffers waiting behind it — see + /// every source as quiet for the whole stall, emit silence up to now, and then drop the + /// audio already queued for that span as `droppedFrames`. The sources were delivering the + /// whole time; it is the mixer that was not listening. So a pass that comes much later than + /// the ticker allows moves each source's last delivery forward by the gap, which measures + /// the grace against the time the queue was actually running. + private func discountQueueStall(until now: CMTime) { + guard let anchor, now.isValid, now.isNumeric else { + return + } + let nowFrame = frameIndex(of: now, from: anchor) + if let lastPassFrame, nowFrame - lastPassFrame > Int64(MixFormat.queueStallFrames) { + let gap = nowFrame - lastPassFrame + for index in sources.indices { + sources[index].lastDeliveryFrame += gap + } + } + lastPassFrame = nowFrame } /// Carries the track out to the end of the take and writes out everything still buffered. diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift index baf8a7c9c1..9a2a686ad0 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift @@ -237,8 +237,7 @@ final class AudioTrackMixerTests: XCTestCase { makeSourceBuffer(burst(seconds: 0.2, left: 0, right: 0.5), at: clock.now, nonInterleaved: false), from: .microphone ) - clock.advance(seconds: 0.2) - mixer.tick() + runTicker(mixer, clock, seconds: 0.2) for _ in 0..<10 { // Each buffer covers the 200 ms about to elapse, the way a live source delivers. @@ -246,8 +245,7 @@ final class AudioTrackMixerTests: XCTestCase { makeSourceBuffer(burst(seconds: 0.2, left: 0.5, right: 0), at: clock.now, nonInterleaved: true), from: .system ) - clock.advance(seconds: 0.2) - mixer.tick() + runTicker(mixer, clock, seconds: 0.2) } clock.advance(seconds: 0.2) mixer.finish(atSourceTime: clock.now) @@ -370,9 +368,9 @@ final class AudioTrackMixerTests: XCTestCase { makeSourceBuffer(burst(seconds: 0.5, left: 0.5, right: 0.5), at: .zero, nonInterleaved: true), from: .system ) - // Long enough past the grace that the source is written off and the cursor runs on. - clock.advance(seconds: 2) - mixer.tick() + // Long enough past the grace that the source is written off and the cursor runs on — + // with the queue running, so this is a quiet source and not a blocked queue. + runTicker(mixer, clock, seconds: 2) // …and only now does it hand over the span it went quiet for, which is already behind // the cursor and cannot be placed. @@ -393,6 +391,67 @@ final class AudioTrackMixerTests: XCTestCase { XCTAssertEqual(report.trimmedSeconds, 0, accuracy: 0.011) } + /// A blocked sample queue is not a silent source (#1117). + /// + /// Both sources keep delivering while the queue is held for a second — by the writer, or by + /// anything else on it — so their buffers for that second wait behind the block. When the + /// queue frees up, the ticker gets there first, which is the worst order: measured on the + /// clock alone, both sources would look a second quiet, the cursor would run to now in + /// silence, and the queued second would then arrive behind it and be dropped. + func testABlockedQueueDoesNotTurnQueuedAudioIntoSilence() { + let sink = RecordingSink() + let clock = TestClock(.zero) + let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1) + mixer.beginTimeline(at: clock.now) + + var delivered = CMTime.zero + func deliverBoth(seconds: Double) { + mixer.ingest( + makeSourceBuffer(burst(seconds: seconds, left: 0.5, right: 0), at: delivered, nonInterleaved: true), + from: .system + ) + mixer.ingest( + makeSourceBuffer(burst(seconds: seconds, left: 0, right: 0.5), at: delivered, nonInterleaved: false), + from: .microphone + ) + delivered = CMTimeAdd(delivered, CMTime(seconds: seconds, preferredTimescale: 48_000)) + } + + for _ in 0..<50 { + deliverBoth(seconds: 0.01) + runTicker(mixer, clock, seconds: 0.01) + } + + // The queue is blocked for a second: no tick, no buffer, while the clock runs on. + clock.advance(seconds: 1) + mixer.tick() + for _ in 0..<100 { + deliverBoth(seconds: 0.01) + } + + for _ in 0..<50 { + deliverBoth(seconds: 0.01) + runTicker(mixer, clock, seconds: 0.01) + } + mixer.finish(atSourceTime: clock.now) + + let frames = mixedFrames(sink) + assertContiguous(sink) + XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011) + XCTAssertEqual(mixer.deliveryReport(for: .system).droppedSeconds, 0, accuracy: 0.0001) + XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, accuracy: 0.0001) + for step in 0..<20 { + let from = Double(step) * 0.1 + for channel in 0.. Int16 { let firstFrame = Int(from * Double(sampleRate)) let lastFrame = min(Int(to * Double(sampleRate)), samples.count / channelCount)