Skip to content

Commit daf23f7

Browse files
committed
fix(audio): don't read a blocked sample queue as silent sources on macOS
AudioTrackMixer measured its 250 ms grace on the timeline clock, which keeps running while the sample queue is blocked. The first pass after a stall saw both sources as quiet for the whole stall, emitted silence up to now and then dropped the audio queued behind the block as droppedFrames. A pass that comes more than 100 ms after the previous one (the ticker fires every 10 ms) now moves each source's last delivery forward by the gap, so the grace counts only time the queue was running. Refs #1117
1 parent b9f9d85 commit daf23f7

2 files changed

Lines changed: 108 additions & 8 deletions

File tree

‎electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ public final class AudioTrackMixer {
8686
/// jitter never trips it, and the file is written offline, so the latency it costs at a
8787
/// transition into silence costs the recording nothing.
8888
static let emissionGraceFrames = sampleRate / 4
89+
/// A gap between two passes of the sample queue longer than this means the queue itself
90+
/// was blocked, not that the sources went quiet. The ticker fires every 10 ms, so ten
91+
/// times that is never ordinary scheduling jitter. See `discountQueueStall(until:)`.
92+
static let queueStallFrames = sampleRate / 10
8993
/// Longest hole a source may silence-pad across. Past this the source has been dead long
9094
/// enough that padding would materialize however long the outage lasted — the cursor
9195
/// carries the track across the gap instead.
@@ -121,6 +125,8 @@ public final class AudioTrackMixer {
121125
private var deliveredThroughFrame = [Int64?](repeating: nil, count: Source.allCases.count)
122126
private var undeliveredFrames = [Int64](repeating: 0, count: Source.allCases.count)
123127
private var longestHoleFrames = [Int64](repeating: 0, count: Source.allCases.count)
128+
/// Where the clock stood at the previous pass of the sample queue, in absolute frames.
129+
private var lastPassFrame: Int64?
124130
private var pending: [CMSampleBuffer] = []
125131
private var didWarnAboutBacklog = false
126132
private var didWarnAboutDecode: Set<Int> = []
@@ -178,6 +184,7 @@ public final class AudioTrackMixer {
178184
}
179185

180186
let now = clock()
187+
discountQueueStall(until: now)
181188
let startFrame = frameIndex(of: presentationTime, from: anchor)
182189
noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount))
183190
sources[source.rawValue].ingest(frames, atFrame: startFrame)
@@ -194,7 +201,33 @@ public final class AudioTrackMixer {
194201
guard anchor != nil else {
195202
return
196203
}
197-
drain(upTo: clock(), grace: Int64(MixFormat.emissionGraceFrames))
204+
let now = clock()
205+
discountQueueStall(until: now)
206+
drain(upTo: now, grace: Int64(MixFormat.emissionGraceFrames))
207+
}
208+
209+
/// Keeps a blocked sample queue from reading as silent sources (#1117).
210+
///
211+
/// The grace is measured on the clock, but the clock keeps running while the queue cannot:
212+
/// a writer or a busy queue that holds it past `emissionGraceFrames` would otherwise make
213+
/// the first pass after it — a tick, or the first of the buffers waiting behind it — see
214+
/// every source as quiet for the whole stall, emit silence up to now, and then drop the
215+
/// audio already queued for that span as `droppedFrames`. The sources were delivering the
216+
/// whole time; it is the mixer that was not listening. So a pass that comes much later than
217+
/// the ticker allows moves each source's last delivery forward by the gap, which measures
218+
/// the grace against the time the queue was actually running.
219+
private func discountQueueStall(until now: CMTime) {
220+
guard let anchor, now.isValid, now.isNumeric else {
221+
return
222+
}
223+
let nowFrame = frameIndex(of: now, from: anchor)
224+
if let lastPassFrame, nowFrame - lastPassFrame > Int64(MixFormat.queueStallFrames) {
225+
let gap = nowFrame - lastPassFrame
226+
for index in sources.indices {
227+
sources[index].lastDeliveryFrame += gap
228+
}
229+
}
230+
lastPassFrame = nowFrame
198231
}
199232

200233
/// Carries the track out to the end of the take and writes out everything still buffered.

‎electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift‎

Lines changed: 74 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -237,17 +237,15 @@ final class AudioTrackMixerTests: XCTestCase {
237237
makeSourceBuffer(burst(seconds: 0.2, left: 0, right: 0.5), at: clock.now, nonInterleaved: false),
238238
from: .microphone
239239
)
240-
clock.advance(seconds: 0.2)
241-
mixer.tick()
240+
runTicker(mixer, clock, seconds: 0.2)
242241

243242
for _ in 0..<10 {
244243
// Each buffer covers the 200 ms about to elapse, the way a live source delivers.
245244
mixer.ingest(
246245
makeSourceBuffer(burst(seconds: 0.2, left: 0.5, right: 0), at: clock.now, nonInterleaved: true),
247246
from: .system
248247
)
249-
clock.advance(seconds: 0.2)
250-
mixer.tick()
248+
runTicker(mixer, clock, seconds: 0.2)
251249
}
252250
clock.advance(seconds: 0.2)
253251
mixer.finish(atSourceTime: clock.now)
@@ -370,9 +368,9 @@ final class AudioTrackMixerTests: XCTestCase {
370368
makeSourceBuffer(burst(seconds: 0.5, left: 0.5, right: 0.5), at: .zero, nonInterleaved: true),
371369
from: .system
372370
)
373-
// Long enough past the grace that the source is written off and the cursor runs on.
374-
clock.advance(seconds: 2)
375-
mixer.tick()
371+
// Long enough past the grace that the source is written off and the cursor runs on —
372+
// with the queue running, so this is a quiet source and not a blocked queue.
373+
runTicker(mixer, clock, seconds: 2)
376374

377375
// …and only now does it hand over the span it went quiet for, which is already behind
378376
// the cursor and cannot be placed.
@@ -393,6 +391,67 @@ final class AudioTrackMixerTests: XCTestCase {
393391
XCTAssertEqual(report.trimmedSeconds, 0, accuracy: 0.011)
394392
}
395393

394+
/// A blocked sample queue is not a silent source (#1117).
395+
///
396+
/// Both sources keep delivering while the queue is held for a second — by the writer, or by
397+
/// anything else on it — so their buffers for that second wait behind the block. When the
398+
/// queue frees up, the ticker gets there first, which is the worst order: measured on the
399+
/// clock alone, both sources would look a second quiet, the cursor would run to now in
400+
/// silence, and the queued second would then arrive behind it and be dropped.
401+
func testABlockedQueueDoesNotTurnQueuedAudioIntoSilence() {
402+
let sink = RecordingSink()
403+
let clock = TestClock(.zero)
404+
let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1)
405+
mixer.beginTimeline(at: clock.now)
406+
407+
var delivered = CMTime.zero
408+
func deliverBoth(seconds: Double) {
409+
mixer.ingest(
410+
makeSourceBuffer(burst(seconds: seconds, left: 0.5, right: 0), at: delivered, nonInterleaved: true),
411+
from: .system
412+
)
413+
mixer.ingest(
414+
makeSourceBuffer(burst(seconds: seconds, left: 0, right: 0.5), at: delivered, nonInterleaved: false),
415+
from: .microphone
416+
)
417+
delivered = CMTimeAdd(delivered, CMTime(seconds: seconds, preferredTimescale: 48_000))
418+
}
419+
420+
for _ in 0..<50 {
421+
deliverBoth(seconds: 0.01)
422+
runTicker(mixer, clock, seconds: 0.01)
423+
}
424+
425+
// The queue is blocked for a second: no tick, no buffer, while the clock runs on.
426+
clock.advance(seconds: 1)
427+
mixer.tick()
428+
for _ in 0..<100 {
429+
deliverBoth(seconds: 0.01)
430+
}
431+
432+
for _ in 0..<50 {
433+
deliverBoth(seconds: 0.01)
434+
runTicker(mixer, clock, seconds: 0.01)
435+
}
436+
mixer.finish(atSourceTime: clock.now)
437+
438+
let frames = mixedFrames(sink)
439+
assertContiguous(sink)
440+
XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011)
441+
XCTAssertEqual(mixer.deliveryReport(for: .system).droppedSeconds, 0, accuracy: 0.0001)
442+
XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, accuracy: 0.0001)
443+
for step in 0..<20 {
444+
let from = Double(step) * 0.1
445+
for channel in 0..<channelCount {
446+
XCTAssertGreaterThan(
447+
peak(frames, from: from, to: from + 0.1, channel: channel),
448+
audibleThreshold,
449+
"silence on channel \(channel) at \(from)s means queued audio was dropped"
450+
)
451+
}
452+
}
453+
}
454+
396455
// MARK: - Pause
397456

398457
/// A pause stops the clock instead of resetting it. `ScreenCaptureRecorder.timelineNow()`
@@ -763,6 +822,14 @@ final class AudioTrackMixerTests: XCTestCase {
763822
}
764823
}
765824

825+
/// Moves the clock the way the recorder's 10 ms ticker sees it, with the queue running.
826+
private func runTicker(_ mixer: AudioTrackMixer, _ clock: TestClock, seconds: Double) {
827+
for _ in 0..<Int((seconds * 100).rounded()) {
828+
clock.advance(seconds: 0.01)
829+
mixer.tick()
830+
}
831+
}
832+
766833
private func peak(_ samples: [Int16], from: Double, to: Double, channel: Int? = nil) -> Int16 {
767834
let firstFrame = Int(from * Double(sampleRate))
768835
let lastFrame = min(Int(to * Double(sampleRate)), samples.count / channelCount)

0 commit comments

Comments
 (0)