Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ public final class AudioTrackMixer {
/// jitter never trips it, and the file is written offline, so the latency it costs at a
/// transition into silence costs the recording nothing.
static let emissionGraceFrames = sampleRate / 4
/// A gap between two passes of the sample queue longer than this means the queue itself
/// was blocked, not that the sources went quiet. The ticker fires every 10 ms, so ten
/// times that is never ordinary scheduling jitter. See `discountQueueStall(until:)`.
static let queueStallFrames = sampleRate / 10
/// Longest hole a source may silence-pad across. Past this the source has been dead long
/// enough that padding would materialize however long the outage lasted — the cursor
/// carries the track across the gap instead.
Expand Down Expand Up @@ -121,6 +125,8 @@ public final class AudioTrackMixer {
private var deliveredThroughFrame = [Int64?](repeating: nil, count: Source.allCases.count)
private var undeliveredFrames = [Int64](repeating: 0, count: Source.allCases.count)
private var longestHoleFrames = [Int64](repeating: 0, count: Source.allCases.count)
/// Where the clock stood at the previous pass of the sample queue, in absolute frames.
private var lastPassFrame: Int64?
private var pending: [CMSampleBuffer] = []
private var didWarnAboutBacklog = false
private var didWarnAboutDecode: Set<Int> = []
Expand Down Expand Up @@ -178,6 +184,7 @@ public final class AudioTrackMixer {
}

let now = clock()
discountQueueStall(until: now)
let startFrame = frameIndex(of: presentationTime, from: anchor)
noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount))
sources[source.rawValue].ingest(frames, atFrame: startFrame)
Expand All @@ -194,7 +201,33 @@ public final class AudioTrackMixer {
guard anchor != nil else {
return
}
drain(upTo: clock(), grace: Int64(MixFormat.emissionGraceFrames))
let now = clock()
discountQueueStall(until: now)
drain(upTo: now, grace: Int64(MixFormat.emissionGraceFrames))
}

/// Keeps a blocked sample queue from reading as silent sources (#1117).
///
/// The grace is measured on the clock, but the clock keeps running while the queue cannot:
/// a writer or a busy queue that holds it past `emissionGraceFrames` would otherwise make
/// the first pass after it — a tick, or the first of the buffers waiting behind it — see
/// every source as quiet for the whole stall, emit silence up to now, and then drop the
/// audio already queued for that span as `droppedFrames`. The sources were delivering the
/// whole time; it is the mixer that was not listening. So a pass that comes much later than
/// the ticker allows moves each source's last delivery forward by the gap, which measures
/// the grace against the time the queue was actually running.
private func discountQueueStall(until now: CMTime) {
guard let anchor, now.isValid, now.isNumeric else {
return
}
let nowFrame = frameIndex(of: now, from: anchor)
if let lastPassFrame, nowFrame - lastPassFrame > Int64(MixFormat.queueStallFrames) {
let gap = nowFrame - lastPassFrame
for index in sources.indices {
sources[index].lastDeliveryFrame += gap
}
}
lastPassFrame = nowFrame
}

/// Carries the track out to the end of the take and writes out everything still buffered.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,17 +237,15 @@ final class AudioTrackMixerTests: XCTestCase {
makeSourceBuffer(burst(seconds: 0.2, left: 0, right: 0.5), at: clock.now, nonInterleaved: false),
from: .microphone
)
clock.advance(seconds: 0.2)
mixer.tick()
runTicker(mixer, clock, seconds: 0.2)

for _ in 0..<10 {
// Each buffer covers the 200 ms about to elapse, the way a live source delivers.
mixer.ingest(
makeSourceBuffer(burst(seconds: 0.2, left: 0.5, right: 0), at: clock.now, nonInterleaved: true),
from: .system
)
clock.advance(seconds: 0.2)
mixer.tick()
runTicker(mixer, clock, seconds: 0.2)
}
clock.advance(seconds: 0.2)
mixer.finish(atSourceTime: clock.now)
Expand Down Expand Up @@ -370,9 +368,9 @@ final class AudioTrackMixerTests: XCTestCase {
makeSourceBuffer(burst(seconds: 0.5, left: 0.5, right: 0.5), at: .zero, nonInterleaved: true),
from: .system
)
// Long enough past the grace that the source is written off and the cursor runs on.
clock.advance(seconds: 2)
mixer.tick()
// Long enough past the grace that the source is written off and the cursor runs on —
// with the queue running, so this is a quiet source and not a blocked queue.
runTicker(mixer, clock, seconds: 2)

// …and only now does it hand over the span it went quiet for, which is already behind
// the cursor and cannot be placed.
Expand All @@ -393,6 +391,67 @@ final class AudioTrackMixerTests: XCTestCase {
XCTAssertEqual(report.trimmedSeconds, 0, accuracy: 0.011)
}

/// A blocked sample queue is not a silent source (#1117).
///
/// Both sources keep delivering while the queue is held for a second — by the writer, or by
/// anything else on it — so their buffers for that second wait behind the block. When the
/// queue frees up, the ticker gets there first, which is the worst order: measured on the
/// clock alone, both sources would look a second quiet, the cursor would run to now in
/// silence, and the queued second would then arrive behind it and be dropped.
func testABlockedQueueDoesNotTurnQueuedAudioIntoSilence() {
let sink = RecordingSink()
let clock = TestClock(.zero)
let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1)
mixer.beginTimeline(at: clock.now)

var delivered = CMTime.zero
func deliverBoth(seconds: Double) {
mixer.ingest(
makeSourceBuffer(burst(seconds: seconds, left: 0.5, right: 0), at: delivered, nonInterleaved: true),
from: .system
)
mixer.ingest(
makeSourceBuffer(burst(seconds: seconds, left: 0, right: 0.5), at: delivered, nonInterleaved: false),
from: .microphone
)
delivered = CMTimeAdd(delivered, CMTime(seconds: seconds, preferredTimescale: 48_000))
}

for _ in 0..<50 {
deliverBoth(seconds: 0.01)
runTicker(mixer, clock, seconds: 0.01)
}

// The queue is blocked for a second: no tick, no buffer, while the clock runs on.
clock.advance(seconds: 1)
mixer.tick()
for _ in 0..<100 {
deliverBoth(seconds: 0.01)
}

for _ in 0..<50 {
deliverBoth(seconds: 0.01)
runTicker(mixer, clock, seconds: 0.01)
}
mixer.finish(atSourceTime: clock.now)

let frames = mixedFrames(sink)
assertContiguous(sink)
XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011)
XCTAssertEqual(mixer.deliveryReport(for: .system).droppedSeconds, 0, accuracy: 0.0001)
XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, accuracy: 0.0001)
for step in 0..<20 {
let from = Double(step) * 0.1
for channel in 0..<channelCount {
XCTAssertGreaterThan(
peak(frames, from: from, to: from + 0.1, channel: channel),
audibleThreshold,
"silence on channel \(channel) at \(from)s means queued audio was dropped"
)
}
}
}

// MARK: - Pause

/// A pause stops the clock instead of resetting it. `ScreenCaptureRecorder.timelineNow()`
Expand Down Expand Up @@ -763,6 +822,14 @@ final class AudioTrackMixerTests: XCTestCase {
}
}

/// Moves the clock the way the recorder's 10 ms ticker sees it, with the queue running.
private func runTicker(_ mixer: AudioTrackMixer, _ clock: TestClock, seconds: Double) {
for _ in 0..<Int((seconds * 100).rounded()) {
clock.advance(seconds: 0.01)
mixer.tick()
}
}

private func peak(_ samples: [Int16], from: Double, to: Double, channel: Int? = nil) -> Int16 {
let firstFrame = Int(from * Double(sampleRate))
let lastFrame = min(Int(to * Double(sampleRate)), samples.count / channelCount)
Expand Down
Loading