Repository navigation
Expand file tree
/
Copy pathScreenCaptureRecorder.swift
More file actions
1390 lines (1288 loc) · 49.7 KB
/
Copy pathScreenCaptureRecorder.swift
File metadata and controls
1390 lines (1288 loc) · 49.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import AVFoundation
import CoreGraphics
import CoreMedia
import Foundation
import OpenScreenCaptureCore
import ScreenCaptureKit
struct Rectangle: Decodable {
let x: Double
let y: Double
let width: Double
let height: Double
}
struct RecordingRequest: Decodable {
struct Source: Decodable {
let type: String
let sourceId: String
let displayId: UInt32?
let windowId: UInt32?
let bounds: Rectangle?
}
struct Video: Decodable {
let fps: Int
let width: Int
let height: Int
let bitrate: Int?
let hideSystemCursor: Bool
}
struct Audio: Decodable {
struct SystemAudio: Decodable {
let enabled: Bool
}
struct Microphone: Decodable {
let enabled: Bool
let deviceId: String?
let deviceName: String?
let gain: Double
}
let system: SystemAudio
let microphone: Microphone
}
struct Webcam: Decodable {
let enabled: Bool
let deviceId: String?
let deviceName: String?
let width: Int
let height: Int
let fps: Int
}
struct Cursor: Decodable {
let mode: String
}
struct Outputs: Decodable {
let screenPath: String
let manifestPath: String?
}
let schemaVersion: Int?
let recordingId: Int?
let excludedWindowIds: [UInt32]?
/// Display captures only. Absent in requests from apps that predate the option.
let hideDesktopIcons: Bool?
let source: Source
let video: Video
let audio: Audio
let webcam: Webcam
let cursor: Cursor
let outputs: Outputs
}
enum HelperError: Error, CustomStringConvertible {
case invalidArguments
case unsupportedMacOS
case unsupportedFeature(String)
case sourceNotFound(String)
case invalidSourceType(String)
case permissionDenied(String)
case writerSetupFailed(String)
var description: String {
switch self {
case .invalidArguments:
return "Expected one JSON recording request argument."
case .unsupportedMacOS:
return "ScreenCaptureKit recording requires macOS 13 or newer."
case .unsupportedFeature(let message):
return message
case .sourceNotFound(let message):
return message
case .invalidSourceType(let sourceType):
return "Unsupported source type: \(sourceType)."
case .permissionDenied(let message):
return message
case .writerSetupFailed(let message):
return message
}
}
}
/// A source chosen in Apple's system picker, kept by `PickerSession` for the takes after it.
struct PickedSource {
let filter: SCContentFilter
/// Global frame (points, top-left origin) of what was picked, for cursor mapping.
let frame: CGRect
let displayId: CGDirectDisplayID?
}
@available(macOS 13.0, *)
final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
private struct CaptureTarget {
let filter: SCContentFilter
let width: Int
let height: Int
// Global frame (points, top-left origin) of the captured region. Used by the
// renderer to normalize cursor positions into the captured window's space.
let captureFrame: CGRect
}
private let request: RecordingRequest
private let sampleQueue = DispatchQueue(label: "app.openscreen.sck-helper.samples")
private let stateQueue = DispatchQueue(label: "app.openscreen.sck-helper.state")
private var stream: SCStream?
/// System audio for a window take, from a display-wide stream. See `recordsSystemAudioFromDisplay`.
private var systemAudioStream: SCStream?
private var writer: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
// One AAC track, never one per source: the editor preview is an HTML5 <video>, which plays
// audio track 0 and nothing else. See AudioTrackMixer.
private var audioInput: AVAssetWriterInput?
private var audioMixer: AudioTrackMixer?
/// Drives the mixer's cursor while nothing is arriving to drive it. Owned by the sample
/// queue: created when the writer session opens, cancelled on the same queue at teardown.
private var audioTicker: DispatchSourceTimer?
private var didStartWriting = false
private var didEmitRecordingStarted = false
private var didReportWriterFailure = false
/// The one shutdown, however many callers ask for it. A writer failure starts it on
/// its own task, and the `stop` Electron sends right after has to wait for that run:
/// the command loop exits the process as soon as `stop()` returns, which would cut
/// `finishWriter()` off before its terminal event.
private var shutdownTask: Task<Void, Never>?
/// Direct display captures only. See `startExclusionRefresh`.
private var exclusionRefreshTask: Task<Void, Never>?
private var isPaused = false
private var pauseStartedAt: CMTime?
private var totalPausedDuration = CMTime.zero
/// Sample queue only. See `VideoTimestampGate` for the failure it exists to prevent.
private var videoTimestampGate = VideoTimestampGate()
/// Sample queue only. Frames the writer input was not ready for, or refused, reported in
/// `recording-stopped` so a macOS drop rate can be measured at all (#937).
private var droppedVideoFrames = 0
private var nativeMicrophoneEnabled = false
private var outputWidth = 1920
private var outputHeight = 1080
private var captureFrame = CGRect.zero
private let microphoneOutputTypeRawValue = 2
private let hostClock = CMClockGetHostTimeClock()
private let picked: PickedSource?
/// A `SystemAudioTap` (macOS 14.2+), held untyped so this class keeps its macOS 13 floor.
/// Sample queue only, like the microphone buffers whose arrival starts it.
private var systemAudioTap: AnyObject?
/// Sample queue only. When the tap starts anyway if the microphone has not delivered yet.
private var tapFallbackDeadline: Date?
/// Sample queue only. Cleared before the writer is finalised: a tap stops on its own
/// queue, and a buffer it had in flight must not reach a finished writer input.
private var acceptsTappedAudio = true
private let tapControlQueue = DispatchQueue(label: "app.openscreen.sck-helper.system-audio-control")
/// `picked` is a choice already made in Apple's system picker (`PickerSession`). The take
/// then records exactly that filter: no source lookup, and no Screen Recording check,
/// because the user's pick IS the consent -- asking for the grant here would put back
/// the very prompt the picker path exists to remove.
init(request: RecordingRequest, picked: PickedSource? = nil) {
self.request = request
self.picked = picked
}
/// Whether this take's system audio comes from a Core Audio tap instead of ScreenCaptureKit.
///
/// Only for a source from Apple's picker: that capture holds no Screen Recording grant,
/// and ScreenCaptureKit hands a grantless capture its system audio as silence. Every other
/// source keeps ScreenCaptureKit's audio, which its grant already covers.
private var tapsSystemAudio: Bool {
guard picked != nil, request.audio.system.enabled else {
return false
}
if #available(macOS 14.2, *) {
return true
}
return false
}
/// Whether this take's system audio comes from a second, display-wide stream.
///
/// A window filter (`SCContentFilter(desktopIndependentWindow:)`) hands ScreenCaptureKit's
/// audio over from the window's own application only, so a Finder window recorded while a
/// video plays elsewhere came out silent (#1216). "System audio" means the whole mix, as
/// on Windows and in a display take, so a window take takes its audio from a stream on a
/// display with nothing excluded. The Screen Recording grant the window capture already
/// holds covers it: no new permission and no new prompt.
private var recordsSystemAudioFromDisplay: Bool {
picked == nil && request.source.type == "window" && request.audio.system.enabled
}
func start() async throws {
try ensureRequestedPermissions(screen: picked == nil)
let target: CaptureTarget
var audioStream: SCStream?
if let picked {
target = makeCaptureTarget(picked: picked)
} else {
let content = try await SCShareableContent.excludingDesktopWindows(
false,
onScreenWindowsOnly: true
)
target = try makeCaptureTarget(from: content)
if recordsSystemAudioFromDisplay, let display = content.displays.first {
let filter = SCContentFilter(display: display, excludingApplications: [], exceptingWindows: [])
let candidate = SCStream(filter: filter, configuration: makeSystemAudioStreamConfiguration(), delegate: self)
do {
try candidate.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
audioStream = candidate
} catch {
// Like a failed start below: the take goes on, without its system audio.
emit([
"event": "warning",
"code": "system-audio-unavailable",
"message": "System audio could not be captured: \(error)",
])
}
}
}
outputWidth = target.width
outputHeight = target.height
captureFrame = target.captureFrame
let configuration = makeStreamConfiguration()
let stream = SCStream(filter: target.filter, configuration: configuration, delegate: self)
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: sampleQueue)
if request.audio.system.enabled && !tapsSystemAudio && !recordsSystemAudioFromDisplay {
try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: sampleQueue)
}
if nativeMicrophoneEnabled {
guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else {
throw HelperError.unsupportedFeature(
"Native microphone capture requires a macOS version with ScreenCaptureKit microphone output."
)
}
try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: sampleQueue)
}
try setupWriter()
self.stream = stream
systemAudioStream = audioStream
emit([
"event": "ready",
"schemaVersion": 1,
])
try await stream.startCapture()
// On an idle Mac the tap's device does not run until something plays, and a tap
// started before the stream's microphone held the microphone back with it until then
// (#1173). So a take with a microphone starts its tap on the first microphone buffer;
// the deadline is the fallback for a microphone that never delivers.
if tapsSystemAudio {
sampleQueue.sync {
if nativeMicrophoneEnabled {
tapFallbackDeadline = Date().addingTimeInterval(1)
} else {
startSystemAudioTapIfNeeded()
}
}
}
if let audioStream {
do {
try await audioStream.startCapture()
} catch {
systemAudioStream = nil
emit([
"event": "warning",
"code": "system-audio-unavailable",
"message": "System audio could not be captured: \(error)",
])
}
}
if picked == nil, request.source.type == "display", let displayID = request.source.displayId {
startExclusionRefresh(displayID: displayID)
}
}
func stop() async {
let task = stateQueue.sync { () -> Task<Void, Never> in
if let shutdownTask {
return shutdownTask
}
let task = Task {
await self.performStop()
}
shutdownTask = task
return task
}
await task.value
}
private func performStop() async {
exclusionRefreshTask?.cancel()
do {
try await stream?.stopCapture()
} catch {
emit([
"event": "warning",
"code": "stop-capture-failed",
"message": "\(error)",
])
}
// Stopped before the writer finishes, like the tap: no audio after the last frame.
try? await systemAudioStream?.stopCapture()
systemAudioStream = nil
stopSystemAudioTap()
await finishWriter()
}
/// Sample queue. A microphone that never delivers must not cost the take its system audio.
private func startSystemAudioTapIfMicrophoneSilent() {
guard let deadline = tapFallbackDeadline, Date() >= deadline else {
return
}
tapFallbackDeadline = nil
startSystemAudioTapIfNeeded()
}
/// Sample queue. Starts the tap once; a no-op after that, and after the take has stopped.
private func startSystemAudioTapIfNeeded() {
guard tapsSystemAudio, systemAudioTap == nil, acceptsTappedAudio, #available(macOS 14.2, *)
else {
return
}
let tap = SystemAudioTap { [weak self] sampleBuffer in
guard let self else {
return
}
self.sampleQueue.async {
self.ingestTappedSystemAudio(sampleBuffer)
}
}
systemAudioTap = tap
// Off the start path: the first time, starting a tap blocks until the user answers
// macOS' prompt, and the app gives a take only seconds to report it has started.
// Audio that arrives late lands where it belongs, with silence in front of it.
tapControlQueue.async {
do {
try tap.start()
} catch {
emit([
"event": "warning",
"code": "system-audio-unavailable",
"message": "System audio could not be captured: \(error)",
])
}
}
}
/// Queued behind the start rather than run beside it: a tap stopped while its start is
/// still inside AudioDeviceStart is torn down under its own feet. Not awaited, so a
/// prompt left unanswered cannot hold the recording's stop hostage; what the tap still
/// delivers is dropped by `acceptsTappedAudio`.
private func stopSystemAudioTap() {
let tap = sampleQueue.sync { () -> AnyObject? in
acceptsTappedAudio = false
defer { systemAudioTap = nil }
return systemAudioTap
}
guard let tap else {
return
}
tapControlQueue.async {
if #available(macOS 14.2, *), let tap = tap as? SystemAudioTap {
tap.stop()
}
}
}
/// Sample queue. The tap's audio takes the same pause and retime path ScreenCaptureKit's
/// system audio took in `stream(_:didOutputSampleBuffer:of:)`.
private func ingestTappedSystemAudio(_ sampleBuffer: CMSampleBuffer) {
guard acceptsTappedAudio else {
return
}
let pauseState = currentPauseState()
if pauseState.paused {
return
}
guard let sampleBuffer = retimedSampleBuffer(sampleBuffer, subtracting: pauseState.offset) else {
return
}
audioMixer?.ingest(sampleBuffer, from: .system)
}
func pause() {
let didPause = stateQueue.sync {
if shutdownTask != nil || isPaused {
return false
}
isPaused = true
pauseStartedAt = CMClockGetTime(hostClock)
return true
}
if didPause {
emit([
"event": "recording-paused",
"timestampMs": Int(Date().timeIntervalSince1970 * 1000),
])
}
}
func resume() {
let didResume = stateQueue.sync {
if shutdownTask != nil || !isPaused {
return false
}
if let pauseStartedAt {
let now = CMClockGetTime(hostClock)
totalPausedDuration = CMTimeAdd(
totalPausedDuration,
CMTimeSubtract(now, pauseStartedAt)
)
}
isPaused = false
pauseStartedAt = nil
return true
}
if didResume {
emit([
"event": "recording-resumed",
"timestampMs": Int(Date().timeIntervalSince1970 * 1000),
])
}
}
/// The effect belongs to the screen stream, never the auxiliary audio stream.
@available(macOS 14.0, *)
func outputVideoEffectDidStart(for stream: SCStream) {
guard stream === self.stream else { return }
emit(["event": "warning", "code": "presenter-overlay-started",
"message": "macOS Presenter Overlay started."])
}
/// Stopping the effect does not establish that a separate camera track recovered.
@available(macOS 14.0, *)
func outputVideoEffectDidStop(for stream: SCStream) {
guard stream === self.stream else { return }
emit(["event": "warning", "code": "presenter-overlay-stopped",
"message": "macOS Presenter Overlay stopped."])
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
// Losing the audio-only stream costs the take its system audio, not the take.
if stream === systemAudioStream {
systemAudioStream = nil
emit([
"event": "warning",
"code": "system-audio-unavailable",
"message": "System audio stopped: \(error)",
])
return
}
emitError(code: "capture-stopped-with-error", message: "\(error)")
Task {
await stop()
}
}
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
if type.rawValue == microphoneOutputTypeRawValue {
startSystemAudioTapIfNeeded()
}
guard CMSampleBufferDataIsReady(sampleBuffer) else {
return
}
let pauseState = currentPauseState()
if pauseState.paused {
return
}
guard let sampleBuffer = retimedSampleBuffer(sampleBuffer, subtracting: pauseState.offset) else {
return
}
if type == .audio {
audioMixer?.ingest(sampleBuffer, from: .system)
return
}
if type.rawValue == microphoneOutputTypeRawValue {
audioMixer?.ingest(sampleBuffer, from: .microphone)
return
}
guard type == .screen else {
return
}
guard isCompleteFrame(sampleBuffer) else {
return
}
guard let videoInput, let writer else {
return
}
let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
// A frame that does not move the timeline forward fails the whole writer one append
// later (-16364), so it is dropped here instead. After a resume that is one frame the
// user never sees; without the check it was the end of the recording.
switch videoTimestampGate.check(presentationTime) {
case .admit:
break
case .invalid:
reportRefusedVideoFrame(presentationTime, previous: nil, pauseOffset: pauseState.offset)
return
case .notAfterPrevious(let previous):
reportRefusedVideoFrame(presentationTime, previous: previous, pauseOffset: pauseState.offset)
return
}
if !didStartWriting {
writer.startWriting()
writer.startSession(atSourceTime: presentationTime)
didStartWriting = true
audioMixer?.beginTimeline(at: presentationTime)
startAudioTicker()
}
// Checked before the readiness gate, not only at a false append: a failed
// writer is not guaranteed to call itself ready, and an append that is never
// attempted can never report the failure (issue #621).
if writer.status == .failed {
reportWriterFailure("writer status")
return
}
if videoInput.isReadyForMoreMediaData {
videoTimestampGate.record(presentationTime)
let appended = videoInput.append(sampleBuffer)
if appended, !didEmitRecordingStarted {
didEmitRecordingStarted = true
emit([
"event": "recording-started",
"timestampMs": Int(Date().timeIntervalSince1970 * 1000),
"width": outputWidth,
"height": outputHeight,
"captureBounds": captureBoundsPayload(),
])
} else if !appended {
droppedVideoFrames += 1
reportWriterFailure("video append")
}
} else {
droppedVideoFrames += 1
}
}
/// A failed AVAssetWriter keeps accepting appends and keeps answering false, so
/// a recorder that discards that Bool records nothing while the HUD counts on.
/// That is how a two-minute take was already lost by its fourth second and only
/// said so at finishWriting(). The Windows helper checks every WriteSample
/// HRESULT and escalates; this is the macOS half of the same contract -- report
/// once, at the append that actually failed, carrying the live writer.error.
///
/// Deliberately not the code finishWriter() emits, and the difference is load
/// bearing. That one is the terminal result of stopping, and the Electron side
/// settles its stop on exactly one of `recording-stopped` or `writer-failed`.
/// Give both sites the same code behind this one-shot guard and a writer that
/// died mid-capture emits nothing at all at stop, so the stop promise never
/// settles and every failure becomes the "Saving..." hang instead of an error.
/// This event answers "when did the writer die"; that one answers "did stopping
/// work". Two questions, two codes.
///
/// It also ends the capture, the way `didStopWithError` does. A failed writer
/// never recovers, so every frame after it is dropped; issue #621 is a take
/// whose writer died at 75 s while capture ran on for 22 more minutes. The
/// process stays up and still answers `stop`, so the Electron stop path works
/// unchanged and reads this event back as the reason.
private func reportWriterFailure(_ stage: String) {
guard !didReportWriterFailure, let writer else {
return
}
didReportWriterFailure = true
emitError(
code: "writer-failed-during-capture",
message: "Recording stopped: the video file could not be written (\(stage): "
+ (writer.error.map { "\($0)" }
?? "AVAssetWriter status \(writer.status.rawValue)")
+ ").",
)
Task {
await stop()
}
}
/// Once per take, with the numbers that tell the cause apart. A refusal with a non-zero pause
/// offset, a few milliseconds behind the previous frame, is the pause shift measured in
/// `VideoTimestampGate`. One with no pause offset at all is a source handing over time that
/// goes backwards, which nothing here has observed yet and is worth a report.
private func reportRefusedVideoFrame(_ presentationTime: CMTime, previous: CMTime?, pauseOffset: CMTime) {
guard videoTimestampGate.rejectedCount == 1 else {
return
}
emit([
"event": "warning",
"code": "video-frame-timestamp-refused",
"message": "Dropped a video frame whose timestamp did not advance.",
"presentationTimeSeconds": presentationTime.isNumeric ? CMTimeGetSeconds(presentationTime) : -1,
"previousSeconds": previous.map { CMTimeGetSeconds($0) } ?? -1,
"pauseOffsetSeconds": pauseOffset.isNumeric ? CMTimeGetSeconds(pauseOffset) : 0,
])
}
private func ensureRequestedPermissions(screen: Bool) throws {
if screen && !CGPreflightScreenCaptureAccess() {
let granted = CGRequestScreenCaptureAccess()
if !granted {
throw HelperError.permissionDenied("Screen recording permission is required for ScreenCaptureKit capture.")
}
}
if request.audio.microphone.enabled {
switch AVCaptureDevice.authorizationStatus(for: .audio) {
case .authorized:
break
case .notDetermined:
let semaphore = DispatchSemaphore(value: 0)
AVCaptureDevice.requestAccess(for: .audio) { _ in
semaphore.signal()
}
let waitResult = semaphore.wait(timeout: .now() + 30)
if waitResult == .timedOut || AVCaptureDevice.authorizationStatus(for: .audio) != .authorized {
throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.")
}
default:
throw HelperError.permissionDenied("Microphone permission is required for native microphone capture.")
}
}
}
private func captureBoundsPayload() -> [String: Double] {
return [
"x": captureFrame.origin.x,
"y": captureFrame.origin.y,
"width": captureFrame.size.width,
"height": captureFrame.size.height,
]
}
private func makeCaptureTarget(picked: PickedSource) -> CaptureTarget {
let size = captureSize(
for: picked.filter,
fallbackPointSize: picked.frame.size,
fallbackDisplayId: picked.displayId ?? CGMainDisplayID()
)
return CaptureTarget(
filter: picked.filter,
width: size.width,
height: size.height,
captureFrame: picked.frame
)
}
/// The app's own windows (HUD, notes) plus the system ones a demo never wants.
private func excludedWindowIDs(in content: SCShareableContent) -> [UInt32] {
let candidates = content.windows.map {
CaptureWindowCandidate(
windowID: $0.windowID,
bundleID: $0.owningApplication?.bundleIdentifier,
layer: $0.windowLayer
)
}
return resolveCaptureExcludedWindowIDs(
requestedWindowIDs: (request.excludedWindowIds ?? [])
+ systemCaptureExcludedWindowIDs(
candidates,
hideDesktopIcons: request.hideDesktopIcons ?? false
),
availableWindowIDs: content.windows.map(\.windowID)
)
}
private func displayFilter(
_ display: SCDisplay,
in content: SCShareableContent,
excluding windowIDs: [UInt32]
) -> SCContentFilter {
let excluded = Set(windowIDs)
return SCContentFilter(
display: display,
excludingWindows: content.windows.filter { excluded.contains($0.windowID) }
)
}
/// A filter excludes windows, not apps, and a notification banner is a window that did
/// not exist when the take started. So the exclusion is re-read while recording and the
/// filter swapped whenever it changes. ponytail: a 250 ms poll, so a banner can show for
/// up to a quarter second as it slides in; the picker path excludes Notification Center
/// by bundle id and has no such gap.
private func startExclusionRefresh(displayID: CGDirectDisplayID) {
exclusionRefreshTask = Task { [weak self] in
var current: [UInt32] = []
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: 250_000_000)
guard let self, let stream = self.stream,
let content = try? await SCShareableContent.excludingDesktopWindows(
false,
onScreenWindowsOnly: true
),
let display = content.displays.first(where: { $0.displayID == displayID })
else {
continue
}
let next = self.excludedWindowIDs(in: content)
guard Set(next) != Set(current) else {
continue
}
current = next
try? await stream.updateContentFilter(
self.displayFilter(display, in: content, excluding: next)
)
}
}
}
private func makeCaptureTarget(from content: SCShareableContent) throws -> CaptureTarget {
switch request.source.type {
case "display":
guard let displayId = request.source.displayId else {
throw HelperError.sourceNotFound("Display capture requires source.displayId.")
}
guard let display = content.displays.first(where: { $0.displayID == displayId }) else {
throw HelperError.sourceNotFound("No ScreenCaptureKit display found for id \(displayId).")
}
let requestedWindowIDs = request.excludedWindowIds ?? []
let resolvedWindowIDs = excludedWindowIDs(in: content)
let filter = displayFilter(display, in: content, excluding: resolvedWindowIDs)
emit([
"event": "capture-window-exclusion",
"requestedWindowIds": requestedWindowIDs,
"resolvedWindowIds": resolvedWindowIDs,
"excludedWindowCount": resolvedWindowIDs.count,
])
let size = captureSize(
for: filter,
fallbackPointSize: display.frame.size,
fallbackDisplayId: display.displayID
)
return CaptureTarget(
filter: filter,
width: size.width,
height: size.height,
captureFrame: display.frame
)
case "window":
guard let windowId = request.source.windowId else {
throw HelperError.sourceNotFound("Window capture requires source.windowId.")
}
guard let window = content.windows.first(where: { $0.windowID == windowId }) else {
throw HelperError.sourceNotFound("No ScreenCaptureKit window found for id \(windowId).")
}
let candidateDisplay = content.displays.first {
$0.frame.intersects(window.frame) || $0.frame.contains(CGPoint(x: window.frame.midX, y: window.frame.midY))
}
let filter = SCContentFilter(desktopIndependentWindow: window)
let size = captureSize(
for: filter,
fallbackPointSize: window.frame.size,
// Unrelated to `initializeCoreGraphicsWindowServerConnection()`: that call
// exists purely for its side effect at process startup, this one wants the
// actual display ID as a fallback when no display intersects the window.
fallbackDisplayId: candidateDisplay?.displayID ?? CGMainDisplayID()
)
return CaptureTarget(
filter: filter,
width: size.width,
height: size.height,
captureFrame: window.frame
)
default:
throw HelperError.invalidSourceType(request.source.type)
}
}
/// The audio-only stream of a window take: the smallest frame ScreenCaptureKit allows, at
/// one a second, since only its audio is read (it has no `.screen` output).
private func makeSystemAudioStreamConfiguration() -> SCStreamConfiguration {
let configuration = SCStreamConfiguration()
configuration.width = 2
configuration.height = 2
configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
configuration.queueDepth = 3
configuration.capturesAudio = true
configuration.sampleRate = 48_000
configuration.channelCount = 2
configuration.excludesCurrentProcessAudio = true
return configuration
}
private func makeStreamConfiguration() -> SCStreamConfiguration {
let configuration = SCStreamConfiguration()
configuration.width = outputWidth
configuration.height = outputHeight
// Belt and braces for the defect `captureOutputSize` exists to prevent. Left at its
// default of `false`, ScreenCaptureKit "only scales down" (SDK header), so any frame
// smaller than the buffer is drawn at native size in a corner and the rest stays
// background black — a whole recording letterboxed inside its own frame (issue #418).
// With the buffer now derived from the filter the two agree and this changes nothing;
// it is here so that a future source whose size we mispredict comes out scaled rather
// than cornered.
configuration.scalesToFit = true
configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps)))
configuration.queueDepth = 6
configuration.showsCursor = !request.video.hideSystemCursor
// Studio-range BT.709 YCbCr from ScreenCaptureKit itself, which the encoder takes as is
// and the compositor decodes (#943). BGRA left the matrix to VideoToolbox. Nothing here
// reads the pixels: a frame goes through `retimedSampleBuffer` (timing only) to
// `append`, and `isCompleteFrame` reads attachments. 1.5 bytes a pixel instead of 4,
// times `queueDepth`.
configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
configuration.colorMatrix = captureYCbCrMatrix
// Unset, the buffers carry the display's colour space: P3 on most Macs.
configuration.colorSpaceName = CGColorSpace.sRGB
configuration.sampleRate = 48_000
configuration.channelCount = 2
configuration.excludesCurrentProcessAudio = true
configuration.capturesAudio =
request.audio.system.enabled && !tapsSystemAudio && !recordsSystemAudioFromDisplay
nativeMicrophoneEnabled = false
guard request.audio.microphone.enabled else {
return configuration
}
let microphone = resolveMicrophoneCapture(
supported: supportsNativeMicrophoneCapture(streamConfig: configuration),
deviceID: request.audio.microphone.deviceId,
deviceName: request.audio.microphone.deviceName,
devices: AVCaptureDevice.devices(for: .audio).map {
(id: $0.uniqueID, name: $0.localizedName)
}
)
switch microphone {
case .unsupported:
emit([
"event": "warning",
"code": "microphone-unavailable",
"message": "Native microphone capture requires ScreenCaptureKit microphone support on this macOS version.",
])
case .noInput:
emit([
"event": "warning",
"code": "microphone-not-found",
"message": "No audio input is connected; recording without the microphone.",
])
case .device(let deviceId):
nativeMicrophoneEnabled = true
configuration.capturesAudio = true
configuration.setValue(true, forKey: "captureMicrophone")
if let deviceId {
configuration.setValue(deviceId, forKey: "microphoneCaptureDeviceID")
} else if requestedASpecificMicrophone {
emit([
"event": "warning",
"code": "microphone-defaulted",
"message": "The requested microphone could not be resolved; capturing the default input.",
])
}
}
return configuration
}
private func setupWriter() throws {
let outputUrl = URL(fileURLWithPath: request.outputs.screenPath)
try? FileManager.default.removeItem(at: outputUrl)
try FileManager.default.createDirectory(
at: outputUrl.deletingLastPathComponent(),
withIntermediateDirectories: true
)
let writer = try AVAssetWriter(outputURL: outputUrl, fileType: .mp4)
// Costs nothing on a clean stop -- finishWriting() still writes a normal
// moov -- and is the difference between a readable file and a total loss
// when the helper dies before reaching it. The Windows helper gets the
// same property from MFCreateFMPEG4MediaSink; see issues #252/#292/#327.
writer.movieFragmentInterval = CMTime(seconds: 1, preferredTimescale: 600)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: outputWidth,
AVVideoHeightKey: outputHeight,
// BT.709 tags, the colour the stream is captured in (#943).
AVVideoColorPropertiesKey: videoColorProperties,
AVVideoCompressionPropertiesKey: [
// From the size this stream really got. The renderer sends none (#924).
AVVideoAverageBitRateKey: request.video.bitrate
?? defaultVideoBitrate(width: outputWidth, height: outputHeight, fps: request.video.fps),
AVVideoExpectedSourceFrameRateKey: request.video.fps,
// Without this the encoder defaults to B-frames, and a reordered
// stream needs a composition offset per sample. AVAssetWriter emits
// those in a version 0 `trun`, where ISO/IEC 14496-12 8.8.8.2 defines
// the field as UNSIGNED -- so a negative offset goes out as
// 0xFFFFFFF6 and the fragment writer refuses the fragment it is
// about to emit. That refusal is -11800 / -16341, raised from the
// single site in MediaToolbox that writes moof/traf/trun, which is
// why it appears if and only if movieFragmentInterval is set and
// lands exactly on a fragment boundary.
//
// Turning reordering off makes every offset zero and PTS == DTS, so
// the fragment stays representable. A screen recorder gives up
// nothing for it: B-frames buy compression on lookahead-friendly
// content and cost encode latency, which is the wrong trade for
// real-time capture.
//
// Measured on macOS 26.5 / M1, 1080p with system audio. How reliably
// the bug bites scales with append rate, so quote the rate with the
// result: at ~57 fps, the rate the app actually drives, reordering
// on dies at 13.0s while reordering off stops clean at 31.6s; at
// 30 fps it is intermittent, dying at 1.0s and 2.0s but once
// surviving 22.2s. That intermittency is why the byte-level evidence
// leads here and the run counts only corroborate: the offsets are
// out of spec in every fragmented file whether or not that
// particular run happened to die. Reordering off is 3/3 clean across
// both rates, and a SIGKILL at 25s still leaves 27 readable `moof`.
AVVideoAllowFrameReorderingKey: false,
// A keyframe every second, by frame count and by time: a still screen
// delivers fewer frames than the fps, and the count alone would then
// stretch the GOP, which is the editor's scrub cost (#937).
AVVideoMaxKeyFrameIntervalKey: videoKeyFrameInterval(fps: request.video.fps),
AVVideoMaxKeyFrameIntervalDurationKey: 1,
],
]
let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
input.expectsMediaDataInRealTime = true
guard writer.canAdd(input) else {
throw HelperError.writerSetupFailed("Unable to add H.264 video input to AVAssetWriter.")
}
writer.add(input)
self.writer = writer
self.videoInput = input
// A single mixed AAC track at the bitrate Windows encodes its own mixed track with
// (24 000 B/s), so the two platforms produce comparable files.
if request.audio.system.enabled || nativeMicrophoneEnabled {
let input = try addAudioInput(to: writer, bitRate: 192_000)
audioInput = input
audioMixer = AudioTrackMixer(
input: input,
includesSystemAudio: request.audio.system.enabled,
includesMicrophone: nativeMicrophoneEnabled,
microphoneGain: request.audio.microphone.gain,
clock: { [weak self] in self?.timelineNow() ?? .invalid }
)
}
}
/// The instant the writer's timeline has reached: the host clock, less the time spent
/// paused — the same transform `retimedSampleBuffer` applies to every sample, so this and
/// the presentation timestamps are answers in one domain.
///
/// Frozen while paused, because `pauseStartedAt` stops moving and `totalPausedDuration`
/// does not yet include the pause in progress. On resume the offset grows by exactly the
/// pause, so the position continues from where it froze: the pause interrupts the audio
/// clock without shifting anything recorded after it. That is the same property the
/// Windows mixer gets by re-deriving its anchor on resume, derived here instead of tracked.
private func timelineNow() -> CMTime {
let (paused, offset, pausedAt) = stateQueue.sync {
(isPaused, totalPausedDuration, pauseStartedAt)
}
let now = paused ? (pausedAt ?? CMClockGetTime(hostClock)) : CMClockGetTime(hostClock)
return CMTimeSubtract(now, offset)
}
/// A take that nothing is playing into is exactly the take whose audio timeline has to keep
/// moving, so the mixer cannot be driven by buffer arrival alone. 10 ms is the chunk size it
/// emits at; the leeway lets the system coalesce the wakeups, since being a few milliseconds
/// late only means the next tick emits two chunks instead of one.
private func startAudioTicker() {
guard audioMixer != nil, audioTicker == nil else {
return
}
let timer = DispatchSource.makeTimerSource(queue: sampleQueue)
timer.schedule(
deadline: .now() + .milliseconds(10),
repeating: .milliseconds(10),
leeway: .milliseconds(5)