Fix capturer state desync when startCapture fails after the base state transition

.changes/capturer-start-failure-balance

@@ -0,0 +1 @@
+patch type="fixed" "Fix capturers permanently reporting started after a failed startCapture: the start/stop counter is now balanced on failure so a subsequent start attempt can retry"

Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift

@@ -45,7 +45,14 @@ class BroadcastScreenCapturer: BufferCapturer, @unchecked Sendable {
             .toEncodeSafeDimensions()
 
         set(dimensions: targetDimensions)
-        return createReceiver()
+
+        guard createReceiver() else {
+            // Balance the counter so the capturer does not report `.started`
+            // while no capture is running.
+            _ = try? await stopCapture()
+            return false
+        }
+        return true
     }
 
     private func createReceiver() -> Bool {

Sources/LiveKit/Track/Capturers/ARCameraCapturer.swift

@@ -37,6 +37,16 @@
     }
 
     override public func startCapture() async throws -> Bool {
+        do {
+            return try await performStartCapture()
+        } catch {
+            // Balance the counter so a subsequent startCapture() can retry.
+            try? await stopCapture()
+            throw error
+        }
+    }
+
+    private func performStartCapture() async throws -> Bool {
         let didStart = try await super.startCapture()
         // Already started
         guard didStart else { return false }
@@ -55,7 +65,7 @@
             guard let self else { return }
             for await frame in frameUpdates {
                 if let sample = frame.sample(for: .left) {
                     capture(pixelBuffer: sample.pixelBuffer, capturer: capturer, options: options)
                 }
             }
         }.cancellable()

Sources/LiveKit/Track/Capturers/CameraCapturer.swift

@@ -151,8 +151,18 @@
         return try await restartCapture()
     }
 
-    // swiftlint:disable:next cyclomatic_complexity function_body_length
     override public func startCapture() async throws -> Bool {
+        do {
+            return try await performStartCapture()
+        } catch {
+            // Balance the counter so a subsequent startCapture() can retry.
+            try? await stopCapture()
+            throw error
+        }
+    }
+
+    // swiftlint:disable:next cyclomatic_complexity function_body_length
+    private func performStartCapture() async throws -> Bool {
         let didStart = try await super.startCapture()
 
         // Already started

Sources/LiveKit/Track/Capturers/InAppCapturer.swift

@@ -33,6 +33,16 @@
     }
 
     override public func startCapture() async throws -> Bool {
+        do {
+            return try await performStartCapture()
+        } catch {
+            // Balance the counter so a subsequent startCapture() can retry.
+            try? await stopCapture()
+            throw error
+        }
+    }
+
+    private func performStartCapture() async throws -> Bool {
         let didStart = try await super.startCapture()
 
         // Already started

Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift

@@ -58,6 +58,16 @@
     }
 
     override public func startCapture() async throws -> Bool {
+        do {
+            return try await performStartCapture()
+        } catch {
+            // Balance the counter so a subsequent startCapture() can retry.
+            try? await stopCapture()
+            throw error
+        }
+    }
+
+    private func performStartCapture() async throws -> Bool {
         let didStart = try await super.startCapture()
 
         // Already started

Tests/LiveKitCoreTests/VideoCapturerStartFailureTests.swift

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+@testable import LiveKit
+import LiveKitWebRTC
+import Testing
+
+/// Emulates a capturer whose device-level start work fails after
+/// `super.startCapture()` has already transitioned the state to `.started`,
+/// using the same balancing pattern as the built-in capturers.
+private final class StartFailingCapturer: VideoCapturer, @unchecked Sendable {
+    override func startCapture() async throws -> Bool {
+        do {
+            let didStart = try await super.startCapture()
+            guard didStart else { return false }
+            throw LiveKitError(.invalidState, message: "Simulated device failure")
+        } catch {
+            try? await stopCapture()
+            throw error
+        }
+    }
+}
+
+private final class NullCapturerDelegate: NSObject, LKRTCVideoCapturerDelegate {
+    func capturer(_: LKRTCVideoCapturer, didCapture _: LKRTCVideoFrame) {}
+}
+
+@Suite(.tags(.media))
+struct VideoCapturerStartFailureTests {
+    @Test func startFailureLeavesCapturerStopped() async throws {
+        let capturer = StartFailingCapturer(delegate: NullCapturerDelegate())
+        #expect(capturer.captureState == .stopped)
+
+        await #expect(throws: LiveKitError.self) {
+            try await capturer.startCapture()
+        }
+        // Without balancing the counter stays at 1: the capturer reports
+        // `.started` while nothing is capturing.
+        #expect(capturer.captureState == .stopped)
+    }
+
+    @Test func startCanRetryAfterFailure() async throws {
+        let capturer = StartFailingCapturer(delegate: NullCapturerDelegate())
+
+        await #expect(throws: LiveKitError.self) {
+            try await capturer.startCapture()
+        }
+        // Without balancing this second call would return `false` early
+        // ("already started") instead of reaching the subclass again.
+        await #expect(throws: LiveKitError.self) {
+            try await capturer.startCapture()
+        }
+        #expect(capturer.captureState == .stopped)
+    }
+}