From 9967a2e110e2864f42e136da921df82d3541f34d Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 13:26:55 -0400 Subject: [PATCH 1/8] Start printing errors for concurrency issues --- Package.swift | 13 +++++++++++-- Sources/FluidAudio/Shared/AppLogger.swift | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index cebb1bb9..999f77f7 100644 --- a/Package.swift +++ b/Package.swift @@ -30,7 +30,10 @@ let package = Package( "FastClusterWrapper", ], path: "Sources/FluidAudio", - exclude: ["Frameworks"] + exclude: ["Frameworks"], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ] ), .target( name: "FastClusterWrapper", @@ -44,11 +47,17 @@ let package = Package( exclude: ["README.md"], resources: [ .process("Utils/english.json") + ], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), ] ), .testTarget( name: "FluidAudioTests", - dependencies: ["FluidAudio"] + dependencies: ["FluidAudio"], + swiftSettings: [ + .enableUpcomingFeature("StrictConcurrency"), + ] ), ], cxxLanguageStandard: .cxx17 diff --git a/Sources/FluidAudio/Shared/AppLogger.swift b/Sources/FluidAudio/Shared/AppLogger.swift index bf4f745a..c69c4547 100644 --- a/Sources/FluidAudio/Shared/AppLogger.swift +++ b/Sources/FluidAudio/Shared/AppLogger.swift @@ -6,7 +6,7 @@ import OSLog public struct AppLogger { /// Default subsystem for all loggers in FluidAudio. /// Keep this consistent; categories should vary per component. - public static var defaultSubsystem: String = "com.fluidinference" + public static let defaultSubsystem: String = "com.fluidinference" public enum Level: Int { case debug = 0 From 9c77d20ed5baff20d64b2e390209e06ce9c1eadc Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 14:01:01 -0400 Subject: [PATCH 2/8] sendable app logger --- Sources/FluidAudio/Shared/AppLogger.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/FluidAudio/Shared/AppLogger.swift b/Sources/FluidAudio/Shared/AppLogger.swift index c69c4547..24cd2e09 100644 --- a/Sources/FluidAudio/Shared/AppLogger.swift +++ b/Sources/FluidAudio/Shared/AppLogger.swift @@ -3,7 +3,7 @@ import OSLog /// Lightweight logger that writes to Unified Logging and, optionally, to console. /// Use this instead of `OSLog.Logger` so CLI runs can surface logs without `print`. -public struct AppLogger { +public struct AppLogger: Sendable { /// Default subsystem for all loggers in FluidAudio. /// Keep this consistent; categories should vary per component. public static let defaultSubsystem: String = "com.fluidinference" From 77e97e9f382fc51ef1b89767f201504084065767 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 14:14:12 -0400 Subject: [PATCH 3/8] ignore cli for now --- Package.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Package.swift b/Package.swift index 999f77f7..31e7a721 100644 --- a/Package.swift +++ b/Package.swift @@ -47,9 +47,6 @@ let package = Package( exclude: ["README.md"], resources: [ .process("Utils/english.json") - ], - swiftSettings: [ - .enableUpcomingFeature("StrictConcurrency"), ] ), .testTarget( From a04b4d54352b495424ce8cdeb47eb69ac7b57e19 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 14:28:41 -0400 Subject: [PATCH 4/8] 735 warnings --- Sources/FluidAudio/ASR/TDT/TdtDecoderState.swift | 2 +- .../FluidAudio/Diarizer/Clustering/SpeakerOperations.swift | 2 +- Sources/FluidAudio/DownloadUtils.swift | 2 +- Sources/FluidAudio/Shared/AppLogger.swift | 5 ++++- Sources/FluidAudio/Shared/AssetDownloader.swift | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Sources/FluidAudio/ASR/TDT/TdtDecoderState.swift b/Sources/FluidAudio/ASR/TDT/TdtDecoderState.swift index e1fa85a4..5287f77a 100644 --- a/Sources/FluidAudio/ASR/TDT/TdtDecoderState.swift +++ b/Sources/FluidAudio/ASR/TDT/TdtDecoderState.swift @@ -3,7 +3,7 @@ import CoreML import Foundation /// Manages LSTM hidden and cell states for the Parakeet decoder -struct TdtDecoderState { +struct TdtDecoderState: Sendable { var hiddenState: MLMultiArray var cellState: MLMultiArray /// Stores the last decoded token from the previous audio chunk. diff --git a/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift b/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift index 83801b48..9098ef2a 100644 --- a/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift +++ b/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift @@ -13,7 +13,7 @@ public enum SpeakerUtilities { // MARK: - Configuration /// Platform-specific configuration for speaker assignment - public struct AssignmentConfig { + public struct AssignmentConfig: Sendable { public let maxDistanceForAssignment: Float public let maxDistanceForUpdate: Float public let minSpeakerDuration: Float diff --git a/Sources/FluidAudio/DownloadUtils.swift b/Sources/FluidAudio/DownloadUtils.swift index a106089b..acec0836 100644 --- a/Sources/FluidAudio/DownloadUtils.swift +++ b/Sources/FluidAudio/DownloadUtils.swift @@ -248,7 +248,7 @@ public class DownloadUtils { public typealias ProgressHandler = (Double) -> Void /// Download configuration - public struct DownloadConfig { + public struct DownloadConfig: Sendable { public let timeout: TimeInterval public init(timeout: TimeInterval = 1800) { // 30 minutes for large models diff --git a/Sources/FluidAudio/Shared/AppLogger.swift b/Sources/FluidAudio/Shared/AppLogger.swift index 24cd2e09..6b795ba9 100644 --- a/Sources/FluidAudio/Shared/AppLogger.swift +++ b/Sources/FluidAudio/Shared/AppLogger.swift @@ -8,7 +8,7 @@ public struct AppLogger: Sendable { /// Keep this consistent; categories should vary per component. public static let defaultSubsystem: String = "com.fluidinference" - public enum Level: Int { + public enum Level: Int, Sendable { case debug = 0 case info case notice @@ -82,6 +82,9 @@ public struct AppLogger: Sendable { } private func logToConsole(_ level: Level, _ message: String) { + let level = level + let category = category + let message = message Task.detached(priority: .utility) { await LogConsole.shared.write(level: level, category: category, message: message) } diff --git a/Sources/FluidAudio/Shared/AssetDownloader.swift b/Sources/FluidAudio/Shared/AssetDownloader.swift index 8b510617..dd5bf0a0 100644 --- a/Sources/FluidAudio/Shared/AssetDownloader.swift +++ b/Sources/FluidAudio/Shared/AssetDownloader.swift @@ -5,8 +5,8 @@ import OSLog /// Provides reusable logic for simple file/data transfers used across FluidAudio modules. public enum AssetDownloader { - public typealias DataWriter = (Data, URL) throws -> Void - public typealias FileMover = (URL, URL) throws -> Void + public typealias DataWriter = @Sendable (Data, URL) throws -> Void + public typealias FileMover = @Sendable (URL, URL) throws -> Void public static let defaultDataWriter: DataWriter = { data, destination in try data.write(to: destination, options: [.atomic]) From 9c712dbec9d4dd97f5170dd67e78dfa8429d5091 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 14:46:43 -0400 Subject: [PATCH 5/8] migrate espeakg2p --- AGENTS.md | 96 +++++++++++++++++++ CLAUDE.md | 29 +++++- .../Kokoro/Assets/Lexicon/EspeakG2P.swift | 41 ++++---- .../Pipeline/Preprocess/KokoroChunker.swift | 36 +++---- .../Synthesize/KokoroSynthesizer.swift | 2 +- 5 files changed, 159 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a0a9261b..6c54352b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,102 @@ swift format --in-place --recursive --configuration .swift-format Sources/ Tests - Thread safety: Use actors, `@MainActor`, or proper locking - never `@unchecked Sendable` - Control flow: Prefer flattened if statements with early returns/continues over nested if statements. Use guard statements and inverted conditions to exit early. Nested if statements should be absolutely avoided. +## Swift 6 Concurrency Migration + +### Common Warning Patterns & Fixes + +#### 1. Non-Sendable Struct Types +When a struct is used across actor boundaries or in concurrent contexts, add `Sendable` conformance: + +```swift +// Before +struct MyConfig { + let timeout: Int +} + +// After +struct MyConfig: Sendable { + let timeout: Int +} +``` + +**Examples from codebase:** +- `TdtDecoderState` (ASR/TDT/TdtDecoderState.swift) - LSTM state management +- `AssignmentConfig` (Diarizer/Clustering/SpeakerOperations.swift) - speaker assignment config +- `DownloadConfig` (DownloadUtils.swift) - download timeout settings + +#### 2. Function Type Aliases +Closure types that cross actor boundaries must be marked `@Sendable`: + +```swift +// Before +public typealias DataWriter = (Data, URL) throws -> Void + +// After +public typealias DataWriter = @Sendable (Data, URL) throws -> Void +``` + +**Applied to:** AssetDownloader.swift - `DataWriter` and `FileMover` typealias + +#### 3. Singleton/Shared Static Properties +Global actor isolation for non-Sendable classes: + +```swift +// Before +static let shared = EspeakG2P() + +// After +@MainActor static let shared = EspeakG2P() +``` + +**Applied to:** EspeakG2P.swift - eSpeak NG wrapper singleton + +#### 4. Mutable Global State (#MutableGlobalVariable) +Mutable static properties require proper synchronization or actor isolation. Options: + +**Option A: Use @MainActor for entire type** +```swift +@MainActor +class MyService { + static var cache: [String: Data] = [:] + static let cacheLock = NSLock() +} +``` + +**Option B: Use actor for concurrent access** +```swift +actor CacheManager { + private var cache: [String: Data] = [:] + + func set(_ key: String, _ value: Data) { + cache[key] = value + } + + func get(_ key: String) -> Data? { + cache[key] + } +} +``` + +**Current cases:** KokoroSynthesizer.swift - voiceEmbeddingPayloads, voiceEmbeddingVectors + +#### 5. Non-Sendable Framework Types (MLMultiArray) +CoreML's `MLMultiArray` doesn't conform to Sendable. When passing across actor boundaries: +- Wrap in a Sendable struct/class +- Use `@MainActor` for related processing +- Create separate Sendable representations for cross-actor data + +### Migration Checklist + +When fixing concurrency warnings: +1. Run `swift build` and capture full warning output +2. Identify warning categories (Sendable, @MainActor, #MutableGlobalVariable, etc.) +3. Start with "low-hanging fruit": simple struct/typealias additions +4. Address mutable state with proper synchronization +5. Handle framework non-Sendable types last (most complex) +6. Run `swift build` again to verify warning reduction +7. Never use `@unchecked Sendable` as a shortcut + ## Clean code - When adding new interfaces, make sure that the API is consistent with the other model managers diff --git a/CLAUDE.md b/CLAUDE.md index 72b50266..07370f2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -232,6 +232,9 @@ FluidAudio/ - **Persistent States**: Decoder states maintained across chunks for streaming - **Memory Management**: Automatic cleanup and ANE optimization - **Parallel Processing**: Multi-stream support for batch operations +- **Swift 6 Strict Concurrency Migration** (in progress): + - **Completed**: TdtDecoderState, AssignmentConfig, DownloadConfig, AssetDownloader, EspeakG2P + - **Remaining**: KokoroSynthesizer mutable state, MLMultiArray non-Sendable issues ### Model Management - **Automatic Downloads**: Models fetched from HuggingFace on first use @@ -297,11 +300,31 @@ The project uses GitHub Actions with the following workflows: 9. **Git Operations**: NEVER run `git push` unless explicitly requested by the user. Only commit when asked. 10. **Code Formatting**: All code must pass swift-format checks before merge +## Swift 6 Concurrency Migration Status + +### Completed Fixes +- **TdtDecoderState** (ASR/TDT/TdtDecoderState.swift): Added `Sendable` conformance for LSTM state across actor boundaries +- **AssignmentConfig** (Diarizer/Clustering/SpeakerOperations.swift): Added `Sendable` conformance for clustering configuration +- **DownloadConfig** (DownloadUtils.swift): Added `Sendable` conformance for download settings +- **AssetDownloader** (Shared/AssetDownloader.swift): Marked `DataWriter` and `FileMover` typealias as `@Sendable` +- **EspeakG2P** (TextToSpeech/Kokoro/Assets/Lexicon/EspeakG2P.swift): Added `@MainActor` to shared singleton + +### Remaining Work +- **KokoroSynthesizer** (TextToSpeech/Kokoro/Pipeline/Synthesize/KokoroSynthesizer.swift): + - Mutable static state: `voiceEmbeddingPayloads`, `voiceEmbeddingVectors` (lines 58-59) + - Requires `@MainActor` annotation or actor-based refactoring + - Multiple MLMultiArray sending warnings (framework limitation) + +### Framework Limitations +- **MLMultiArray** (CoreML): Does not conform to Sendable - wrap in Sendable types for cross-actor use +- **Non-Sendable Types**: Create Sendable wrapper structs when passing non-Sendable data across actor boundaries + ## Next Steps -1. **Multi-file validation**: Test optimal config on all AMI files -2. **Real-world testing**: Validate on non-AMI audio -3. **Documentation**: Update API documentation +1. **Complete Swift 6 Migration**: Finish KokoroSynthesizer mutable state handling +2. **Multi-file validation**: Test optimal config on all AMI files +3. **Real-world testing**: Validate on non-AMI audio +4. **Documentation**: Update API documentation ## Testing Strategy diff --git a/Sources/FluidAudio/TextToSpeech/Kokoro/Assets/Lexicon/EspeakG2P.swift b/Sources/FluidAudio/TextToSpeech/Kokoro/Assets/Lexicon/EspeakG2P.swift index b713a4f9..33d51adf 100644 --- a/Sources/FluidAudio/TextToSpeech/Kokoro/Assets/Lexicon/EspeakG2P.swift +++ b/Sources/FluidAudio/TextToSpeech/Kokoro/Assets/Lexicon/EspeakG2P.swift @@ -3,7 +3,7 @@ import Foundation /// Thread-safe wrapper around eSpeak NG C API to get IPA phonemes for a word. /// Uses espeak_TextToPhonemes with IPA mode. -final class EspeakG2P { +actor EspeakG2P { enum EspeakG2PError: Error, LocalizedError { case frameworkBundleMissing case dataBundleMissing @@ -30,38 +30,33 @@ final class EspeakG2P { static let shared = EspeakG2P() private let logger = AppLogger(subsystem: "com.fluidaudio.tts", category: "EspeakG2P") - private let queue = DispatchQueue(label: "com.fluidaudio.tts.espeak.g2p") private var initialized = false private var currentVoice: String = "" private init() {} deinit { - queue.sync { - if initialized { - espeak_Terminate() - } + if initialized { + espeak_Terminate() } } func phonemize(word: String, espeakVoice: String = "en-us") throws -> [String]? { - return try queue.sync { - try initializeIfNeeded(espeakVoice: espeakVoice) - return word.withCString { cstr -> [String]? in - var raw: UnsafeRawPointer? = UnsafeRawPointer(cstr) - let modeIPA = Int32(espeakPHONEMES_IPA) - let textmode = Int32(espeakCHARS_AUTO) - guard let outPtr = espeak_TextToPhonemes(&raw, textmode, modeIPA) else { - logger.warning("espeak_TextToPhonemes returned nil for word: \(word)") - return nil - } - let phonemeString = String(cString: outPtr) - if phonemeString.isEmpty { return nil } - if phonemeString.contains(where: { $0.isWhitespace }) { - return phonemeString.split { $0.isWhitespace }.map { String($0) } - } else { - return phonemeString.unicodeScalars.map { String($0) } - } + try initializeIfNeeded(espeakVoice: espeakVoice) + return word.withCString { cstr -> [String]? in + var raw: UnsafeRawPointer? = UnsafeRawPointer(cstr) + let modeIPA = Int32(espeakPHONEMES_IPA) + let textmode = Int32(espeakCHARS_AUTO) + guard let outPtr = espeak_TextToPhonemes(&raw, textmode, modeIPA) else { + logger.warning("espeak_TextToPhonemes returned nil for word: \(word)") + return nil + } + let phonemeString = String(cString: outPtr) + if phonemeString.isEmpty { return nil } + if phonemeString.contains(where: { $0.isWhitespace }) { + return phonemeString.split { $0.isWhitespace }.map { String($0) } + } else { + return phonemeString.unicodeScalars.map { String($0) } } } } diff --git a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/KokoroChunker.swift b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/KokoroChunker.swift index 4b0dd8aa..54fdf2c7 100644 --- a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/KokoroChunker.swift +++ b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/KokoroChunker.swift @@ -40,7 +40,7 @@ enum KokoroChunker { hasLanguageToken: Bool, allowedPhonemes: Set, phoneticOverrides: [TtsPhoneticOverride] - ) throws -> [TextChunk] { + ) async throws -> [TextChunk] { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return [] } @@ -60,7 +60,7 @@ enum KokoroChunker { return [] } - let mergedSentences = try mergeShortSentences( + let mergedSentences = try await mergeShortSentences( refinedSentences, lexicon: wordToPhonemes, caseSensitiveLexicon: caseSensitiveLexicon, @@ -74,7 +74,7 @@ enum KokoroChunker { segmentsByPunctuations.reserveCapacity(segmentsByPeriods.count) for (periodIndex, segment) in segmentsByPeriods.enumerated() { - let count = try tokenCountForSegment( + let count = try await tokenCountForSegment( for: segment, lexicon: wordToPhonemes, caseSensitiveLexicon: caseSensitiveLexicon, @@ -84,7 +84,7 @@ enum KokoroChunker { if count > capacity { let fragments = splitByPunctuation(segment) - let reassembled = try reassembleFragments( + let reassembled = try await reassembleFragments( fragments, lexicon: wordToPhonemes, caseSensitiveLexicon: caseSensitiveLexicon, @@ -120,7 +120,7 @@ enum KokoroChunker { chunks.reserveCapacity(segmentsByPunctuations.count) for chunkText in segmentsByPunctuations { - let built = try buildChunks( + let built = try await buildChunks( from: chunkText, lexicon: wordToPhonemes, caseSensitiveLexicon: caseSensitiveLexicon, @@ -181,7 +181,7 @@ enum KokoroChunker { caseSensitiveLexicon: [String: [String]], allowed: Set, capacity: Int - ) throws -> [String] { + ) async throws -> [String] { guard !sentences.isEmpty else { return [] } let threshold = max(1, min(capacity, TtsConstants.shortSentenceMergeTokenThreshold)) @@ -203,7 +203,7 @@ enum KokoroChunker { let trimmed = sentence.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } - let sentenceTokens = try tokenCountForSegment( + let sentenceTokens = try await tokenCountForSegment( for: trimmed, lexicon: lexicon, caseSensitiveLexicon: caseSensitiveLexicon, @@ -231,7 +231,7 @@ enum KokoroChunker { } let candidate = appendSegment(buffer, with: trimmed) - let candidateTokens = try tokenCountForSegment( + let candidateTokens = try await tokenCountForSegment( for: candidate, lexicon: lexicon, caseSensitiveLexicon: caseSensitiveLexicon, @@ -270,7 +270,7 @@ enum KokoroChunker { wordIndex: inout Int, overrides: [TtsPhoneticOverride], overrideIndex: inout Int - ) throws -> [TextChunk] { + ) async throws -> [TextChunk] { let atoms = tokenizeAtoms(text) guard !atoms.isEmpty else { return [] } @@ -345,7 +345,7 @@ enum KokoroChunker { if resolved == nil { guard - let fallback = try resolvePhonemes( + let fallback = try await resolvePhonemes( for: original, normalized: normalized, lexicon: lexicon, @@ -482,7 +482,7 @@ enum KokoroChunker { caseSensitiveLexicon: [String: [String]], allowed: Set, missing: inout Set - ) throws -> [String]? { + ) async throws -> [String]? { var phonemes = caseSensitiveLexicon[original] if phonemes == nil, let exactNormalized = caseSensitiveLexicon[normalized] { @@ -493,7 +493,7 @@ enum KokoroChunker { phonemes = lexicon[normalized] } - if phonemes == nil, let ipa = try EspeakG2P.shared.phonemize(word: normalized) { + if phonemes == nil, let ipa = try await EspeakG2P.shared.phonemize(word: normalized) { let mapped = PhonemeMapper.mapIPA(ipa, allowed: allowed) if !mapped.isEmpty { phonemes = mapped @@ -510,7 +510,7 @@ enum KokoroChunker { for spelled in spelledTokens { var segment = lexicon[spelled] - if segment == nil, let ipa = try EspeakG2P.shared.phonemize(word: spelled) { + if segment == nil, let ipa = try await EspeakG2P.shared.phonemize(word: spelled) { let mapped = PhonemeMapper.mapIPA(ipa, allowed: allowed) if !mapped.isEmpty { segment = mapped @@ -574,7 +574,7 @@ enum KokoroChunker { caseSensitiveLexicon: [String: [String]], allowed: Set, capacity: Int - ) throws -> Int { + ) async throws -> Int { let atoms = tokenizeAtoms(text) guard !atoms.isEmpty else { return 0 } @@ -589,7 +589,7 @@ enum KokoroChunker { let normalized = normalize(original) guard !normalized.isEmpty else { continue } guard - let phonemes = try resolvePhonemes( + let phonemes = try await resolvePhonemes( for: original, normalized: normalized, lexicon: lexicon, @@ -625,7 +625,7 @@ enum KokoroChunker { caseSensitiveLexicon: [String: [String]], allowed: Set, capacity: Int - ) throws -> [String] { + ) async throws -> [String] { guard !fragments.isEmpty else { return [] } var assembled: [String] = [] @@ -647,7 +647,7 @@ enum KokoroChunker { current.isEmpty ? trimmedFragment : appendSegment(current, with: trimmedFragment) - let candidateTokens = try tokenCountForSegment( + let candidateTokens = try await tokenCountForSegment( for: candidate, lexicon: lexicon, caseSensitiveLexicon: caseSensitiveLexicon, @@ -660,7 +660,7 @@ enum KokoroChunker { } else { flushCurrent() current = trimmedFragment - let fragmentTokens = try tokenCountForSegment( + let fragmentTokens = try await tokenCountForSegment( for: current, lexicon: lexicon, caseSensitiveLexicon: caseSensitiveLexicon, diff --git a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Synthesize/KokoroSynthesizer.swift b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Synthesize/KokoroSynthesizer.swift index 63839fa4..6f76c02e 100644 --- a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Synthesize/KokoroSynthesizer.swift +++ b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Synthesize/KokoroSynthesizer.swift @@ -68,7 +68,7 @@ public struct KokoroSynthesizer { try await loadSimplePhonemeDictionary() let hasLang = false let lexicons = await lexiconCache.lexicons() - return try KokoroChunker.chunk( + return try await KokoroChunker.chunk( text: text, wordToPhonemes: lexicons.word, caseSensitiveLexicon: lexicons.caseSensitive, From 5799571de687bc642901aeafb6cfa67b5f591831 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 14:53:12 -0400 Subject: [PATCH 6/8] sendable public structs --- .../FluidAudio/Diarizer/Clustering/SpeakerOperations.swift | 6 +++--- Sources/FluidAudio/FluidAudioSwift.swift | 2 +- Sources/FluidAudio/Shared/AssetDownloader.swift | 4 ++-- Sources/FluidAudio/Shared/StreamingAudioSampleSource.swift | 4 ++-- .../Pipeline/Postprocess/KokoroSynthesizer+Types.swift | 2 +- .../Kokoro/Pipeline/Preprocess/TtsTextPreprocessor.swift | 2 +- Sources/FluidAudioCLI/Commands/ASR/AsrBenchmarkTypes.swift | 2 +- Sources/FluidAudioCLI/Commands/TTSCommand.swift | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift b/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift index 9098ef2a..2bf16aba 100644 --- a/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift +++ b/Sources/FluidAudio/Diarizer/Clustering/SpeakerOperations.swift @@ -128,7 +128,7 @@ public enum SpeakerUtilities { // MARK: - Speaker Assignment Decision /// Decision result for speaker assignment - public struct AssignmentDecision { + public struct AssignmentDecision: Sendable { public let shouldAssign: Bool public let shouldUpdate: Bool public let confidence: Float @@ -199,7 +199,7 @@ public enum SpeakerUtilities { // MARK: - Speaker Creation /// Validated speaker creation parameters - public struct SpeakerCreationParams { + public struct SpeakerCreationParams: Sendable { public let id: String public let name: String public let duration: Float @@ -379,7 +379,7 @@ public enum SpeakerUtilities { // MARK: - Complete Speaker Update Operations /// Complete speaker update operation including raw tracking - public struct SpeakerUpdateResult { + public struct SpeakerUpdateResult: Sendable { public let updatedMainEmbedding: [Float]? public let updatedRawEmbeddings: [RawEmbedding] public let updatedDuration: Float diff --git a/Sources/FluidAudio/FluidAudioSwift.swift b/Sources/FluidAudio/FluidAudioSwift.swift index 86d23ef3..43125449 100644 --- a/Sources/FluidAudio/FluidAudioSwift.swift +++ b/Sources/FluidAudio/FluidAudioSwift.swift @@ -26,6 +26,6 @@ public typealias SpeakerDiarizationError = DiarizerError // Types like RawEmbedding and SendableSpeaker are already public in their respective files // and will be available when importing FluidAudio module -public struct FluidAudio { +public struct FluidAudio: Sendable { // Empty struct for namespace - all functionality is in the module's types } diff --git a/Sources/FluidAudio/Shared/AssetDownloader.swift b/Sources/FluidAudio/Shared/AssetDownloader.swift index dd5bf0a0..10cfa7d6 100644 --- a/Sources/FluidAudio/Shared/AssetDownloader.swift +++ b/Sources/FluidAudio/Shared/AssetDownloader.swift @@ -19,12 +19,12 @@ public enum AssetDownloader { try FileManager.default.moveItem(at: tempURL, to: destination) } - public enum TransferMode { + public enum TransferMode: Sendable { case data(DataWriter = AssetDownloader.defaultDataWriter) case file(FileMover = AssetDownloader.defaultFileMover) } - public struct Descriptor { + public struct Descriptor: Sendable { public let description: String public let remoteURL: URL public let destinationURL: URL diff --git a/Sources/FluidAudio/Shared/StreamingAudioSampleSource.swift b/Sources/FluidAudio/Shared/StreamingAudioSampleSource.swift index faf84f36..ea4f9b27 100644 --- a/Sources/FluidAudio/Shared/StreamingAudioSampleSource.swift +++ b/Sources/FluidAudio/Shared/StreamingAudioSampleSource.swift @@ -9,7 +9,7 @@ public protocol StreamingAudioSampleSource: Sendable { ) throws } -public struct ArrayAudioSampleSource: StreamingAudioSampleSource { +public struct ArrayAudioSampleSource: StreamingAudioSampleSource, Sendable { private let samples: [Float] public init(samples: [Float]) { @@ -39,7 +39,7 @@ public struct ArrayAudioSampleSource: StreamingAudioSampleSource { } } -public struct DiskBackedAudioSampleSource: StreamingAudioSampleSource { +public struct DiskBackedAudioSampleSource: StreamingAudioSampleSource, Sendable { private let mappedData: Data private let floatStride = MemoryLayout.stride private let fileURL: URL diff --git a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Postprocess/KokoroSynthesizer+Types.swift b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Postprocess/KokoroSynthesizer+Types.swift index 5ba26eb1..56b1d9d3 100644 --- a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Postprocess/KokoroSynthesizer+Types.swift +++ b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Postprocess/KokoroSynthesizer+Types.swift @@ -1,7 +1,7 @@ import Foundation extension KokoroSynthesizer { - public struct TokenCapacities { + public struct TokenCapacities: Sendable { public let short: Int public let long: Int diff --git a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/TtsTextPreprocessor.swift b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/TtsTextPreprocessor.swift index 98fb157a..73969e97 100644 --- a/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/TtsTextPreprocessor.swift +++ b/Sources/FluidAudio/TextToSpeech/Kokoro/Pipeline/Preprocess/TtsTextPreprocessor.swift @@ -1,6 +1,6 @@ import Foundation -public struct TtsPreprocessingResult { +public struct TtsPreprocessingResult: Sendable { public let text: String public let phoneticOverrides: [TtsPhoneticOverride] diff --git a/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmarkTypes.swift b/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmarkTypes.swift index ff4aacaa..5ec9b791 100644 --- a/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmarkTypes.swift +++ b/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmarkTypes.swift @@ -125,7 +125,7 @@ public struct ASRBenchmarkConfig: Sendable { } /// LibriSpeech file representation -public struct LibriSpeechFile { +public struct LibriSpeechFile: Sendable { public let fileName: String public let audioPath: URL public let transcript: String diff --git a/Sources/FluidAudioCLI/Commands/TTSCommand.swift b/Sources/FluidAudioCLI/Commands/TTSCommand.swift index 6ce01bec..0348e14d 100644 --- a/Sources/FluidAudioCLI/Commands/TTSCommand.swift +++ b/Sources/FluidAudioCLI/Commands/TTSCommand.swift @@ -1,7 +1,7 @@ import FluidAudio import Foundation -public struct TTS { +public struct TTS: Sendable { private static let logger = AppLogger(category: "TTSCommand") private static let artifactsDirectoryName = "fluidaudio_cli" From bde2983a14eb6b3ddcb6943ea1da2e60b3cffab7 Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 15:16:15 -0400 Subject: [PATCH 7/8] warpper for snapshot converter --- .../FluidAudio/Shared/AudioConverter.swift | 98 +++++++++++++++++-- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/Sources/FluidAudio/Shared/AudioConverter.swift b/Sources/FluidAudio/Shared/AudioConverter.swift index ae4fbb1d..61681a2f 100644 --- a/Sources/FluidAudio/Shared/AudioConverter.swift +++ b/Sources/FluidAudio/Shared/AudioConverter.swift @@ -2,6 +2,77 @@ import AVFoundation import Accelerate import Foundation import OSLog +import os + +private struct PCMBufferSnapshot: Sendable { + struct Plane: Sendable { + let data: Data + let byteSize: UInt32 + } + + let streamDescription: AudioStreamBasicDescription + let planes: [Plane] + let frameLength: AVAudioFrameCount + + init?(source buffer: AVAudioPCMBuffer) { + streamDescription = buffer.format.streamDescription.pointee + frameLength = buffer.frameLength + + let audioBuffers = UnsafeMutableAudioBufferListPointer(buffer.mutableAudioBufferList) + var collected: [Plane] = [] + collected.reserveCapacity(audioBuffers.count) + + for audioBuffer in audioBuffers { + let byteCount = Int(audioBuffer.mDataByteSize) + let planeData: Data + if let baseAddress = audioBuffer.mData, byteCount > 0 { + planeData = Data(bytes: baseAddress, count: byteCount) + } else { + planeData = Data() + } + collected.append(Plane(data: planeData, byteSize: audioBuffer.mDataByteSize)) + } + + if collected.isEmpty && frameLength > 0 { + return nil + } + + planes = collected + } + + func makeBuffer() -> AVAudioPCMBuffer? { + var descriptionCopy = streamDescription + guard let format = AVAudioFormat(streamDescription: &descriptionCopy) else { + return nil + } + + let capacity = max(frameLength, 1) + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { + return nil + } + + buffer.frameLength = frameLength + + let audioBuffers = UnsafeMutableAudioBufferListPointer(buffer.mutableAudioBufferList) + guard audioBuffers.count == planes.count else { + return nil + } + + for (index, plane) in planes.enumerated() { + audioBuffers[index].mDataByteSize = plane.byteSize + guard plane.byteSize > 0 else { continue } + plane.data.withUnsafeBytes { rawBytes in + guard + let srcBase = rawBytes.baseAddress, + let dstBase = audioBuffers[index].mData + else { return } + memcpy(dstBase, srcBase, rawBytes.count) + } + } + + return buffer + } +} /// Converts audio buffers to the format required by ASR (16kHz, mono, Float32). /// @@ -92,17 +163,30 @@ final public class AudioConverter { var aggregated: [Float] = [] aggregated.reserveCapacity(Int(estimatedOutputFrames)) - // Provide input once, then signal end-of-stream - var provided = false + guard let snapshot = PCMBufferSnapshot(source: buffer) else { + throw AudioConverterError.failedToCreateBuffer + } + + // Provide input once, then signal end-of-stream (guarded for strict concurrency) + let inputProvidedFlag = OSAllocatedUnfairLock(initialState: false) let inputBlock: AVAudioConverterInputBlock = { _, status in - if !provided { - provided = true - status.pointee = .haveData - return buffer - } else { + let shouldProvide = inputProvidedFlag.withLock { state -> Bool in + if state { + return false + } + state = true + return true + } + guard shouldProvide else { + status.pointee = .endOfStream + return nil + } + guard let nextBuffer = snapshot.makeBuffer() else { status.pointee = .endOfStream return nil } + status.pointee = .haveData + return nextBuffer } var error: NSError? From ff17f97193294ee7b046ea0d7d2f0dd27d9ffd5c Mon Sep 17 00:00:00 2001 From: Brandon Weng Date: Wed, 29 Oct 2025 15:50:12 -0400 Subject: [PATCH 8/8] Preconcurrency for audio --- Sources/FluidAudio/ASR/AsrManager.swift | 2 +- Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift | 2 +- Sources/FluidAudio/Shared/AudioConverter.swift | 2 +- Sources/FluidAudio/Shared/StreamingAudioSourceFactory.swift | 2 +- Sources/FluidAudio/VAD/VadManager.swift | 2 +- Sources/FluidAudioCLI/Commands/ASR/AsrBenchmark.swift | 2 +- Sources/FluidAudioCLI/Commands/ASR/FleursBenchmark.swift | 2 +- Sources/FluidAudioCLI/Commands/ASR/TranscribeCommand.swift | 2 +- Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift | 2 +- Sources/FluidAudioCLI/Commands/MultiStreamCommand.swift | 2 +- Sources/FluidAudioCLI/Commands/ProcessCommand.swift | 2 +- Sources/FluidAudioCLI/Commands/VadAnalyzeCommand.swift | 2 +- Sources/FluidAudioCLI/Commands/VadBenchmark.swift | 2 +- Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift | 2 +- Sources/FluidAudioCLI/main.swift | 2 +- Tests/FluidAudioTests/AudioConverterTests.swift | 2 +- Tests/FluidAudioTests/StreamingAsrManagerTests.swift | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Sources/FluidAudio/ASR/AsrManager.swift b/Sources/FluidAudio/ASR/AsrManager.swift index 7b1850b4..fd4995a7 100644 --- a/Sources/FluidAudio/ASR/AsrManager.swift +++ b/Sources/FluidAudio/ASR/AsrManager.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import CoreML import Foundation import OSLog diff --git a/Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift b/Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift index c837df77..a84dd6a1 100644 --- a/Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift +++ b/Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import Foundation import OSLog diff --git a/Sources/FluidAudio/Shared/AudioConverter.swift b/Sources/FluidAudio/Shared/AudioConverter.swift index 61681a2f..c89d0ad3 100644 --- a/Sources/FluidAudio/Shared/AudioConverter.swift +++ b/Sources/FluidAudio/Shared/AudioConverter.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import Accelerate import Foundation import OSLog diff --git a/Sources/FluidAudio/Shared/StreamingAudioSourceFactory.swift b/Sources/FluidAudio/Shared/StreamingAudioSourceFactory.swift index fd296385..618a15b7 100644 --- a/Sources/FluidAudio/Shared/StreamingAudioSourceFactory.swift +++ b/Sources/FluidAudio/Shared/StreamingAudioSourceFactory.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import Foundation import OSLog diff --git a/Sources/FluidAudio/VAD/VadManager.swift b/Sources/FluidAudio/VAD/VadManager.swift index fc2b714b..1ac6f014 100644 --- a/Sources/FluidAudio/VAD/VadManager.swift +++ b/Sources/FluidAudio/VAD/VadManager.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import Accelerate import CoreML import Foundation diff --git a/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmark.swift b/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmark.swift index 46b42104..fe359d2d 100644 --- a/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/ASR/AsrBenchmark.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import OSLog diff --git a/Sources/FluidAudioCLI/Commands/ASR/FleursBenchmark.swift b/Sources/FluidAudioCLI/Commands/ASR/FleursBenchmark.swift index faabdbbe..25c21d76 100644 --- a/Sources/FluidAudioCLI/Commands/ASR/FleursBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/ASR/FleursBenchmark.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation import OSLog diff --git a/Sources/FluidAudioCLI/Commands/ASR/TranscribeCommand.swift b/Sources/FluidAudioCLI/Commands/ASR/TranscribeCommand.swift index 3833c681..073badd9 100644 --- a/Sources/FluidAudioCLI/Commands/ASR/TranscribeCommand.swift +++ b/Sources/FluidAudioCLI/Commands/ASR/TranscribeCommand.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift index 571e2b8b..02603a12 100644 --- a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/Commands/MultiStreamCommand.swift b/Sources/FluidAudioCLI/Commands/MultiStreamCommand.swift index 08235877..fb4afaad 100644 --- a/Sources/FluidAudioCLI/Commands/MultiStreamCommand.swift +++ b/Sources/FluidAudioCLI/Commands/MultiStreamCommand.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/Commands/ProcessCommand.swift b/Sources/FluidAudioCLI/Commands/ProcessCommand.swift index bfc2b07b..296fdef3 100644 --- a/Sources/FluidAudioCLI/Commands/ProcessCommand.swift +++ b/Sources/FluidAudioCLI/Commands/ProcessCommand.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/Commands/VadAnalyzeCommand.swift b/Sources/FluidAudioCLI/Commands/VadAnalyzeCommand.swift index 6868fb92..eaa430a2 100644 --- a/Sources/FluidAudioCLI/Commands/VadAnalyzeCommand.swift +++ b/Sources/FluidAudioCLI/Commands/VadAnalyzeCommand.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/Commands/VadBenchmark.swift b/Sources/FluidAudioCLI/Commands/VadBenchmark.swift index a7bd4ad9..41584eef 100644 --- a/Sources/FluidAudioCLI/Commands/VadBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/VadBenchmark.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift b/Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift index 5bfec4ba..4c2e1091 100644 --- a/Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift +++ b/Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import Foundation import FluidAudio diff --git a/Sources/FluidAudioCLI/main.swift b/Sources/FluidAudioCLI/main.swift index 64a46d2e..858bc81b 100644 --- a/Sources/FluidAudioCLI/main.swift +++ b/Sources/FluidAudioCLI/main.swift @@ -1,5 +1,5 @@ #if os(macOS) -import AVFoundation +@preconcurrency import AVFoundation import FluidAudio import Foundation diff --git a/Tests/FluidAudioTests/AudioConverterTests.swift b/Tests/FluidAudioTests/AudioConverterTests.swift index 01a773fe..5dfef24d 100644 --- a/Tests/FluidAudioTests/AudioConverterTests.swift +++ b/Tests/FluidAudioTests/AudioConverterTests.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import Foundation import XCTest diff --git a/Tests/FluidAudioTests/StreamingAsrManagerTests.swift b/Tests/FluidAudioTests/StreamingAsrManagerTests.swift index d136b426..23728cf1 100644 --- a/Tests/FluidAudioTests/StreamingAsrManagerTests.swift +++ b/Tests/FluidAudioTests/StreamingAsrManagerTests.swift @@ -1,4 +1,4 @@ -import AVFoundation +@preconcurrency import AVFoundation import XCTest @testable import FluidAudio