Add experimental CTC zh-CN Mandarin ASR

Documentation/Benchmarks.md

@@ -734,3 +734,52 @@ Both the English BART G2P and multilingual ByT5 G2P models run fastest on CPU-on
 | cpuOnly | **13.0** |
 | all (ANE+GPU+CPU) | 17.3 |
 | cpuAndGPU | 23.4 |
+
+## CTC zh-CN Mandarin ASR (Experimental)
+
+Parakeet CTC 0.6B zh-CN model converted to CoreML for on-device Mandarin Chinese transcription.
+
+> **⚠️ Experimental Feature**: This is an early preview of Mandarin Chinese ASR support. The API and performance characteristics may change in future releases.
+
+Model: [FluidInference/parakeet-ctc-0.6b-zh-cn-coreml](https://huggingface.co/FluidInference/parakeet-ctc-0.6b-zh-cn-coreml)
+
+Hardware: Apple M2, 2022, macOS 26
+
+### THCHS-30 Test Set
+
+Full benchmark on the complete THCHS-30 test set — 2,495 utterances (250 unique sentences × 10 speakers) from the THCHS-30 corpus.
+
+Dataset: [FluidInference/THCHS-30-tests](https://huggingface.co/datasets/FluidInference/THCHS-30-tests)
+
+```bash
+swift run -c release fluidaudiocli ctc-zh-cn-benchmark --auto-download
+```
+
+| Metric | int8 encoder (0.55 GB) |
+|---|---|
+| **Mean CER** | **8.23%** |
+| **Median CER** | **6.45%** |
+| CER = 0% (perfect) | 435 (17.4%) |
+| CER < 5% | 947 (38.0%) |
+| CER < 10% | 1,674 (67.1%) |
+| CER < 20% | 2,325 (93.2%) |
+| Mean Latency | 614 ms |
+| Mean RTFx | 14.83x |
+
+### Error Analysis
+
+Error analysis from the 100 highest-CER samples (out of the full 2,495) identified 862 substitution errors. The dominant patterns:
+
+- **Homophones / near-homophones**: acoustically similar syllables (e.g. 呢/了, 了/的) account for the majority of substitutions — unavoidable without a language model
+- **Digit representation**: the model may output Arabic digits (1, 5, 2011) when references use Chinese characters (一五, 二零一一); the benchmark normalizer converts digits before scoring to avoid penalizing this
+- **Sentence-final particles**: 了/的/呢/吧 are frequently confused, contributing a disproportionate share of errors given their high occurrence
+
+### Beam Search
+
+Beam search does not improve CER for this model without a language model. Greedy decoding (beam width 1) is recommended.
+
+### Recommendations
+
+- **Greedy decoding** is sufficient for production use at this CER level.
+- For applications requiring <8% CER, a character-level language model would be needed.
+- Int8 encoder (0.55 GB) performs on par with FP32 (1.1 GB).

Sources/FluidAudio/ASR/Parakeet/AsrManager.swift

@@ -293,6 +293,10 @@ public actor AsrManager {
                 isLastChunk: isLastChunk,
                 globalFrameOffset: globalFrameOffset
             )
+        case .ctcZhCn:
+            throw ASRError.processingFailed(
+                "CTC-only model .ctcZhCn does not support TDT decoding. Use CtcZhCnManager instead."
+            )
         }
     }
 

Sources/FluidAudio/ASR/Parakeet/AsrModels.swift

@@ -7,12 +7,15 @@ public enum AsrModelVersion: Sendable {
     case v3
     /// 110M parameter hybrid TDT-CTC model with fused preprocessor+encoder
     case tdtCtc110m
+    /// 600M parameter CTC-only model for Mandarin Chinese (zh-CN)
+    case ctcZhCn
 
     var repo: Repo {
         switch self {
         case .v2: return .parakeetV2
         case .v3: return .parakeet
         case .tdtCtc110m: return .parakeetTdtCtc110m
+        case .ctcZhCn: return .parakeetCtcZhCn
         }
     }
 
@@ -24,10 +27,19 @@ public enum AsrModelVersion: Sendable {
         }
     }
 
+    /// Whether this model is CTC-only (no TDT decoder+joint)
+    public var isCtcOnly: Bool {
+        switch self {
+        case .ctcZhCn: return true
+        default: return false
+        }
+    }
+
     /// Encoder hidden dimension for this model version
     public var encoderHiddenSize: Int {
         switch self {
         case .tdtCtc110m: return 512
+        case .ctcZhCn: return 1024
         default: return 1024
         }
     }
@@ -37,6 +49,7 @@ public enum AsrModelVersion: Sendable {
         switch self {
         case .v2, .tdtCtc110m: return 1024
         case .v3: return 8192
+        case .ctcZhCn: return 7000
         }
     }
 

Sources/FluidAudio/ASR/Parakeet/CtcZhCnManager.swift

@@ -0,0 +1,207 @@
+@preconcurrency import CoreML
+import Foundation
+
+/// Manager for Parakeet CTC zh-CN transcription
+///
+/// This manager handles the full pipeline for Mandarin Chinese CTC transcription:
+/// 1. Preprocessor: Audio → Mel spectrogram
+/// 2. Encoder: Mel → Encoder features
+/// 3. CTC Decoder: Encoder features → CTC logits
+/// 4. Greedy CTC decoding: Logits → Text
+public actor CtcZhCnManager {
+
+    private let models: CtcZhCnModels
+    private let maxAudioSamples: Int
+    private let sampleRate: Int
+
+    private static let logger = AppLogger(category: "CtcZhCnManager")
+
+    /// Initialize with pre-loaded models
+    public init(models: CtcZhCnModels, maxAudioSamples: Int = 240_000, sampleRate: Int = 16_000) {
+        self.models = models
+        self.maxAudioSamples = maxAudioSamples
+        self.sampleRate = sampleRate
+    }
+
+    /// Convenience initializer that loads models from default cache directory
+    public static func load(
+        useInt8Encoder: Bool = true,
+        configuration: MLModelConfiguration? = nil,
+        progressHandler: DownloadUtils.ProgressHandler? = nil
+    ) async throws -> CtcZhCnManager {
+        let models = try await CtcZhCnModels.downloadAndLoad(
+            useInt8Encoder: useInt8Encoder,
+            configuration: configuration,
+            progressHandler: progressHandler
+        )
+        return CtcZhCnManager(models: models)
+    }
+
+    /// Transcribe audio to text using CTC decoding
+    ///
+    /// - Parameters:
+    ///   - audio: Audio samples (mono, 16kHz)
+    ///   - audioLength: Optional audio length (if nil, uses audio.count)
+    /// - Returns: Transcribed text
+    public func transcribe(
+        audio: [Float],
+        audioLength: Int? = nil
+    ) throws -> String {
+        let actualLength = audioLength ?? audio.count
+
+        // Pad or truncate audio to maxAudioSamples
+        let paddedAudio = padOrTruncateAudio(audio, targetLength: maxAudioSamples)
+
+        // Step 1: Preprocessor (audio → mel spectrogram)
+        let melOutput = try runPreprocessor(audio: paddedAudio, audioLength: actualLength)
+
+        // Step 2: Encoder (mel → encoder features)
+        let encoderOutput = try runEncoder(mel: melOutput.mel, melLength: melOutput.melLength)
+
+        // Step 3: CTC Decoder (encoder features → CTC logits)
+        let ctcLogits = try runCtcDecoder(encoderOutput: encoderOutput)
+
+        // Step 4: CTC decoding (logits → text)
+        let text = greedyCtcDecode(logits: ctcLogits)
+
+        return text
+    }
+
+    /// Transcribe audio file to text
+    ///
+    /// - Parameters:
+    ///   - audioURL: URL to audio file (will be resampled to 16kHz mono)
+    /// - Returns: Transcribed text
+    public func transcribe(audioURL: URL) throws -> String {
+        // Load and convert audio
+        let converter = AudioConverter(sampleRate: Double(sampleRate))
+        let samples = try converter.resampleAudioFile(audioURL)
+
+        return try transcribe(audio: samples)
+    }
+
+    // MARK: - Private Pipeline Methods
+
+    private struct MelOutput {
+        let mel: MLMultiArray
+        let melLength: MLMultiArray
+    }
+
+    private func runPreprocessor(audio: [Float], audioLength: Int) throws -> MelOutput {
+        // Create input arrays
+        let audioArray = try MLMultiArray(shape: [1, maxAudioSamples as NSNumber], dataType: .float32)
+        for (i, sample) in audio.enumerated() where i < maxAudioSamples {
+            audioArray[i] = NSNumber(value: sample)
+        }
+
+        let audioLengthArray = try MLMultiArray(shape: [1], dataType: .int32)
+        audioLengthArray[0] = NSNumber(value: min(audioLength, maxAudioSamples))
+
+        // Run preprocessor
+        let input = try MLDictionaryFeatureProvider(
+            dictionary: [
+                "audio_signal": MLFeatureValue(multiArray: audioArray),
+                "audio_length": MLFeatureValue(multiArray: audioLengthArray),
+            ]
+        )
+        let output = try models.preprocessor.prediction(from: input)
+
+        guard
+            let mel = output.featureValue(for: "mel")?.multiArrayValue,
+            let melLength = output.featureValue(for: "mel_length")?.multiArrayValue
+        else {
+            throw ASRError.processingFailed("Failed to extract mel or mel_length from preprocessor output")
+        }
+
+        return MelOutput(mel: mel, melLength: melLength)
+    }
+
+    private func runEncoder(mel: MLMultiArray, melLength: MLMultiArray) throws -> MLMultiArray {
+        // Run encoder
+        let input = try MLDictionaryFeatureProvider(
+            dictionary: [
+                "audio_signal": MLFeatureValue(multiArray: mel),
+                "length": MLFeatureValue(multiArray: melLength),
+            ]
+        )
+        let output = try models.encoder.prediction(from: input)
+
+        guard let encoderOutput = output.featureValue(for: "encoder_output")?.multiArrayValue else {
+            throw ASRError.processingFailed("Failed to extract encoder_output from encoder")
+        }
+
+        return encoderOutput
+    }
+
+    private func runCtcDecoder(encoderOutput: MLMultiArray) throws -> MLMultiArray {
+        // Run CTC decoder head
+        let input = try MLDictionaryFeatureProvider(
+            dictionary: [
+                "encoder_output": MLFeatureValue(multiArray: encoderOutput)
+            ]
+        )
+        let output = try models.decoder.prediction(from: input)
+
+        guard let ctcLogits = output.featureValue(for: "ctc_logits")?.multiArrayValue else {
+            throw ASRError.processingFailed("Failed to extract ctc_logits from decoder")
+        }
+
+        return ctcLogits
+    }
+
+    private func greedyCtcDecode(logits: MLMultiArray) -> String {
+        // logits shape: [1, T, vocab_size+1] where T is time steps (188)
+        // vocab_size = 7000, blank_id = 7000
+
+        let timeSteps = logits.shape[1].intValue
+        let vocabSize = logits.shape[2].intValue
+
+        var decoded: [Int] = []
+        var prevLabel: Int? = nil
+
+        for t in 0..<timeSteps {
+            // Find argmax at this time step
+            var maxLogit: Float = -.infinity
+            var maxLabel = 0
+
+            for v in 0..<vocabSize {
+                let logit = logits[[0, t as NSNumber, v as NSNumber]].floatValue
+                if logit > maxLogit {
+                    maxLogit = logit
+                    maxLabel = v
+                }
+            }
+
+            // CTC collapse: skip blanks and repeats
+            if maxLabel != models.blankId && maxLabel != prevLabel {
+                decoded.append(maxLabel)
+            }
+            prevLabel = maxLabel
+        }
+
+        // Convert token IDs to text
+        var text = ""
+        for tokenId in decoded {
+            if let token = models.vocabulary[tokenId] {
+                text += token
+            }
+        }
+
+        // Replace SentencePiece underscores with spaces
+        text = text.replacingOccurrences(of: "▁", with: " ")
+
+        return text.trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+
+    private func padOrTruncateAudio(_ audio: [Float], targetLength: Int) -> [Float] {
+        var result = audio
+        if result.count < targetLength {
+            // Pad with zeros
+            result.append(contentsOf: Array(repeating: 0.0, count: targetLength - result.count))
+        } else if result.count > targetLength {
+            // Truncate
+            result = Array(result.prefix(targetLength))
+        }
+        return result
+    }
+}