Skip to content

Commit a044a72

Browse files
BrandonWengBrandonWeng
committed
Fix sampels
1 parent d1d01b7 commit a044a72

3 files changed

Lines changed: 59 additions & 12 deletions

File tree

‎Sources/FluidAudio/ASR/AsrTranscription.swift‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,11 +189,13 @@ extension AsrManager {
189189
encoderSequenceLength: Int,
190190
audioSamples: [Float],
191191
processingTime: TimeInterval,
192-
tokenTimings: [TokenTiming] = []
192+
tokenTimings: [TokenTiming] = [],
193+
totalSampleCount: Int? = nil
193194
) -> ASRResult {
194195

195196
let (text, finalTimings) = convertTokensWithExistingTimings(tokenIds, timings: tokenTimings)
196-
let duration = TimeInterval(audioSamples.count) / TimeInterval(config.sampleRate)
197+
let effectiveSampleCount = totalSampleCount ?? audioSamples.count
198+
let duration = TimeInterval(effectiveSampleCount) / TimeInterval(config.sampleRate)
197199

198200
// Convert timestamps to TokenTiming objects if provided
199201
let timingsFromTimestamps = createTokenTimings(

‎Sources/FluidAudio/ASR/Streaming/StreamingAsrManager.swift‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public actor StreamingAsrManager {
2828
private var accumulatedTokens: [Int] = []
2929
private var accumulatedTokenTimestamps: [Int] = []
3030
private var accumulatedTokenConfidences: [Float] = []
31+
private var totalSamplesProcessed: Int = 0
3132

3233
// Raw sample buffer for sliding-window assembly (absolute indexing)
3334
private var sampleBuffer: [Float] = []
@@ -95,6 +96,7 @@ public actor StreamingAsrManager {
9596
segmentIndex = 0
9697
lastProcessedFrame = 0
9798
resetAccumulatedMetadata()
99+
totalSamplesProcessed = 0
98100

99101
startTime = Date()
100102

@@ -163,6 +165,16 @@ public actor StreamingAsrManager {
163165
throw error
164166
}
165167

168+
let now = Date()
169+
let elapsedTime = startTime.map { now.timeIntervalSince($0) } ?? 0
170+
let minimumProcessingTime: TimeInterval = 1e-6
171+
172+
let maxTimestampFrame = accumulatedTokenTimestamps.max() ?? 0
173+
let derivedSampleCount = maxTimestampFrame > 0
174+
? (maxTimestampFrame + 1) * ASRConstants.samplesPerEncoderFrame : 0
175+
let finalSampleCount = max(totalSamplesProcessed, derivedSampleCount)
176+
let finalProcessingTime = finalSampleCount > 0 ? max(elapsedTime, minimumProcessingTime) : elapsedTime
177+
166178
// Convert final accumulated tokens to ASRResult (proper way to avoid duplicates)
167179
let finalResult: ASRResult
168180
if let asrManager = asrManager, !accumulatedTokens.isEmpty {
@@ -195,17 +207,18 @@ public actor StreamingAsrManager {
195207
timestamps: timestamps,
196208
confidences: confidences,
197209
encoderSequenceLength: 0,
198-
audioSamples: [], // Not needed for final text conversion
199-
processingTime: 0
210+
audioSamples: [], // No need to retain audio samples for final aggregation
211+
processingTime: finalProcessingTime,
212+
totalSampleCount: finalSampleCount
200213
)
201214
} else {
202215
// Fallback to text concatenation if no tokens available
203216
let fallbackText = confirmedTranscript + volatileTranscript
204217
finalResult = ASRResult(
205218
text: fallbackText,
206219
confidence: 1.0,
207-
duration: 0,
208-
processingTime: 0,
220+
duration: TimeInterval(finalSampleCount) / TimeInterval(config.asrConfig.sampleRate),
221+
processingTime: finalProcessingTime,
209222
tokenTimings: nil
210223
)
211224
}
@@ -233,6 +246,7 @@ public actor StreamingAsrManager {
233246
segmentIndex = 0
234247
lastProcessedFrame = 0
235248
resetAccumulatedMetadata()
249+
totalSamplesProcessed = 0
236250

237251
logger.info("StreamingAsrManager reset for source: \(String(describing: self.audioSource))")
238252
}
@@ -257,6 +271,11 @@ public actor StreamingAsrManager {
257271
accumulatedTokenConfidences.removeAll()
258272
}
259273

274+
private func calculateGlobalFrameOffset(for sampleIndex: Int) -> Int {
275+
guard sampleIndex > 0 else { return 0 }
276+
return sampleIndex / ASRConstants.samplesPerEncoderFrame
277+
}
278+
260279
func accumulateTokenMetadata(tokens: [Int], timestamps: [Int], confidences: [Float]) {
261280
guard !tokens.isEmpty else { return }
262281

@@ -285,6 +304,7 @@ public actor StreamingAsrManager {
285304
private func appendSamplesAndProcess(_ samples: [Float]) async {
286305
// Append samples to buffer
287306
sampleBuffer.append(contentsOf: samples)
307+
totalSamplesProcessed += samples.count
288308

289309
// Process while we have at least chunk + right ahead of the current center start
290310
let chunk = config.chunkSamples
@@ -304,7 +324,11 @@ public actor StreamingAsrManager {
304324

305325
let window = Array(sampleBuffer[startIdx..<endIdx])
306326
let actualLeftSecs = Double(nextWindowCenterStart - leftStartAbs) / Double(sampleRate)
307-
await processWindow(window, actualLeftSeconds: actualLeftSecs)
327+
await processWindow(
328+
window,
329+
actualLeftSeconds: actualLeftSecs,
330+
windowStartSample: leftStartAbs
331+
)
308332

309333
// Advance by chunk size
310334
nextWindowCenterStart += chunk
@@ -342,7 +366,11 @@ public actor StreamingAsrManager {
342366

343367
let window = Array(sampleBuffer[startIdx..<endIdx])
344368
let actualLeftSecs = Double(nextWindowCenterStart - leftStartAbs) / Double(sampleRate)
345-
await processWindow(window, actualLeftSeconds: actualLeftSecs)
369+
await processWindow(
370+
window,
371+
actualLeftSeconds: actualLeftSecs,
372+
windowStartSample: leftStartAbs
373+
)
346374

347375
nextWindowCenterStart += effectiveChunk
348376

@@ -359,7 +387,11 @@ public actor StreamingAsrManager {
359387
}
360388

361389
/// Process a single assembled window: [left, chunk, right]
362-
private func processWindow(_ windowSamples: [Float], actualLeftSeconds: Double) async {
390+
private func processWindow(
391+
_ windowSamples: [Float],
392+
actualLeftSeconds _: Double,
393+
windowStartSample: Int
394+
) async {
363395
guard let asrManager = asrManager else { return }
364396

365397
do {
@@ -374,9 +406,12 @@ public actor StreamingAsrManager {
374406
previousTokens: accumulatedTokens
375407
)
376408

409+
let frameOffset = calculateGlobalFrameOffset(for: windowStartSample)
410+
let adjustedTimestamps = timestamps.map { $0 + frameOffset }
411+
377412
// Update state
378-
accumulateTokenMetadata(tokens: tokens, timestamps: timestamps, confidences: confidences)
379-
lastProcessedFrame = max(lastProcessedFrame, timestamps.max() ?? 0)
413+
accumulateTokenMetadata(tokens: tokens, timestamps: adjustedTimestamps, confidences: confidences)
414+
lastProcessedFrame = max(lastProcessedFrame, adjustedTimestamps.max() ?? 0)
380415
segmentIndex += 1
381416

382417
let processingTime = Date().timeIntervalSince(chunkStartTime)
@@ -386,7 +421,7 @@ public actor StreamingAsrManager {
386421
// The final result will use all accumulated tokens for proper deduplication
387422
let interim = asrManager.processTranscriptionResult(
388423
tokenIds: tokens, // Only current chunk tokens for progress updates
389-
timestamps: timestamps,
424+
timestamps: adjustedTimestamps,
390425
confidences: confidences,
391426
encoderSequenceLength: 0,
392427
audioSamples: windowSamples,
@@ -524,6 +559,10 @@ extension StreamingAsrManager {
524559
internal func setAsrManagerForTesting(_ manager: AsrManager?) {
525560
asrManager = manager
526561
}
562+
563+
internal func setTotalSamplesProcessedForTesting(_ count: Int) {
564+
totalSamplesProcessed = count
565+
}
527566
}
528567
#endif
529568

‎Tests/FluidAudioTests/StreamingAsrManagerTests.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ final class StreamingAsrManagerTests: XCTestCase {
198198
timestamps: [10, 25],
199199
confidences: [0.9, 0.8]
200200
)
201+
await manager.setTotalSamplesProcessedForTesting((25 + 1) * ASRConstants.samplesPerEncoderFrame)
201202

202203
let result = try await manager.finish()
203204

@@ -213,6 +214,8 @@ final class StreamingAsrManagerTests: XCTestCase {
213214
XCTAssertEqual(Double(tokenTimings[0].confidence), 0.9, accuracy: 1e-6)
214215
XCTAssertEqual(Double(tokenTimings[1].confidence), 0.8, accuracy: 1e-6)
215216
XCTAssertEqual(result.text, "hello world")
217+
XCTAssertGreaterThan(result.duration, 0)
218+
XCTAssertGreaterThan(result.processingTime, 0)
216219
}
217220

218221
func testFinishOmitsTimingsWhenMetadataMismatchOccurs() async throws {
@@ -231,11 +234,14 @@ final class StreamingAsrManagerTests: XCTestCase {
231234
timestamps: [10],
232235
confidences: [0.9]
233236
)
237+
await manager.setTotalSamplesProcessedForTesting(32_000)
234238

235239
let result = try await manager.finish()
236240

237241
XCTAssertTrue(result.tokenTimings?.isEmpty ?? true)
238242
XCTAssertEqual(result.text, "hello world")
243+
XCTAssertGreaterThan(result.duration, 0)
244+
XCTAssertGreaterThan(result.processingTime, 0)
239245
}
240246

241247
// MARK: - Audio Source Tests

0 commit comments

Comments
 (0)