feat: return per-segment timestamps from speech2text/transcribe

/v1/infer/speech2text/transcribe returned a single flattened string
(generate_all_text()), silently dropping the segment-level start/end
timestamps that Python's s2t_inference.transcribe_segments() (and the
UI, which iterates d.segments) expects.

Add yu_hailort_s2t_generate_segments() to the shim, wrapping the SDK's
generate_all_segments() and marshaling the result as a JSON array
(reusing the existing escape_json_string() helper already used for
chat message JSON). The Rust wrapper deserializes it into Vec<Segment>
via serde. The route now returns both `segments` (array of
{text,start,end}) and `text` (the segments' text joined with spaces,
kept for callers that only want the flat transcript).

Strengthened the existing #[ignore] real-hw smoke test to assert on
the response shape (data.segments is an array, data.text is a string)
instead of only the HTTP status, so a broken JSON marshal wouldn't
silently pass. Verified against Whisper-Tiny.hef on real Hailo-10H
hardware (all 56 non-hardware tests still pass, clippy clean).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HoKquUKDTB333MNkdBi6uV

Author: eauesque

yu-hailo-infer/src/hailort/shim.cpp

@@ -345,6 +345,71 @@ int yu_hailort_s2t_generate_text(
     return (nullptr == *out_text) ? HAILO_OUT_OF_HOST_MEMORY : HAILO_SUCCESS;
 }
 
+int yu_hailort_s2t_generate_segments(
+    YuHailortSpeech2Text *ctx,
+    const float *audio,
+    size_t audio_count,
+    int task,
+    const char *language,
+    float repetition_penalty,
+    uint32_t timeout_ms,
+    char **out_json)
+{
+    if ((nullptr == ctx) || (nullptr == audio) || (nullptr == out_json)) {
+        return HAILO_INVALID_ARGUMENT;
+    }
+    *out_json = nullptr;
+
+    auto params = ctx->speech2text->create_generator_params();
+    if (!params) {
+        return params.status();
+    }
+    auto generator_params = params.release();
+    auto status = generator_params.set_task(static_cast<hailort::genai::Speech2TextTask>(task));
+    if (HAILO_SUCCESS != status) {
+        return status;
+    }
+    if ((nullptr != language) && ('\0' != language[0])) {
+        status = generator_params.set_language(std::string_view(language));
+        if (HAILO_SUCCESS != status) {
+            return status;
+        }
+    }
+    status = generator_params.set_repetition_penalty(repetition_penalty);
+    if (HAILO_SUCCESS != status) {
+        return status;
+    }
+
+    auto audio_view = hailort::MemoryView(
+        const_cast<float *>(audio),
+        audio_count * sizeof(float));
+    auto segments = ctx->speech2text->generate_all_segments(
+        audio_view,
+        generator_params,
+        std::chrono::milliseconds(timeout_ms));
+    if (!segments) {
+        return segments.status();
+    }
+
+    std::string json = "[";
+    bool first = true;
+    for (const auto &segment : segments.value()) {
+        if (!first) {
+            json += ",";
+        }
+        first = false;
+        char bounds[64];
+        std::snprintf(bounds, sizeof(bounds), "{\"start_sec\":%.6f,\"end_sec\":%.6f,",
+            static_cast<double>(segment.start_sec), static_cast<double>(segment.end_sec));
+        json += bounds;
+        json += "\"text\":\"" + escape_json_string(segment.text) + "\"}";
+    }
+    json += "]";
+
+    *out_json = copy_string_to_c(json);
+    return (nullptr == *out_json) ? HAILO_OUT_OF_HOST_MEMORY : HAILO_SUCCESS;
+}
+
 int yu_hailort_s2t_tokenize(
     YuHailortSpeech2Text *ctx,
     const char *text,

yu-hailo-infer/src/hailort/shim.h

@@ -67,6 +67,19 @@ int yu_hailort_s2t_generate_text(
     float repetition_penalty,
     uint32_t timeout_ms,
     char **out_text);
+// Same generation as yu_hailort_s2t_generate_text, but returns the SDK's
+// per-segment breakdown (start/end timestamps + text) as a JSON array
+// string: [{"start_sec":F,"end_sec":F,"text":"..."},...]. Caller frees
+// *out_json with yu_hailort_string_free, same as out_text above.
+int yu_hailort_s2t_generate_segments(
+    YuHailortSpeech2Text *ctx,
+    const float *audio,
+    size_t audio_count,
+    int task,
+    const char *language,
+    float repetition_penalty,
+    uint32_t timeout_ms,
+    char **out_json);
 int yu_hailort_s2t_tokenize(
     YuHailortSpeech2Text *ctx,
     const char *text,

yu-hailo-infer/src/hailort/shim_stub.cpp

@@ -100,6 +100,7 @@ void yu_hailort_s2t_release(YuHailortSpeech2Text *ctx)
     ++s2t_release_count;
 }
 int yu_hailort_s2t_generate_text(YuHailortSpeech2Text *, const float *, size_t, int, const char *, float, uint32_t, char **) { return HAILO_STUB_ERR; }
+int yu_hailort_s2t_generate_segments(YuHailortSpeech2Text *, const float *, size_t, int, const char *, float, uint32_t, char **) { return HAILO_STUB_ERR; }
 int yu_hailort_s2t_tokenize(YuHailortSpeech2Text *, const char *, int *, size_t *) { return HAILO_STUB_ERR; }
 
 int yu_hailort_llm_create(const char *, const char *, bool, YuHailortLlm **out)

yu-hailo-infer/src/hailort/speech2text.rs

@@ -1,6 +1,8 @@
 use std::ffi::{c_char, c_float, c_int, CString};
 use std::ptr::NonNull;
 
+use serde::Deserialize;
+
 use super::{check_status, ffi, HailoRtError, HailoRtResult};
 
 pub(super) enum YuHailortSpeech2Text {}
@@ -11,6 +13,15 @@ pub(crate) enum Speech2TextTask {
     Translate = 1,
 }
 
+/// One transcribed segment, mirroring the SDK's `Speech2Text::SegmentInfo`
+/// (`generate_all_segments()`).
+#[derive(Debug, Clone, Deserialize)]
+pub(crate) struct Segment {
+    pub(crate) start_sec: f32,
+    pub(crate) end_sec: f32,
+    pub(crate) text: String,
+}
+
 extern "C" {
     pub(super) fn yu_hailort_s2t_create(
         model_path: *const c_char,
@@ -27,6 +38,16 @@ extern "C" {
         timeout_ms: u32,
         out_text: *mut *mut c_char,
     ) -> ffi::HailoStatus;
+    fn yu_hailort_s2t_generate_segments(
+        ctx: *mut YuHailortSpeech2Text,
+        audio: *const c_float,
+        audio_count: usize,
+        task: c_int,
+        language: *const c_char,
+        repetition_penalty: c_float,
+        timeout_ms: u32,
+        out_json: *mut *mut c_char,
+    ) -> ffi::HailoStatus;
     fn yu_hailort_s2t_tokenize(
         ctx: *mut YuHailortSpeech2Text,
         text: *const c_char,
@@ -79,6 +100,40 @@ impl Speech2Text {
         take_c_string(out)
     }
 
+    /// Same generation as [`Speech2Text::generate_text`], but returns the
+    /// SDK's per-segment breakdown (start/end timestamps + text) instead of
+    /// a single flattened string. Python's `s2t_inference.transcribe_segments()`
+    /// returns this shape; the flat-text path silently drops it.
+    pub(crate) fn generate_segments(
+        &mut self,
+        audio: &[f32],
+        task: Speech2TextTask,
+        language: Option<&str>,
+        repetition_penalty: f32,
+        timeout_ms: u32,
+    ) -> HailoRtResult<Vec<Segment>> {
+        let language = CString::new(language.unwrap_or(""))?;
+        let mut out = std::ptr::null_mut();
+        // SAFETY: audio slice and language C string live until call returns; out pointer is valid.
+        let status = unsafe {
+            yu_hailort_s2t_generate_segments(
+                self.raw.as_ptr(),
+                audio.as_ptr(),
+                audio.len(),
+                task as c_int,
+                language.as_ptr(),
+                repetition_penalty,
+                timeout_ms,
+                &mut out,
+            )
+        };
+        check_status("s2t_generate_segments", status)?;
+        let json = take_c_string(out)?;
+        serde_json::from_str(&json).map_err(|_| {
+            HailoRtError::InvalidMetadata("shim returned malformed segments JSON")
+        })
+    }
+
     pub(crate) fn tokenize(&mut self, text: &str) -> HailoRtResult<Vec<i32>> {
         let text = CString::new(text)?;
         let mut count = 0usize;

yu-hailo-infer/src/speech2text_route.rs

@@ -246,7 +246,7 @@ async fn speech2text_transcribe(
         let hef_path_str = hef_path_str.clone();
         move |ctx| {
             ctx.speech2text(&hef_path_str).and_then(|mut s2t| {
-                s2t.generate_text(
+                s2t.generate_segments(
                     &audio,
                     task,
                     language.as_deref(),
@@ -258,7 +258,34 @@ async fn speech2text_transcribe(
     })
     .await;
     match result {
-        Ok(text) => api_ok(json!({"hef_path": hef_path_str, "text": text, "task": task_name})),
+        Ok(segments) => {
+            // The flat `text` field is kept for callers that only want the
+            // transcript (and for backward compat with the pre-segment
+            // response shape) -- it is exactly what generate_all_text()
+            // would have produced, joined the same way Whisper's own
+            // segment-to-text concatenation does.
+            let text = segments
+                .iter()
+                .map(|segment| segment.text.as_str())
+                .collect::<Vec<_>>()
+                .join(" ");
+            let segments_json: Vec<_> = segments
+                .into_iter()
+                .map(|segment| {
+                    json!({
+                        "text": segment.text,
+                        "start": segment.start_sec,
+                        "end": segment.end_sec,
+                    })
+                })
+                .collect();
+            api_ok(json!({
+                "hef_path": hef_path_str,
+                "text": text,
+                "segments": segments_json,
+                "task": task_name,
+            }))
+        }
         Err(error) => hailort_api_error(error, "hailort_s2t_generate_failed"),
     }
 }
@@ -561,5 +588,25 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(response.status(), StatusCode::OK);
+
+        // Lock in the response contract added for segment support: `segments`
+        // must be present and array-shaped even for silence (which may
+        // legitimately transcribe to zero segments) -- this is what would
+        // regress silently if generate_all_segments()'s JSON marshaling
+        // broke, since the flat `text` field alone wouldn't catch it.
+        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("read response body");
+        let parsed: serde_json::Value =
+            serde_json::from_slice(&body).expect("response body is valid JSON");
+        let data = &parsed["data"];
+        assert!(
+            data["segments"].is_array(),
+            "expected data.segments to be an array, got: {parsed}"
+        );
+        assert!(
+            data["text"].is_string(),
+            "expected data.text to be a string, got: {parsed}"
+        );
     }
 }