feat: add browser audio model research and update model session infrastructure

Author: Jefreesujit

src/models.ts

@@ -326,6 +326,30 @@ export const CURATED_MODELS: ModelDescriptor[] = [
       visionLoaderKind: "qwen3_5",
     },
   }),
+  createVisionModel({
+    id: "LiquidAI/LFM2.5-VL-450M-ONNX",
+    label: "LFM2.5-VL 450M",
+    summary: "Liquid AI's compact vision-language model with bounding box prediction and multilingual support.",
+    publisher: "LiquidAI",
+    paramsLabel: "450M params",
+    parameterTier: "S",
+    estimatedDownloadLabel: "~650 MB download (fp16+q4)",
+    category: "vision",
+    starter: false,
+    tested: true,
+    hf: {
+      modelId: "LiquidAI/LFM2.5-VL-450M-ONNX",
+      pipelineTag: "image-text-to-text",
+      libraryName: "transformers.js",
+      tags: ["onnx", "image-text-to-text", "conversational", "webgpu"],
+      baseModel: "LiquidAI/LFM2.5-VL-450M",
+      hasChatTemplate: true,
+    },
+    runtime: {
+      contextWindowTokens: 32768,
+      visionLoaderKind: "lfm2_5_vl",
+    },
+  }),
   createTextModel({
     id: "HuggingFaceTB/SmolLM3-3B-ONNX",
     label: "SmolLM3 3B",

src/types.ts

@@ -75,7 +75,7 @@ export type CuratedCategoryKey =
   | "vision"
   | "desktop_experimental";
 export type Dtype = "q4f16" | "q4" | "q8" | "int8" | "uint8" | "fp16" | "fp32";
-export type VisionLoaderKind = "qwen3_5";
+export type VisionLoaderKind = "qwen3_5" | "lfm2_5_vl";
 export type ChatPersistenceStatus =
   | "ready"
   | "fallback_local_storage"

src/worker/generation.ts

@@ -278,9 +278,16 @@ export const generateVisionReply = async ({
     },
   );
   const rawImage = image ? await RawImage.read(image) : null;
-  const inputs = rawImage
-    ? await processor(prompt, rawImage)
-    : await processor(prompt);
+  const inputs = await (() => {
+    if (model.runtime.visionLoaderKind === "lfm2_5_vl") {
+      // LFM2.5-VL requires image first, then text, with add_special_tokens:false
+      // to avoid the processor double-adding special tokens
+      return rawImage
+        ? processor(rawImage, prompt, { add_special_tokens: false })
+        : processor(prompt, { add_special_tokens: false });
+    }
+    return rawImage ? processor(prompt, rawImage) : processor(prompt);
+  })();
 
   let streamedText = "";
   const stoppingCriteria = new InterruptableStoppingCriteria();

src/worker/model-session.ts

@@ -1,5 +1,6 @@
 import type { InterruptableStoppingCriteria } from "@huggingface/transformers";
 import {
+  AutoModelForImageTextToText,
   AutoProcessor,
   pipeline,
   Qwen3_5ForConditionalGeneration,
@@ -178,33 +179,57 @@ export const createModelSession = (postMessageToUi: WorkerMessagePoster) => {
   };
 
   const loadVisionResources = async (model: ModelDescriptor) => {
-    if (model.runtime.visionLoaderKind !== "qwen3_5") {
-      throw new Error(
-        "This vision loader is not supported in the browser worker yet.",
-      );
+    const progressHandler = getProgressHandler(model.id);
+
+    if (model.runtime.visionLoaderKind === "qwen3_5") {
+      const [nextProcessor, nextVisionModel] = await Promise.all([
+        AutoProcessor.from_pretrained(model.hf.modelId, {
+          progress_callback: progressHandler,
+        }),
+        Qwen3_5ForConditionalGeneration.from_pretrained(model.hf.modelId, {
+          dtype: {
+            embed_tokens: "q4",
+            vision_encoder: "fp16",
+            decoder_model_merged: "q4",
+          },
+          device: "webgpu",
+          progress_callback: progressHandler,
+        }),
+      ]);
+
+      return {
+        textGenerator: null,
+        processor: nextProcessor,
+        visionModel: nextVisionModel as VisionModelInstance,
+      } satisfies LoadResources;
     }
 
-    const progressHandler = getProgressHandler(model.id);
-    const [nextProcessor, nextVisionModel] = await Promise.all([
-      AutoProcessor.from_pretrained(model.hf.modelId, {
-        progress_callback: progressHandler,
-      }),
-      Qwen3_5ForConditionalGeneration.from_pretrained(model.hf.modelId, {
-        dtype: {
-          embed_tokens: "q4",
-          vision_encoder: "fp16",
-          decoder_model_merged: "q4",
-        },
-        device: "webgpu",
-        progress_callback: progressHandler,
-      }),
-    ]);
+    if (model.runtime.visionLoaderKind === "lfm2_5_vl") {
+      const [nextProcessor, nextVisionModel] = await Promise.all([
+        AutoProcessor.from_pretrained(model.hf.modelId, {
+          progress_callback: progressHandler,
+        }),
+        AutoModelForImageTextToText.from_pretrained(model.hf.modelId, {
+          dtype: {
+            embed_tokens: "fp16",
+            vision_encoder: "fp16",
+            decoder_model_merged: "q4",
+          },
+          device: "webgpu",
+          progress_callback: progressHandler,
+        }),
+      ]);
+
+      return {
+        textGenerator: null,
+        processor: nextProcessor,
+        visionModel: nextVisionModel as VisionModelInstance,
+      } satisfies LoadResources;
+    }
 
-    return {
-      textGenerator: null,
-      processor: nextProcessor,
-      visionModel: nextVisionModel,
-    } satisfies LoadResources;
+    throw new Error(
+      "This vision loader is not supported in the browser worker yet.",
+    );
   };
 
   const ensureModelReady = async (model: ModelDescriptor) => {