feat(api): support more OpenAI types (image, embeddings, audio-transcriptions, audio-speech) (#1297)

Integrated into release/v3.6.6 — adds embeddings, audio-transcriptions, audio-speech, and images-generations support for custom OpenAI-compatible providers, plus Pollinations image registry

Author: hartmark

open-sse/config/imageRegistry.ts

@@ -150,6 +150,26 @@ export const IMAGE_PROVIDERS = {
     ],
     supportedSizes: ["1024x1024", "1024x1792", "1792x1024", "256x256", "512x512"],
   },
+
+  pollinations: {
+    id: "pollinations",
+    alias: "pol",
+    baseUrl: "https://gen.pollinations.ai/v1/images/generations",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "openai",
+    models: [
+      { id: "flux", name: "Flux Schnell" },
+      { id: "zimage", name: "Z-Image Turbo" },
+      { id: "klein", name: "FLUX.2 Klein 4B" },
+      { id: "gptimage", name: "GPT Image 1 Mini" },
+      { id: "qwen-image", name: "Qwen Image Plus" },
+      { id: "wan-image", name: "Wan 2.7 Image" },
+      { id: "kontext", name: "FLUX.1 Kontext" },
+      { id: "gptimage-large", name: "GPT Image 1.5" },
+    ],
+    supportedSizes: ["1024x1024", "512x512"],
+  },
 };
 
 /**
@@ -171,6 +191,10 @@ export function parseImageModel(modelStr) {
     if (modelStr.startsWith(providerId + "/")) {
       return { provider: providerId, model: modelStr.slice(providerId.length + 1) };
     }
+    // Check alias if available
+    if (config.alias && modelStr.startsWith(config.alias + "/")) {
+      return { provider: providerId, model: modelStr.slice(config.alias.length + 1) };
+    }
   }
 
   // No provider prefix — try to find the model in every provider

open-sse/services/provider.ts

@@ -37,15 +37,38 @@ export function getOpenAICompatibleType(provider, providerSpecificData = null) {
     typeof providerSpecificData.apiType === "string"
       ? providerSpecificData.apiType
       : null;
-  if (configuredType === "responses" || configuredType === "chat") {
+  if (
+    configuredType === "responses" ||
+    configuredType === "chat" ||
+    configuredType === "embeddings" ||
+    configuredType === "audio-transcriptions" ||
+    configuredType === "audio-speech" ||
+    configuredType === "images-generations"
+  ) {
     return configuredType;
   }
-  return provider.includes("responses") ? "responses" : "chat";
+  if (provider.includes("responses")) return "responses";
+  if (provider.includes("embeddings")) return "embeddings";
+  if (provider.includes("audio-transcriptions")) return "audio-transcriptions";
+  if (provider.includes("audio-speech")) return "audio-speech";
+  if (provider.includes("images-generations")) return "images-generations";
+  return "chat";
 }
 
 function buildOpenAICompatibleUrl(baseUrl, apiType) {
   const normalized = baseUrl.replace(/\/$/, "");
-  const path = apiType === "responses" ? "/responses" : "/chat/completions";
+  let path = "/chat/completions";
+  if (apiType === "responses") {
+    path = "/responses";
+  } else if (apiType === "embeddings") {
+    path = "/embeddings";
+  } else if (apiType === "audio-transcriptions") {
+    path = "/audio/transcriptions";
+  } else if (apiType === "audio-speech") {
+    path = "/audio/speech";
+  } else if (apiType === "images-generations") {
+    path = "/images/generations";
+  }
   return `${normalized}${path}`;
 }
 

src/app/(dashboard)/dashboard/providers/[id]/page.tsx

@@ -1004,6 +1004,50 @@ export default function ProviderDetailPage() {
   const providerStorageAlias = isCompatible ? providerId : providerAlias;
   const providerDisplayAlias = isCompatible ? providerNode?.prefix || providerId : providerAlias;
 
+  const getApiLabel = () => {
+    if (isAnthropicProtocolCompatible) return t("messagesApi");
+    const type = providerNode?.apiType;
+    switch (type) {
+      case "responses":
+        return t("responsesApi");
+      case "embeddings":
+        return t("embeddings");
+      case "audio-transcriptions":
+        return t("audioTranscriptions");
+      case "audio-speech":
+        return t("audioSpeech");
+      case "images-generations":
+        return t("imagesGenerations");
+      default:
+        return t("chatCompletions");
+    }
+  };
+
+  const getApiDefaultPath = () => {
+    if (isCcCompatible) return CC_COMPATIBLE_DEFAULT_CHAT_PATH;
+    if (isAnthropicCompatible) return "/messages";
+    const type = providerNode?.apiType;
+    switch (type) {
+      case "responses":
+        return "/responses";
+      case "embeddings":
+        return "/embeddings";
+      case "audio-transcriptions":
+        return "/audio/transcriptions";
+      case "audio-speech":
+        return "/audio/speech";
+      case "images-generations":
+        return "/images/generations";
+      default:
+        return "/chat/completions";
+    }
+  };
+
+  const getApiPath = () => {
+    const defaultPath = getApiDefaultPath();
+    return (providerNode?.chatPath || defaultPath).replace(/^\//, "");
+  };
+
   // Define callbacks BEFORE the useEffect that uses them
   const fetchAliases = useCallback(async () => {
     try {
@@ -2495,19 +2539,7 @@ export default function ProviderDetailPage() {
                     : t("openaiCompatibleDetails")}
               </h2>
               <p className="text-sm text-text-muted">
-                {isAnthropicProtocolCompatible
-                  ? t("messagesApi")
-                  : providerNode.apiType === "responses"
-                    ? t("responsesApi")
-                    : t("chatCompletions")}{" "}
-                · {(providerNode.baseUrl || "").replace(/\/$/, "")}/
-                {isCcCompatible
-                  ? (providerNode.chatPath || CC_COMPATIBLE_DEFAULT_CHAT_PATH).replace(/^\//, "")
-                  : isAnthropicCompatible
-                    ? (providerNode.chatPath || "/messages").replace(/^\//, "")
-                    : providerNode.apiType === "responses"
-                      ? (providerNode.chatPath || "/responses").replace(/^\//, "")
-                      : (providerNode.chatPath || "/chat/completions").replace(/^\//, "")}
+                {getApiLabel()} · {(providerNode.baseUrl || "").replace(/\/$/, "")}/{getApiPath()}
               </p>
             </div>
             <div className="flex items-center gap-2">
@@ -3845,8 +3877,12 @@ function CustomModelsSection({
               onChange={(e) => setNewApiFormat(e.target.value)}
               className="w-full px-3 py-2 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
             >
-              <option value="chat-completions">Chat Completions</option>
-              <option value="responses">Responses API</option>
+              <option value="chat-completions">{t("chatCompletions")}</option>
+              <option value="responses">{t("responsesApi")}</option>
+              <option value="embeddings">{t("embeddings")}</option>
+              <option value="audio-transcriptions">{t("audioTranscriptions")}</option>
+              <option value="audio-speech">{t("audioSpeech")}</option>
+              <option value="images-generations">{t("imagesGenerations")}</option>
             </select>
           </div>
           <div className="flex-1">
@@ -3972,8 +4008,12 @@ function CustomModelsSection({
                             onChange={(e) => setEditingApiFormat(e.target.value)}
                             className="w-full px-2.5 py-2 text-xs border border-border rounded-lg bg-background text-text-main focus:outline-none focus:border-primary"
                           >
-                            <option value="chat-completions">Chat Completions</option>
-                            <option value="responses">Responses API</option>
+                            <option value="chat-completions">{t("chatCompletions")}</option>
+                            <option value="responses">{t("responsesApi")}</option>
+                            <option value="embeddings">{t("embeddings")}</option>
+                            <option value="audio-transcriptions">{t("audioTranscriptions")}</option>
+                            <option value="audio-speech">{t("audioSpeech")}</option>
+                            <option value="images-generations">{t("imagesGenerations")}</option>
                           </select>
                         </div>
                         <div className="flex min-w-0 flex-1 flex-wrap items-center gap-x-3 gap-y-1 overflow-x-auto overflow-y-visible [scrollbar-width:thin]">
@@ -6092,6 +6132,10 @@ function EditCompatibleNodeModal({
   const apiTypeOptions = [
     { value: "chat", label: t("chatCompletions") },
     { value: "responses", label: t("responsesApi") },
+    { value: "embeddings", label: t("embeddings") },
+    { value: "audio-transcriptions", label: t("audioTranscriptions") },
+    { value: "audio-speech", label: t("audioSpeech") },
+    { value: "images-generations", label: t("imagesGenerations") },
   ];
 
   const handleSubmit = async () => {

src/app/(dashboard)/dashboard/providers/page.tsx

@@ -1149,6 +1149,10 @@ function AddOpenAICompatibleModal({ isOpen, onClose, onCreated }) {
   const apiTypeOptions = [
     { value: "chat", label: t("chatCompletions") },
     { value: "responses", label: t("responsesApi") },
+    { value: "embeddings", label: t("embeddings") },
+    { value: "audio-transcriptions", label: t("audioTranscriptions") },
+    { value: "audio-speech", label: t("audioSpeech") },
+    { value: "images-generations", label: t("imagesGenerations") },
   ];
 
   useEffect(() => {

src/app/api/provider-nodes/[id]/route.ts

@@ -62,10 +62,15 @@ export async function PUT(request: Request, { params }: { params: Promise<{ id:
     }
 
     // Only validate apiType for OpenAI Compatible nodes
-    if (
-      node.type === "openai-compatible" &&
-      (!apiType || !["chat", "responses"].includes(apiType))
-    ) {
+    const validApiTypes = [
+      "chat",
+      "responses",
+      "embeddings",
+      "audio-transcriptions",
+      "audio-speech",
+      "images-generations",
+    ];
+    if (node.type === "openai-compatible" && (!apiType || !validApiTypes.includes(apiType))) {
       return NextResponse.json({ error: "Invalid OpenAI compatible API type" }, { status: 400 });
     }
 

src/app/api/v1/embeddings/route.ts

@@ -120,9 +120,10 @@ export async function POST(request) {
     const nodes = (await getProviderNodes()) as unknown as EmbeddingProviderNodeRow[];
     dynamicProviders = (Array.isArray(nodes) ? nodes : [])
       .filter((n) => {
-        // provider_nodes apiType is "chat" or "responses" (not "embeddings") — local OpenAI-compatible
+        // provider_nodes apiType is "chat", "responses" or "embeddings" — local OpenAI-compatible
         // backends expose /embeddings under the same base URL as chat, so we build the URL as baseUrl + /embeddings.
-        if (n.apiType !== "chat" && n.apiType !== "responses") return false;
+        const validTypes = ["chat", "responses", "embeddings"];
+        if (!validTypes.includes(n.apiType || "")) return false;
         try {
           const hostname = new URL(n.baseUrl).hostname;
           // Strictly matching 172.16.0.0/12 (Docker/local) and explicitly blocking ::1 per SSRF hardening
@@ -170,7 +171,9 @@ export async function POST(request) {
       const allNodes = (await getProviderNodes()) as unknown as EmbeddingProviderNodeRow[];
       const matchingNode = (Array.isArray(allNodes) ? allNodes : []).find(
         (n) =>
-          n.prefix === provider && (n.apiType === "chat" || n.apiType === "responses") && n.baseUrl
+          n.prefix === provider &&
+          (n.apiType === "chat" || n.apiType === "responses" || n.apiType === "embeddings") &&
+          n.baseUrl
       );
       if (matchingNode) {
         const baseUrl = String(matchingNode.baseUrl).replace(/\/+$/, "");

src/i18n/messages/en.json

@@ -1704,6 +1704,10 @@
     "openaiCompatibleDetails": "OpenAI Compatible Details",
     "messagesApi": "Messages API",
     "responsesApi": "Responses API",
+    "embeddings": "Embeddings",
+    "audioTranscriptions": "Audio Transcriptions",
+    "audioSpeech": "Audio Speech",
+    "imagesGenerations": "Images Generations",
     "chatCompletions": "Chat Completions",
     "importingModels": "Importing...",
     "importFromModels": "Import from /models",

src/lib/db/models.ts

@@ -343,7 +343,13 @@ export async function addCustomModel(
   modelId: string,
   modelName?: string,
   source = "manual",
-  apiFormat: "chat-completions" | "responses" = "chat-completions",
+  apiFormat:
+    | "chat-completions"
+    | "responses"
+    | "embeddings"
+    | "audio-transcriptions"
+    | "audio-speech"
+    | "images-generations" = "chat-completions",
   supportedEndpoints: string[] = ["chat"]
 ) {
   const db = getDbInstance();

src/shared/validation/schemas.ts

@@ -516,8 +516,8 @@ export const providerModelMutationSchema = z.object({
   modelId: z.string().trim().min(1, "modelId is required").max(240),
   modelName: z.string().trim().max(240).optional(),
   source: z.string().trim().max(80).optional(),
-  apiFormat: z.enum(["chat-completions", "responses"]).default("chat-completions"),
-  supportedEndpoints: z.array(z.enum(["chat", "embeddings", "images", "audio"])).default(["chat"]),
+  apiFormat: z.enum(["chat-completions", "responses", "embeddings", "audio-transcriptions", "audio-speech", "images-generations"]).default("chat-completions"),
+  supportedEndpoints: z.array(z.enum(["chat", "embeddings", "images", "audio", "audio-transcriptions", "audio-speech", "images-generations"])).default(["chat"]),
   normalizeToolCallId: z.boolean().optional(),
   preserveOpenAIDeveloperRole: z.boolean().nullable().optional(),
   upstreamHeaders: upstreamHeadersRecordSchema.nullable().optional(),
@@ -1124,7 +1124,7 @@ export const createProviderNodeSchema = z
   .object({
     name: z.string().trim().min(1, "Name is required"),
     prefix: z.string().trim().min(1, "Prefix is required"),
-    apiType: z.enum(["chat", "responses"]).optional(),
+    apiType: z.enum(["chat", "responses", "embeddings", "audio-transcriptions", "audio-speech", "images-generations"]).optional(),
     baseUrl: z.string().trim().min(1).optional(),
     type: z.enum(["openai-compatible", "anthropic-compatible"]).optional(),
     compatMode: z.enum(["cc"]).optional(),
@@ -1145,7 +1145,7 @@ export const createProviderNodeSchema = z
 export const updateProviderNodeSchema = z.object({
   name: z.string().trim().min(1, "Name is required"),
   prefix: z.string().trim().min(1, "Prefix is required"),
-  apiType: z.enum(["chat", "responses"]).optional(),
+  apiType: z.enum(["chat", "responses", "embeddings", "audio-transcriptions", "audio-speech", "images-generations"]).optional(),
   baseUrl: z.string().trim().min(1, "Base URL is required"),
   chatPath: z.string().trim().startsWith("/").max(500).optional().or(z.literal("")),
   modelsPath: z.string().trim().startsWith("/").max(500).optional().or(z.literal("")),

tests/manual/embedding.http

@@ -0,0 +1,11 @@
+###
+# @name Embedding - Test embedding response
+POST {{omniroute-address}}/v1/embeddings
+Authorization: Bearer {{OMNIROUTE_API_KEY}}
+Content-Type: application/json
+
+{
+  "model": "pinecone/llama-text-embed-v2",
+  "input": "The food was delicious and the waiter was very attentive."
+}
+