From afc8330daf35728415e06f74a654a01bfe38d00c Mon Sep 17 00:00:00 2001 From: x3M3x Date: Mon, 31 Aug 2026 10:44:18 +0400 Subject: [PATCH 1/5] fix(catalog): advertise image input for modalities-declared sidecar rows The runtime vision gate (isModelTextOnly) covers models listed in noVisionModels OR declared text-only via modelInputModalities (upstream fde2a9537, #1024), but both catalog advertise sites only checked noVisionModels. A sidecar-covered model - and every combo built from it - stayed advertised text-only in /v1/models, so the Codex app blocked image attachments client-side before the sidecar could run ('This model does not support image inputs'). Mirror isModelTextOnly in applyProviderConfigHints and the custom-model override: a declared text-only modelInputModalities entry now advertises image on top of its configured base. Discovery-derived text-only rows stay untouched (the runtime predicate does not cover those), and declared-image rows are never duplicated. Combos inherit the fix through their hinted members; no config hand-editing needed. Update the three tests that encoded the old drift and add regressions for the hint pass, the custom-model override, and combo derivation. (cherry picked from commit ed8f5a4fd30cbee26a658e28211e187af73421aa) --- docs-site/src/content/docs/guides/sidecars.md | 7 +- src/codex/catalog/provider-fetch.ts | 23 ++++- .../catalog-vision-sidecar-modalities.test.ts | 92 +++++++++++++++++++ tests/codex-catalog.test.ts | 19 +++- 4 files changed, 129 insertions(+), 12 deletions(-) diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index bb4c4346df..08300ae87b 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -123,8 +123,11 @@ failures after response headers have started are delivered as `response.failed` ## Vision sidecar -When the routed model is listed in its provider's `noVisionModels` and a request carries an image, -opencodex describes each image **before** the main call and replaces it with text. When +When the routed model is listed in its provider's `noVisionModels` — or declared text-only for +that model via `modelInputModalities` — and a request carries an image, opencodex describes each +image **before** the main call and replaces it with text. The model catalog advertises image input +for every sidecar-covered model (combos advertise image when every member is covered), so clients +such as the Codex app allow attachments instead of blocking them before the sidecar runs. When `visionSidecar.model` is absent or blank, the OpenAI execution path, Dashboard, and management API use the `gpt-5.4-mini` fallback. Startup still migrates an explicitly persisted legacy `gpt-5.4-mini` value to `gpt-5.6-luna`; that migration applies to a stored value, not to an absent diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index 87afd0bb12..882ea1e6d8 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -670,11 +670,16 @@ export function applyProviderConfigHints(name: string, prov: OcxProviderConfig, const configuredMaxInput = configuredMaxInputTokens(prov, model.id); const configuredAutoCompact = configuredAutoCompactTokenLimit(prov, model.id); let inputModalities = configuredInputModalities(prov, model.id); - // Vision-sidecar coverage: `noVisionModels` marks models whose images the PROXY describes - // (src/vision/index.ts). The catalog must still advertise image input for them — the Codex app + // Vision-sidecar coverage mirrors isModelTextOnly (src/vision/index.ts): `noVisionModels` OR + // a `modelInputModalities` declaration excluding "image" both mean the PROXY describes images + // for this model at request time. The catalog must still advertise image input — the Codex app // gates attachments client-side on input_modalities, and a text-only entry would block images - // before the sidecar ever runs ("This model does not support image inputs"). - if (modelInList(prov.noVisionModels, model.id)) { + // before the sidecar ever runs ("This model does not support image inputs"). Discovery-derived + // text-only rows stay untouched: the runtime predicate only reads these two config sources, so + // it would not convert those. + const sidecarCovered = modelInList(prov.noVisionModels, model.id) + || (Array.isArray(inputModalities) && inputModalities.length > 0 && !inputModalities.includes("image")); + if (sidecarCovered) { const base = inputModalities ?? model.inputModalities ?? ["text"]; inputModalities = base.includes("image") ? [...base] : [...base, "image"]; } @@ -2141,7 +2146,15 @@ async function gatherRoutedModelsUncached( } : mergedWithHardBounds; const enrichedProvider = enrichedByName.get(cm.provider) ?? rawProvider; - if (enrichedProvider && modelInList(enrichedProvider.noVisionModels, mergedWithAutoCompact.id)) { + // Same vision-sidecar coverage rule as applyProviderConfigHints (isModelTextOnly parity): + // noVisionModels OR a text-only modelInputModalities declaration means the sidecar converts + // images at request time, so the custom row must advertise image input. + const declaredModalities = enrichedProvider + ? modelRecordValue(enrichedProvider.modelInputModalities, mergedWithAutoCompact.id) + : undefined; + if (enrichedProvider + && (modelInList(enrichedProvider.noVisionModels, mergedWithAutoCompact.id) + || (Array.isArray(declaredModalities) && declaredModalities.length > 0 && !declaredModalities.includes("image")))) { const current = mergedWithAutoCompact.inputModalities ?? ["text"]; if (!current.includes("image")) { return { ...mergedWithAutoCompact, inputModalities: [...current, "image"] }; diff --git a/tests/catalog-vision-sidecar-modalities.test.ts b/tests/catalog-vision-sidecar-modalities.test.ts index f5ff51431d..253b8cc8e2 100644 --- a/tests/catalog-vision-sidecar-modalities.test.ts +++ b/tests/catalog-vision-sidecar-modalities.test.ts @@ -48,6 +48,40 @@ describe("vision-sidecar catalog modalities", () => { expect(hinted.inputModalities).toEqual(["text", "image"]); }); + test("modelInputModalities-declared text-only models advertise image without a noVisionModels entry", () => { + // Regression: isModelTextOnly (the RUNTIME sidecar gate) treats a text-only + // modelInputModalities declaration exactly like a noVisionModels entry, but the catalog hint + // pass only checked noVisionModels — so sidecar-covered models stayed advertised text-only + // and the Codex app blocked image paste before the sidecar could run. + const prov: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://api.example/v1", + modelInputModalities: { "deepseek-chat": ["text"] }, + }; + const hinted = applyProviderConfigHints("azu-lab2", prov, { id: "deepseek-chat", provider: "azu-lab2" }); + expect(hinted.inputModalities).toEqual(["text", "image"]); + }); + + test("modelInputModalities declaring image stays untouched (no duplication)", () => { + const prov: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://api.example/v1", + modelInputModalities: { "glm-5.3": ["text", "image"] }, + }; + const hinted = applyProviderConfigHints("vdi", prov, { id: "glm-5.3", provider: "vdi" }); + expect(hinted.inputModalities).toEqual(["text", "image"]); + }); + + test("discovery-derived text-only rows are NOT advertised image (the runtime would not convert them)", () => { + // Only the two config sources the runtime predicate reads (noVisionModels, + // modelInputModalities) may widen the catalog; a listing that merely reports + // ["text"] without either must stay text-only. + const hinted = applyProviderConfigHints("opencode-go", base, { + id: "listing-text-model", provider: "opencode-go", inputModalities: ["text"], + }); + expect(hinted.inputModalities).toEqual(["text"]); + }); + test("MiMo token-plan sends only the Pro model through the sidecar (#1927)", () => { const canonical: OcxProviderConfig = { adapter: "openai-chat", @@ -173,6 +207,39 @@ describe("vision-sidecar custom-model override (#349/#344)", () => { globalThis.fetch = originalFetch; } }); + + test("a custom row whose modelId is declared text-only via modelInputModalities still advertises image", async () => { + // Same isModelTextOnly parity as the hint pass, applied to the custom-model override path: + // the registry/config text-only declaration covers the row at request time, so the catalog + // must let images through to the sidecar here too. + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { throw new Error("fetch should not be called"); }) as typeof fetch; + try { + const models = await gatherRoutedModels({ + port: 10100, + defaultProvider: "text-sidecar-provider", + providers: { + "text-sidecar-provider": { + baseUrl: "https://text-sidecar.example/v1", + adapter: "openai-chat", + authMode: "key", + liveModels: false, + models: ["baseline-model"], + modelInputModalities: { "glm-5.2": ["text"] }, + }, + }, + customModels: [ + { id: "cm-4", provider: "text-sidecar-provider", modelId: "glm-5.2", displayName: "GLM 5.2", addedAt: "2026-01-01T00:00:00.000Z" }, + ], + }); + const custom = models.find(m => m.provider === "text-sidecar-provider" && m.id === "glm-5.2"); + expect(custom).toBeDefined(); + expect(custom?.inputModalities).toEqual(["text", "image"]); + } finally { + globalThis.fetch = originalFetch; + clearModelCache("text-sidecar-provider"); + } + }); }); describe("vision-capable provider models feed combo modalities", () => { @@ -258,6 +325,31 @@ describe("vision-capable provider models feed combo modalities", () => { const hinted = applyProviderConfigHints("xai", prov, { provider: "xai", id: "grok-4.5" }); expect(hinted.inputModalities).toEqual(["text", "image"]); }); + + test("a combo of modelInputModalities-declared text-only members advertises image through the hints", () => { + // End-to-end shape of the /planners bug: every member is sidecar-covered via + // modelInputModalities (not noVisionModels). The hinted members all carry image, so the + // derived combo keeps image input instead of collapsing to text-only. + const memberFor = (provider: string, id: string): CatalogModel => applyProviderConfigHints(provider, { + adapter: "openai-chat", + baseUrl: `https://${provider}.example/v1`, + modelInputModalities: { [id]: ["text"] }, + }, { id, provider, contextWindow: 200_000 }); + const memberA = memberFor("azu-lab2", "deepseek-chat"); + const memberB = memberFor("vdi", "glm-5.2"); + const derived = deriveComboCatalogModel( + "planners", + { + targets: [ + { provider: "azu-lab2", model: "deepseek-chat" }, + { provider: "vdi", model: "glm-5.2" }, + ], + defaultEffort: "high", + } as never, + [memberA, memberB], + ); + expect(derived?.inputModalities).toEqual(["text", "image"]); + }); }); describe("Cursor native vs sidecar vision registry", () => { diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index f174acbcde..123cd15d0b 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -1061,7 +1061,9 @@ describe("combo catalog capability intersection", () => { liveModels: false, models: ["m2"], modelContextWindows: { m2: 128_000 }, - modelInputModalities: { m2: ["text"] }, + // No declaration: text-only by catalog default, NOT sidecar-covered. A declared + // text-only member would widen to image (isModelTextOnly parity) and the pair + // would no longer be disjoint. }, }, combos: { @@ -1202,7 +1204,9 @@ describe("combo catalog capability intersection", () => { contextWindow: 128_000, contextCapped: false, maxInputTokens: 100_000, - inputModalities: ["text"], + // The text-only modelInputModalities declaration is sidecar-covered at runtime + // (isModelTextOnly), so the combo advertises image input like its member does. + inputModalities: ["text", "image"], reasoningEfforts: ["high"], }); expect(rows.find(row => row.provider === "combo" && row.id === "nova-sol")) @@ -1515,7 +1519,7 @@ describe("combo catalog capability intersection", () => { liveModels: false, models: ["m1"], modelContextWindows: { m1: 128_000 }, - // Disjoint modalities with b → empty intersection (incompatible_modalities). + // Image-only member. An image-only declaration is not sidecar-covered. modelInputModalities: { m1: ["image"] }, }, b: { @@ -1524,7 +1528,10 @@ describe("combo catalog capability intersection", () => { liveModels: false, models: ["m2"], modelContextWindows: { m2: 128_000 }, - modelInputModalities: { m2: ["audio"] }, + // No modality declaration: the member is text-only by catalog default and the + // sidecar does not cover it, so text and image stay genuinely disjoint + // (incompatible_modalities). A DECLARED text-only member would be widened to + // image (isModelTextOnly parity) and no longer be disjoint. }, }, combos: { @@ -5335,7 +5342,9 @@ describe("Codex catalog routed normalization", () => { expect(models.find(m => m.id === "wide-model")).toMatchObject({ contextWindow: 100_000, maxInputTokens: 100_000, - inputModalities: ["text"], + // Declared text-only modalities are sidecar-covered at runtime (isModelTextOnly), + // so the catalog advertises image on top of the configured base. + inputModalities: ["text", "image"], }); expect(models.find(m => m.id === "small-model")?.contextWindow).toBe(64_000); }); From 4735fe5ca65d4dc25df454b1b01ea4da6cf8d342 Mon Sep 17 00:00:00 2001 From: x3M3x Date: Mon, 31 Aug 2026 11:11:21 +0400 Subject: [PATCH 2/5] docs(sidecars): qualify combo image advertising with imageInput setting (cherry picked from commit 77d614a9fccb66e739b960ba948fe13b0109cc7b) --- docs-site/src/content/docs/guides/sidecars.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index 08300ae87b..dfec4b5c58 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -126,8 +126,9 @@ failures after response headers have started are delivered as `response.failed` When the routed model is listed in its provider's `noVisionModels` — or declared text-only for that model via `modelInputModalities` — and a request carries an image, opencodex describes each image **before** the main call and replaces it with text. The model catalog advertises image input -for every sidecar-covered model (combos advertise image when every member is covered), so clients -such as the Codex app allow attachments instead of blocking them before the sidecar runs. When +for every sidecar-covered model. Combos advertise image only when every member is covered and +`imageInput` is not disabled, so clients such as the Codex app allow attachments instead of +blocking them before the sidecar runs. When `visionSidecar.model` is absent or blank, the OpenAI execution path, Dashboard, and management API use the `gpt-5.4-mini` fallback. Startup still migrates an explicitly persisted legacy `gpt-5.4-mini` value to `gpt-5.6-luna`; that migration applies to a stored value, not to an absent From 1a0ba1ed5a6f0af25cc3fc0e38d110451a593ae4 Mon Sep 17 00:00:00 2001 From: x3M3x Date: Mon, 31 Aug 2026 16:52:51 +0400 Subject: [PATCH 3/5] docs(sidecars): condition image description on sidecar plan availability Addresses the open CodeRabbit review comments: state that description runs only when a vision sidecar plan is available (raw image stripped otherwise, no description attempted), and name the combo imageInput setting explicitly with sidecar-covered membership. (cherry picked from commit 374139e1f538d21f24ca3dd9a6bf965be481908f) --- docs-site/src/content/docs/guides/sidecars.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index dfec4b5c58..93b43e3331 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -125,10 +125,12 @@ failures after response headers have started are delivered as `response.failed` When the routed model is listed in its provider's `noVisionModels` — or declared text-only for that model via `modelInputModalities` — and a request carries an image, opencodex describes each -image **before** the main call and replaces it with text. The model catalog advertises image input -for every sidecar-covered model. Combos advertise image only when every member is covered and -`imageInput` is not disabled, so clients such as the Codex app allow attachments instead of -blocking them before the sidecar runs. When +image **before** the main call and replaces it with text, provided a vision sidecar plan is +available. Without an available plan the raw image is stripped rather than forwarded to a +text-only backend. The model catalog advertises image input for every sidecar-covered model. +Combos advertise image input only when every member is sidecar-covered and the combo's +`imageInput` setting is not disabled, so clients such as the Codex app allow attachments instead +of blocking them before the sidecar runs. When `visionSidecar.model` is absent or blank, the OpenAI execution path, Dashboard, and management API use the `gpt-5.4-mini` fallback. Startup still migrates an explicitly persisted legacy `gpt-5.4-mini` value to `gpt-5.6-luna`; that migration applies to a stored value, not to an absent @@ -150,8 +152,8 @@ model field. remote `https` images are fetched by the OpenAI backend, not by the proxy. - `noVisionModels` matching ignores an Ollama-style `:size` suffix, so a `gpt-oss` entry also covers `gpt-oss:120b`. -- If description fails, the model receives a short processing-error marker. If no sidecar plan is - available, the raw image is stripped rather than forwarded to a text-only backend. +- If description fails, the model receives a short processing-error marker. (Without an available + sidecar plan, no description is attempted — the raw image is stripped, as described above.) - `maxDescriptionsPerTurn` (default 8) limits new descriptions per main-model turn. Cache hits and same-turn duplicates do not consume it. Successful `data:` image descriptions are cached by backend, model, detail, image bytes, and message context — plus the reasoning effort on OpenAI From 4b90f2836e6bee2cd14d733a409324faad55c522 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Tue, 1 Sep 2026 09:26:36 +0900 Subject: [PATCH 4/5] fix(vision): share sidecar consumer predicate --- .../src/content/docs/fr/guides/sidecars.md | 15 ++++--- docs-site/src/content/docs/guides/sidecars.md | 6 +-- .../src/content/docs/ja/guides/sidecars.md | 13 ++++-- .../src/content/docs/ko/guides/sidecars.md | 13 ++++-- .../src/content/docs/ru/guides/sidecars.md | 15 ++++--- .../src/content/docs/tr/guides/sidecars.md | 19 +++++---- .../src/content/docs/zh-cn/guides/sidecars.md | 11 +++-- .../src/content/docs/zh-tw/guides/sidecars.md | 11 +++-- src/codex/catalog/provider-fetch.ts | 20 +++------- src/vision/eligibility.ts | 20 +++++++++- src/vision/index.ts | 22 ++-------- .../catalog-vision-sidecar-modalities.test.ts | 40 +++++++++++++++++++ tests/vision-eligibility.test.ts | 19 +++++++++ tests/vision-text-only-predicate.test.ts | 4 ++ 14 files changed, 159 insertions(+), 69 deletions(-) diff --git a/docs-site/src/content/docs/fr/guides/sidecars.md b/docs-site/src/content/docs/fr/guides/sidecars.md index 763b02ae73..b778c4c99e 100644 --- a/docs-site/src/content/docs/fr/guides/sidecars.md +++ b/docs-site/src/content/docs/fr/guides/sidecars.md @@ -85,9 +85,14 @@ les échecs de génération postérieurs à l'envoi des en-têtes sont transmis ## Service auxiliaire de vision -Lorsque le modèle routé est répertorié dans le `noVisionModels` de son fournisseur et qu'une requête porte une image, -opencodex décrit chaque image **avant** l'appel principal et la remplace par du texte. Quand -si `visionSidecar.model` est absent ou vide, le chemin d'exécution OpenAI, le tableau de bord et l'API de gestion +Lorsqu'un modèle routé figure dans le `noVisionModels` de son fournisseur — ou est déclaré texte seul pour ce modèle +via `modelInputModalities` — et qu'une requête porte une image, opencodex décrit chaque image **avant** l'appel principal +et la remplace par du texte, à condition qu'un plan de sidecar vision soit disponible. Sans plan disponible, l'image brute +est supprimée au lieu d'être transmise à un backend texte seul. Le catalogue de modèles annonce l'entrée image pour chaque +modèle couvert par le sidecar. Les combos annoncent l'entrée image seulement lorsque chaque membre accepte les images, +nativement ou via un sidecar, et que le paramètre `imageInput` du combo n'est pas désactivé, afin que des clients comme +l'application Codex autorisent les pièces jointes au lieu de les bloquer avant l'exécution du sidecar. Lorsque +`visionSidecar.model` est absent ou vide, le chemin d'exécution OpenAI, le tableau de bord et l'API de gestion utilisent le modèle de repli `gpt-5.4-mini`. Au démarrage, une ancienne valeur `gpt-5.4-mini` explicitement enregistrée est toujours migrée vers `gpt-5.6-luna` ; cette migration s'applique à une valeur stockée, et non à l'absence du champ du modèle. @@ -108,8 +113,8 @@ champ du modèle. les images distantes `https` sont récupérées par le moteur OpenAI, et non par le proxy. - La correspondance `noVisionModels` ignore un suffixe `:size` de style Ollama, donc une entrée `gpt-oss` couvre également `gpt-oss:120b`. -- Si la description échoue, le modèle reçoit un bref marqueur d'erreur de traitement. Si aucun service auxiliaire n'est - disponible, l'image brute est supprimée plutôt que transmise à un moteur limité au texte. +- Si la description échoue, le modèle reçoit un bref marqueur d'erreur de traitement. (Sans plan de sidecar disponible, + aucune description n'est tentée : l'image brute est supprimée comme indiqué ci-dessus.) - `maxDescriptionsPerTurn` (8 par défaut) limite les nouvelles descriptions par tour du modèle principal. Les résultats du cache et les doublons au même tour ne le consomment pas. Les descriptions d'images `data:` réussies sont mises en cache par moteur, modèle, niveau de détail, octets de l'image et contexte du message — ainsi que l'effort de raisonnement dans les diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index 93b43e3331..0606129f39 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -128,9 +128,9 @@ that model via `modelInputModalities` — and a request carries an image, openco image **before** the main call and replaces it with text, provided a vision sidecar plan is available. Without an available plan the raw image is stripped rather than forwarded to a text-only backend. The model catalog advertises image input for every sidecar-covered model. -Combos advertise image input only when every member is sidecar-covered and the combo's -`imageInput` setting is not disabled, so clients such as the Codex app allow attachments instead -of blocking them before the sidecar runs. When +Combos advertise image input only when every member accepts images, either natively or through a +sidecar, and the combo's `imageInput` setting is not disabled, so clients such as the Codex app +allow attachments instead of blocking them before the sidecar runs. When `visionSidecar.model` is absent or blank, the OpenAI execution path, Dashboard, and management API use the `gpt-5.4-mini` fallback. Startup still migrates an explicitly persisted legacy `gpt-5.4-mini` value to `gpt-5.6-luna`; that migration applies to a stored value, not to an absent diff --git a/docs-site/src/content/docs/ja/guides/sidecars.md b/docs-site/src/content/docs/ja/guides/sidecars.md index 48af198fe3..dec111160f 100644 --- a/docs-site/src/content/docs/ja/guides/sidecars.md +++ b/docs-site/src/content/docs/ja/guides/sidecars.md @@ -69,8 +69,13 @@ stall は全体生成 timeout ではありません。SSE 開始前の失敗は ## ビジョンサイドカー -ルーティングモデルが該当プロバイダーの `noVisionModels` にありリクエストに画像が来る場合、opencodex は -メイン呼び出し**前に**各画像を説明したテキストに差し替えます。`visionSidecar.model` が未設定または空の場合、 +ルーティングモデルが該当プロバイダーの `noVisionModels` にある、またはそのモデルが +`modelInputModalities` でテキスト専用と宣言され、リクエストに画像が来る場合、opencodex は利用可能な +ビジョンサイドカー計画があるときに限り、メイン呼び出し**前に**各画像を説明したテキストに差し替えます。 +計画が利用できない場合は、生の画像をテキスト専用バックエンドへ転送せず削除します。モデルカタログは +サイドカー対象の各モデルに画像入力を広告します。コンボは、すべてのメンバーがネイティブまたはサイドカーを +通じて画像を受け入れ、かつコンボの `imageInput` 設定が無効でない場合にのみ画像入力を広告します。これにより +Codex アプリなどのクライアントは、サイドカー実行前に添付をブロックせず許可できます。`visionSidecar.model` が未設定または空の場合、 OpenAI 実行経路、ダッシュボード、管理 API は `gpt-5.4-mini` をフォールバックとして使います。起動時には 明示的に保存された旧 `gpt-5.4-mini` 値を引き続き `gpt-5.6-luna` にマイグレーションしますが、この マイグレーションは保存済みの値だけが対象で、モデルフィールドがない場合には適用されません。 @@ -92,8 +97,8 @@ OpenAI 実行経路、ダッシュボード、管理 API は `gpt-5.4-mini` を リモート `https` 画像はプロキシではなく OpenAI バックエンドが取得します。 - `noVisionModels` 比較は Ollama 式の `:size` 接尾辞を無視するため `gpt-oss` 項目 1 つで `gpt-oss:120b` も処理できます。 -- 画像説明が失敗すると短い処理エラー案内文をモデルに渡します。サイドカー計画自体を作れない場合は - テキスト専用バックエンドに元画像を送らず削除します。 +- 画像説明が失敗すると短い処理エラー案内文をモデルに渡します。(利用可能なサイドカー計画がない場合は + 説明を試みず、上記のとおり元画像を削除します。) - `maxDescriptionsPerTurn`(デフォルト 8)はメインモデル 1 ターンで新規実行する説明数を制限します。キャッシュ ヒットと同じターンの重複要求は限度を消費しません。成功した `data:` 画像説明はバックエンド、モデル、 detail、画像バイト、メッセージ文脈を基準にキャッシュし、OpenAI のキーには推論負荷も含まれます diff --git a/docs-site/src/content/docs/ko/guides/sidecars.md b/docs-site/src/content/docs/ko/guides/sidecars.md index 2aab84f438..55af37a150 100644 --- a/docs-site/src/content/docs/ko/guides/sidecars.md +++ b/docs-site/src/content/docs/ko/guides/sidecars.md @@ -71,8 +71,13 @@ stall은 전체 생성 timeout이 아닙니다. SSE가 시작되기 전 실패 ## 비전 사이드카 -라우팅 모델이 해당 프로바이더의 `noVisionModels`에 있고 요청에 이미지가 들어오면, opencodex는 -메인 호출 **전에** 각 이미지를 설명한 텍스트로 바꿉니다. `visionSidecar.model`이 없거나 빈 값이면 +라우팅 모델이 해당 프로바이더의 `noVisionModels`에 있거나 해당 모델이 `modelInputModalities`에서 +텍스트 전용으로 선언되어 있고 요청에 이미지가 들어오면, opencodex는 사용 가능한 비전 사이드카 계획이 있을 때에만 +메인 호출 **전에** 각 이미지를 설명한 텍스트로 바꿉니다. 사용 가능한 계획이 없으면 원본 이미지는 텍스트 전용 +백엔드로 전달되지 않고 제거됩니다. 모델 카탈로그는 사이드카로 처리되는 모든 모델에 image input을 알립니다. +콤보는 모든 멤버가 네이티브로 또는 사이드카를 통해 이미지를 수용하고 콤보의 `imageInput` 설정이 비활성화되지 않은 +경우에만 image input을 알립니다. 따라서 Codex 앱 같은 클라이언트는 사이드카가 실행되기 전에 첨부를 차단하지 않고 허용합니다. +`visionSidecar.model`이 없거나 빈 값이면 OpenAI 실행 경로, Dashboard, 관리 API는 `gpt-5.4-mini`를 폴백으로 사용합니다. 시작 시 명시적으로 저장된 기존 `gpt-5.4-mini` 값은 계속 `gpt-5.6-luna`로 마이그레이션되지만, 이 마이그레이션은 저장된 값에만 적용되고 모델 필드가 없는 경우에는 적용되지 않습니다. @@ -94,8 +99,8 @@ OpenAI 실행 경로, Dashboard, 관리 API는 `gpt-5.4-mini`를 폴백으로 원격 `https` 이미지는 프록시가 아니라 OpenAI 백엔드가 가져옵니다. - `noVisionModels` 비교는 Ollama식 `:size` 접미사를 무시하므로 `gpt-oss` 항목 하나로 `gpt-oss:120b`도 처리할 수 있습니다. -- 이미지 설명이 실패하면 짧은 처리 오류 안내문을 모델에 전달합니다. 사이드카 계획 자체를 만들 수 - 없으면 텍스트 전용 백엔드에 원본 이미지를 보내지 않고 제거합니다. +- 이미지 설명이 실패하면 짧은 처리 오류 안내문을 모델에 전달합니다. (사용 가능한 사이드카 계획이 없으면 + 설명을 시도하지 않고 위에서 설명한 대로 원본 이미지를 제거합니다.) - `maxDescriptionsPerTurn`(기본값 8)은 메인 모델 한 턴에서 새로 실행할 설명 수를 제한합니다. 캐시 적중과 같은 턴의 중복 요청은 한도를 쓰지 않습니다. 성공한 `data:` 이미지 설명은 백엔드, 모델, detail, 이미지 바이트, 메시지 문맥을 기준으로 캐시하며, OpenAI 키에는 추론 강도도 포함됩니다 diff --git a/docs-site/src/content/docs/ru/guides/sidecars.md b/docs-site/src/content/docs/ru/guides/sidecars.md index bc2a691a67..cdb6dc7087 100644 --- a/docs-site/src/content/docs/ru/guides/sidecars.md +++ b/docs-site/src/content/docs/ru/guides/sidecars.md @@ -80,9 +80,14 @@ SSE-событие `response.failed`. ## Vision-сайдкар -Когда маршрутизируемая модель указана в `noVisionModels` своего провайдера, а запрос содержит -изображение, opencodex описывает каждое изображение **до** основного вызова и заменяет его текстом. -Если `visionSidecar.model` отсутствует или пуст, путь выполнения OpenAI, дашборд и API управления +Когда маршрутизируемая модель указана в `noVisionModels` своего провайдера — либо для неё в +`modelInputModalities` явно указана только текстовая модальность — и запрос содержит изображение, +opencodex описывает каждое изображение **до** основного вызова и заменяет его текстом, если доступен +план vision-сайдкара. Без доступного плана исходное изображение удаляется, а не передаётся текстовому +бэкенду. Каталог моделей объявляет вход изображений для каждой модели, покрытой сайдкаром. Комбо +объявляют вход изображений только если каждый участник принимает изображения нативно или через +сайдкар и параметр комбо `imageInput` не отключён; поэтому такие клиенты, как приложение Codex, +разрешают вложения вместо их блокировки до запуска сайдкара. Если `visionSidecar.model` отсутствует или пуст, путь выполнения OpenAI, дашборд и API управления используют фолбэк `gpt-5.4-mini`. При запуске явно сохранённое устаревшее значение `gpt-5.4-mini` по-прежнему мигрирует на `gpt-5.6-luna`; миграция применяется только к сохранённому значению, а не к отсутствующему полю модели. @@ -106,8 +111,8 @@ SSE-событие `response.failed`. `data:` и `https:`; удалённые `https`-изображения загружает бэкенд OpenAI, а не прокси. - Сопоставление `noVisionModels` игнорирует суффикс `:size` в стиле Ollama, поэтому запись `gpt-oss` покрывает и `gpt-oss:120b`. -- Если описание не удалось, модель получает короткий маркер ошибки обработки. Если план сайдкара - недоступен, исходное изображение удаляется, а не пересылается текстовому бэкенду. +- Если описание не удалось, модель получает короткий маркер ошибки обработки. (Если доступного плана + сайдкара нет, описание не запускается, а исходное изображение удаляется, как описано выше.) - `maxDescriptionsPerTurn` (по умолчанию 8) ограничивает число новых описаний за один ход основной модели. Попадания в кэш и дубликаты в рамках того же хода лимит не расходуют. Успешные описания `data:`-изображений кэшируются по бэкенду, модели, детализации, байтам изображения и контексту diff --git a/docs-site/src/content/docs/tr/guides/sidecars.md b/docs-site/src/content/docs/tr/guides/sidecars.md index 5a46fbabc3..5205f0041f 100644 --- a/docs-site/src/content/docs/tr/guides/sidecars.md +++ b/docs-site/src/content/docs/tr/guides/sidecars.md @@ -103,9 +103,16 @@ durma, toplam bir üretim zaman aşımı değildir. SSE başlamadan önceki arı ## Vizyon sidecar'ı -Yönlendirilen model sağlayıcısının `noVisionModels` listesinde yer aldığında ve -bir istek görsel taşıdığında, opencodex ana çağrıdan **önce** her görseli -açıklar ve onu metinle değiştirir. `visionSidecar.model` olmadığında veya boş +Yönlendirilen model sağlayıcısının `noVisionModels` listesinde yer aldığında ya da +bu model için `modelInputModalities` ile salt metin olarak bildirildiğinde ve bir +istek görsel taşıdığında, opencodex kullanılabilir bir vision sidecar planı varsa +ana çağrıdan **önce** her görseli açıklar ve onu metinle değiştirir. Kullanılabilir +bir plan yoksa ham görsel salt metin arka ucuna iletilmek yerine kaldırılır. Model +kataloğu sidecar kapsamında olan her model için görsel girdisini bildirir. Kombolar, +her üye görselleri yerel olarak veya bir sidecar üzerinden kabul ettiğinde ve kombonun +`imageInput` ayarı devre dışı olmadığında görsel girdisini bildirir; böylece Codex +uygulaması gibi istemciler, sidecar çalışmadan önce ekleri engellemek yerine kabul eder. +`visionSidecar.model` olmadığında veya boş olduğunda, OpenAI yürütme yolu, Kontrol Paneli ve yönetim API'si `gpt-5.4-mini` geri dönüşünü kullanır. Başlangıç hala açıkça kalıcı hale getirilmiş eski bir `gpt-5.4-mini` değerini `gpt-5.6-luna`'ya geçirir; bu geçiş, bulunmayan bir @@ -135,9 +142,8 @@ model alanına değil, saklanan bir değere uygulanır. görselleri proxy tarafından değil, OpenAI arka ucu tarafından getirilir. - `noVisionModels` eşleştirmesi Ollama tarzı bir `:size` sonekini yok sayar, bu nedenle bir `gpt-oss` girdisi `gpt-oss:120b`'yi de kapsar. -- Açıklama başarısız olursa model kısa bir işleme hatası işaretçisi alır. - Kullanılabilir hiçbir sidecar planı yoksa ham görsel salt metin bir arka uca - iletilmek yerine kaldırılır. +- Açıklama başarısız olursa model kısa bir işleme hatası işaretçisi alır. (Kullanılabilir bir + sidecar planı yoksa açıklama denenmez; ham görsel yukarıda belirtildiği gibi kaldırılır.) - `maxDescriptionsPerTurn` (varsayılan 8), ana model turu başına yeni açıklamaları sınırlar. Önbellek isabetleri ve aynı turdaki kopyalar bunu tüketmez. Başarılı `data:` görsel açıklamaları arka uç, model, ayrıntı, görsel @@ -197,4 +203,3 @@ hedeflenen hesap ve iş yükü ile kapsamlı bir şekilde test edilmelidir. Her alan için [Yapılandırma referansı](/tr/reference/configuration/#sidecars) bölümüne bakın. - diff --git a/docs-site/src/content/docs/zh-cn/guides/sidecars.md b/docs-site/src/content/docs/zh-cn/guides/sidecars.md index ea758b48e8..f2dfec5c95 100644 --- a/docs-site/src/content/docs/zh-cn/guides/sidecars.md +++ b/docs-site/src/content/docs/zh-cn/guides/sidecars.md @@ -64,8 +64,11 @@ OAuth 账户时使用 `anthropic`,否则使用 `openai`。显式选择 `anthro ## Vision sidecar -当路由模型列在其 provider 的 `noVisionModels` 中,并且请求包含图像时,opencodex 会在主调用 -**之前**描述每张图像,并用文字替换图像。当 `visionSidecar.model` 缺失或为空时,OpenAI 执行路径、 +当路由模型列在其 provider 的 `noVisionModels` 中,或该模型在 `modelInputModalities` 中被声明为仅文本, +并且请求包含图像时,只要有可用的 vision sidecar plan,opencodex 就会在主调用**之前**描述每张图像并用文字替换图像。 +如果没有可用 plan,原始图像会被移除,而不会继续转发给纯文本后端。模型目录会为每个由 sidecar 覆盖的模型声明图像输入。 +只有当每个 combo 成员都能原生或通过 sidecar 接受图像、且 combo 的 `imageInput` 设置未禁用时,combo 才会声明图像输入; +这样 Codex 应用等客户端会允许附件,而不会在 sidecar 运行前阻止它们。当 `visionSidecar.model` 缺失或为空时,OpenAI 执行路径、 Dashboard 和管理 API 都使用 `gpt-5.4-mini` 作为回退。启动时仍会把明确保存的旧 `gpt-5.4-mini` 值迁移到 `gpt-5.6-luna`;该迁移只作用于已保存值,不适用于缺失的 model 字段。 @@ -83,8 +86,8 @@ Dashboard 和管理 API 都使用 `gpt-5.4-mini` 作为回退。启动时仍会 而不是代理。 - `noVisionModels` 匹配会忽略 Ollama 风格的 `:size` 后缀,因此一个 `gpt-oss` 条目也能覆盖 `gpt-oss:120b`。 -- 如果描述失败,模型会收到简短的处理错误提示。若根本无法建立 sidecar plan,原始图像会被 - 移除,而不会继续转发给纯文本后端。 +- 如果描述失败,模型会收到简短的处理错误提示。(如果没有可用的 sidecar plan,则不会尝试描述, + 原始图像会按上文所述被移除。) - `maxDescriptionsPerTurn`(默认 8)限制每个主模型 turn 的新增描述次数。缓存命中和同一 turn 的重复请求不会消耗配额。成功的 `data:` 图像描述会按后端、模型、detail、图像字节和消息上下文 缓存;OpenAI 的缓存键还会额外包含推理强度(Anthropic 键不含,因为该字段在那里被忽略)。 diff --git a/docs-site/src/content/docs/zh-tw/guides/sidecars.md b/docs-site/src/content/docs/zh-tw/guides/sidecars.md index 5d9c7d1de4..1ac1c12f0e 100644 --- a/docs-site/src/content/docs/zh-tw/guides/sidecars.md +++ b/docs-site/src/content/docs/zh-tw/guides/sidecars.md @@ -64,8 +64,11 @@ OAuth 帳號時使用 `anthropic`,否則使用 `openai`。明確選擇 `anthro ## Vision sidecar -當路由模型列在其 provider 的 `noVisionModels` 中,並且請求包含圖像時,opencodex 會在主呼叫 -**之前**描述每張圖像,並用文字替換圖像。Dashboard 和管理 API 目前顯示的預設值是 +當路由模型列在其 provider 的 `noVisionModels` 中,或該模型在 `modelInputModalities` 中被宣告為僅文字, +且請求包含圖像時,只要有可用的 vision sidecar plan,opencodex 就會在主呼叫**之前**描述每張圖像並用文字替換圖像。 +若沒有可用 plan,原始圖像會被移除,不會繼續轉送給純文字後端。模型目錄會為每個由 sidecar 處理的模型宣告圖像輸入。 +只有當每個 combo 成員都能原生或透過 sidecar 接受圖像,且 combo 的 `imageInput` 設定未停用時,combo 才會宣告圖像輸入; +如此 Codex 應用程式等用戶端會允許附件,而不會在 sidecar 執行前阻擋它們。Dashboard 和管理 API 目前顯示的預設值是 `gpt-5.6-luna`,啟動時也會把明確儲存的舊 `gpt-5.4-mini` 值遷移到 Luna。只有在 `visionSidecar.model` 欄位完全不存在時,vision 執行路徑才會使用程式碼中的 `gpt-5.4-mini` 回退值。 @@ -80,8 +83,8 @@ OAuth 帳號時使用 `anthropic`,否則使用 `openai`。明確選擇 `anthro 而不是代理。 - `noVisionModels` 匹配會忽略 Ollama 風格的 `:size` 字尾,因此一個 `gpt-oss` 條目也能覆蓋 `gpt-oss:120b`。 -- 如果描述失敗,模型會收到簡短的處理錯誤提示。若根本無法建立 sidecar plan,原始圖像會被 - 移除,而不會繼續轉發給純文字後端。 +- 如果描述失敗,模型會收到簡短的處理錯誤提示。(如果沒有可用的 sidecar plan,就不會嘗試描述, + 原始圖像會依上文所述被移除。) - `maxDescriptionsPerTurn`(預設 8)限制每個主模型 turn 的新增描述次數。快取命中和同一 turn 的重複請求不會消耗配額。成功的 `data:` 圖像描述會按後端、模型、detail、圖像位元組和訊息上下文 快取;內容可變的 `https:` 圖像不會快取。 diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index 882ea1e6d8..36c14395a8 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -30,6 +30,7 @@ import { import type { OcxConfig, OcxProviderConfig } from "../../types"; import { modelInList } from "../../types"; import { CODEX_REASONING_LEVELS, codexEffortRank, configuredReasoningEfforts, modelRecordValue, sanitizeCodexReasoningEfforts } from "../../reasoning-effort"; +import { isModelVisionSidecarConsumer } from "../../vision/eligibility"; import { getModelMetadata, getModelMetadataCaseInsensitive, listModelMetadata, resolveMetadataProvider } from "../../generated/model-metadata"; import { enrichProviderFromRegistry, shouldCaseFoldMetadataModelId } from "../../providers/derive"; import { @@ -670,15 +671,13 @@ export function applyProviderConfigHints(name: string, prov: OcxProviderConfig, const configuredMaxInput = configuredMaxInputTokens(prov, model.id); const configuredAutoCompact = configuredAutoCompactTokenLimit(prov, model.id); let inputModalities = configuredInputModalities(prov, model.id); - // Vision-sidecar coverage mirrors isModelTextOnly (src/vision/index.ts): `noVisionModels` OR - // a `modelInputModalities` declaration excluding "image" both mean the PROXY describes images - // for this model at request time. The catalog must still advertise image input — the Codex app + // The shared vision-sidecar consumer predicate keeps catalog advertisement and request-time + // planning aligned. The catalog must still advertise image input — the Codex app // gates attachments client-side on input_modalities, and a text-only entry would block images // before the sidecar ever runs ("This model does not support image inputs"). Discovery-derived // text-only rows stay untouched: the runtime predicate only reads these two config sources, so // it would not convert those. - const sidecarCovered = modelInList(prov.noVisionModels, model.id) - || (Array.isArray(inputModalities) && inputModalities.length > 0 && !inputModalities.includes("image")); + const sidecarCovered = isModelVisionSidecarConsumer(prov, model.id); if (sidecarCovered) { const base = inputModalities ?? model.inputModalities ?? ["text"]; inputModalities = base.includes("image") ? [...base] : [...base, "image"]; @@ -2146,15 +2145,8 @@ async function gatherRoutedModelsUncached( } : mergedWithHardBounds; const enrichedProvider = enrichedByName.get(cm.provider) ?? rawProvider; - // Same vision-sidecar coverage rule as applyProviderConfigHints (isModelTextOnly parity): - // noVisionModels OR a text-only modelInputModalities declaration means the sidecar converts - // images at request time, so the custom row must advertise image input. - const declaredModalities = enrichedProvider - ? modelRecordValue(enrichedProvider.modelInputModalities, mergedWithAutoCompact.id) - : undefined; - if (enrichedProvider - && (modelInList(enrichedProvider.noVisionModels, mergedWithAutoCompact.id) - || (Array.isArray(declaredModalities) && declaredModalities.length > 0 && !declaredModalities.includes("image")))) { + // Reuse the request-time consumer predicate so custom rows cannot drift from catalog hints. + if (enrichedProvider && isModelVisionSidecarConsumer(enrichedProvider, mergedWithAutoCompact.id)) { const current = mergedWithAutoCompact.inputModalities ?? ["text"]; if (!current.includes("image")) { return { ...mergedWithAutoCompact, inputModalities: [...current, "image"] }; diff --git a/src/vision/eligibility.ts b/src/vision/eligibility.ts index 06a4ecce02..09a83d44db 100644 --- a/src/vision/eligibility.ts +++ b/src/vision/eligibility.ts @@ -21,6 +21,7 @@ * explicit modalities. */ import { modelInList, type OcxConfig, type OcxProviderConfig } from "../types"; +import { modelRecordValue } from "../reasoning-effort"; import { getModelMetadataCaseInsensitive, resolveMetadataProvider } from "../generated/model-metadata"; import { nativeInputModalities } from "../codex/catalog/metadata"; import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models"; @@ -68,6 +69,22 @@ export interface VisionModelOption { type EnrichedProviderCache = Map; +/** + * Whether the proxy must describe images for this model before dispatching its main request. + * + * `noVisionModels` is an explicit override. A modality declaration is only evidence for this + * path when it describes a text model that excludes image input: an audio-only declaration is + * not a text-only model and must not be widened to image through the vision sidecar. + */ +export function isModelVisionSidecarConsumer( + provider: Pick, + modelId: string, +): boolean { + if (modelInList(provider.noVisionModels, modelId)) return true; + const modalities = modelRecordValue(provider.modelInputModalities, modelId); + return Array.isArray(modalities) && modalities.includes("text") && !modalities.includes("image"); +} + function advertisesImageInput(modalities: readonly string[] | undefined): boolean | undefined { if (!modalities || modalities.length === 0) return undefined; return modalities.includes("image"); @@ -105,7 +122,8 @@ function isVisionSidecarConsumerWithCache( modelId: string, cache: EnrichedProviderCache, ): boolean { - return modelInList(enrichedProviderForVision(config, providerName, cache)?.noVisionModels, modelId); + const provider = enrichedProviderForVision(config, providerName, cache); + return provider !== undefined && isModelVisionSidecarConsumer(provider, modelId); } /** diff --git a/src/vision/index.ts b/src/vision/index.ts index b945620ca6..3f85258624 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -1,12 +1,10 @@ import { createHash } from "node:crypto"; import type { OcxConfig, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTextContent } from "../types"; -import { modelInList } from "../types"; -import { modelRecordValue } from "../reasoning-effort"; import type { VisionReasoningEffort } from "../reasoning-effort"; import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe"; import { describeImageAnthropic } from "./anthropic-describe"; import { describeImageRouted } from "./routed-describe"; -import { modelAcceptsImageInput } from "./eligibility"; +import { isModelVisionSidecarConsumer as isModelTextOnly, modelAcceptsImageInput } from "./eligibility"; import { normalizeVisionReasoningForModel } from "./reasoning"; import type { CodexAuthContext } from "../codex/auth-context"; import { resolveSidecarAuth } from "../sidecar/auth"; @@ -22,24 +20,12 @@ import { export { describeImage } from "./describe"; -/** - * True when the model is explicitly known to be text-only — either listed in - * `noVisionModels` or declared with `modelInputModalities` that exclude "image". - * Returns false for unknown models (no evidence either way) so they fall through - * to native image passthrough, which is the safe default for an unclassified model. - */ -export function isModelTextOnly( - provider: OcxProviderConfig, - modelId: string, -): boolean { - if (modelInList(provider.noVisionModels, modelId)) return true; - const modalities = modelRecordValue(provider.modelInputModalities, modelId); - if (Array.isArray(modalities) && modalities.length > 0 && !modalities.includes("image")) return true; - return false; -} +/** Backward-compatible request-time name for the shared vision-sidecar consumer predicate. */ +export { isModelVisionSidecarConsumer as isModelTextOnly } from "./eligibility"; export { describeImageAnthropic, parseAnthropicVisionSSE } from "./anthropic-describe"; export { BASELINE_VISION_MODELS, + isModelVisionSidecarConsumer, isVisionEligibleModel, isVisionSidecarConsumer, modelAcceptsImageInput, diff --git a/tests/catalog-vision-sidecar-modalities.test.ts b/tests/catalog-vision-sidecar-modalities.test.ts index 253b8cc8e2..8e71876dff 100644 --- a/tests/catalog-vision-sidecar-modalities.test.ts +++ b/tests/catalog-vision-sidecar-modalities.test.ts @@ -72,6 +72,16 @@ describe("vision-sidecar catalog modalities", () => { expect(hinted.inputModalities).toEqual(["text", "image"]); }); + test("audio-only modelInputModalities do not advertise image", () => { + const prov: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://api.example/v1", + modelInputModalities: { "audio-model": ["audio"] }, + }; + const hinted = applyProviderConfigHints("audio-provider", prov, { id: "audio-model", provider: "audio-provider" }); + expect(hinted.inputModalities).toEqual(["audio"]); + }); + test("discovery-derived text-only rows are NOT advertised image (the runtime would not convert them)", () => { // Only the two config sources the runtime predicate reads (noVisionModels, // modelInputModalities) may widen the catalog; a listing that merely reports @@ -240,6 +250,36 @@ describe("vision-sidecar custom-model override (#349/#344)", () => { clearModelCache("text-sidecar-provider"); } }); + + test("a custom row whose modelId is declared audio-only does not advertise image", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { throw new Error("fetch should not be called"); }) as typeof fetch; + try { + const models = await gatherRoutedModels({ + port: 10100, + defaultProvider: "audio-sidecar-provider", + providers: { + "audio-sidecar-provider": { + baseUrl: "https://audio-sidecar.example/v1", + adapter: "openai-chat", + authMode: "key", + liveModels: false, + models: ["baseline-model"], + modelInputModalities: { "audio-model": ["audio"] }, + }, + }, + customModels: [ + { id: "cm-audio", provider: "audio-sidecar-provider", modelId: "audio-model", displayName: "Audio Model", addedAt: "2026-01-01T00:00:00.000Z" }, + ], + }); + const custom = models.find(m => m.provider === "audio-sidecar-provider" && m.id === "audio-model"); + expect(custom).toBeDefined(); + expect(custom?.inputModalities?.includes("image") ?? false).toBe(false); + } finally { + globalThis.fetch = originalFetch; + clearModelCache("audio-sidecar-provider"); + } + }); }); describe("vision-capable provider models feed combo modalities", () => { diff --git a/tests/vision-eligibility.test.ts b/tests/vision-eligibility.test.ts index 6469b6cb0a..e28d406978 100644 --- a/tests/vision-eligibility.test.ts +++ b/tests/vision-eligibility.test.ts @@ -4,6 +4,7 @@ import type { OcxConfig } from "../src/types"; import { BASELINE_VISION_MODELS, isVisionEligibleModel, + isVisionSidecarConsumer, modelAcceptsImageInput, visionBackendForCandidate, visionEligibleModelOptions, @@ -54,6 +55,24 @@ describe("vision eligibility core", () => { expect(modelAcceptsImageInput(config, candidate)).toBe(false); }); + test("2b. configured text-only rows are consumers, but audio-only rows are not", () => { + const config = configWithProviders({ + test: { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelInputModalities: { + "text-only": ["text"], + "audio-only": ["audio"], + "text-and-image": ["text", "image"], + }, + }, + }); + + expect(isVisionSidecarConsumer(config, "test", "text-only")).toBe(true); + expect(isVisionSidecarConsumer(config, "test", "audio-only")).toBe(false); + expect(isVisionSidecarConsumer(config, "test", "text-and-image")).toBe(false); + }); + test("3a. silent catalog rows fall back to generated metadata (live /api/models shape)", () => { // anthropic / claude-opus-4-6 carries ["text","image"] in the generated table, but live // /api/models rows omit inputModalities entirely. diff --git a/tests/vision-text-only-predicate.test.ts b/tests/vision-text-only-predicate.test.ts index 02577902b6..05574bf9bb 100644 --- a/tests/vision-text-only-predicate.test.ts +++ b/tests/vision-text-only-predicate.test.ts @@ -19,6 +19,10 @@ describe("isModelTextOnly (#1024)", () => { expect(isModelTextOnly(provider({ modelInputModalities: { "vision-model": ["text", "image"] } }), "vision-model")).toBe(false); }); + test("returns false for audio-only modelInputModalities", () => { + expect(isModelTextOnly(provider({ modelInputModalities: { "audio-model": ["audio"] } }), "audio-model")).toBe(false); + }); + test("returns false for unknown models (no evidence)", () => { expect(isModelTextOnly(provider(), "unknown-model")).toBe(false); }); From 1cbb8906984d1a7f5cf991b0450d2b11d3af4a70 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Tue, 1 Sep 2026 09:53:59 +0900 Subject: [PATCH 5/5] docs(vision): align fallback and coverage wording --- docs-site/src/content/docs/zh-tw/guides/sidecars.md | 2 +- src/codex/catalog/provider-fetch.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs-site/src/content/docs/zh-tw/guides/sidecars.md b/docs-site/src/content/docs/zh-tw/guides/sidecars.md index 1ac1c12f0e..60131afd19 100644 --- a/docs-site/src/content/docs/zh-tw/guides/sidecars.md +++ b/docs-site/src/content/docs/zh-tw/guides/sidecars.md @@ -70,7 +70,7 @@ OAuth 帳號時使用 `anthropic`,否則使用 `openai`。明確選擇 `anthro 只有當每個 combo 成員都能原生或透過 sidecar 接受圖像,且 combo 的 `imageInput` 設定未停用時,combo 才會宣告圖像輸入; 如此 Codex 應用程式等用戶端會允許附件,而不會在 sidecar 執行前阻擋它們。Dashboard 和管理 API 目前顯示的預設值是 `gpt-5.6-luna`,啟動時也會把明確儲存的舊 `gpt-5.4-mini` 值遷移到 Luna。只有在 -`visionSidecar.model` 欄位完全不存在時,vision 執行路徑才會使用程式碼中的 `gpt-5.4-mini` 回退值。 +`visionSidecar.model` 欄位不存在或為空字串時,vision 執行路徑才會使用程式碼中的 `gpt-5.4-mini` 回退值。 - 圖像可以來自 user、developer 和 tool-result message,也包括 Codex 的 `view_image` 結果。 - 每張圖像會以 `reasoning.effort: "low"` 傳送給設定的原生 vision 模型,描述結果會就地替換 diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index 36c14395a8..b116f4d8c9 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -2117,9 +2117,10 @@ async function gatherRoutedModelsUncached( ...(base.codexToolMode === undefined && replaced.codexToolMode !== undefined ? { codexToolMode: replaced.codexToolMode } : {}), ...(base.capabilities === undefined && replaced.capabilities !== undefined ? { capabilities: replaced.capabilities } : {}), } : base; - // Vision-sidecar coverage ONLY: if the custom model is in the enriched provider's - // noVisionModels, advertise image input so the Codex app lets images reach the sidecar - // (#349/#344). Deliberately NOT the full applyProviderConfigHints pass — custom rows are a + // Vision-sidecar coverage only: when the enriched provider's shared predicate matches + // noVisionModels or text-without-image modelInputModalities, advertise image input so the + // Codex app lets images reach the sidecar (#349/#344). Deliberately NOT the full + // applyProviderConfigHints pass — custom rows are a // user override, so their explicit contextWindow / inputModalities / reasoning fields must be // preserved verbatim (the hint pass would cap context and overwrite modalities from registry). const mergedContext = typeof merged.contextWindow === "number" && merged.contextWindow > 0