diff --git a/docs/providers/lm-studio-http.md b/docs/providers/lm-studio-http.md new file mode 100644 index 0000000000..4665d15820 --- /dev/null +++ b/docs/providers/lm-studio-http.md @@ -0,0 +1,35 @@ +# LM Studio HTTP compatibility + +Zoo Code discovers models without `@lmstudio/sdk`. It tries these endpoints in order: + +| Endpoint | Available metadata | +| --------------------------------------- | ------------------------------------------------------------------------------------------ | +| `GET /api/v1/models` (LM Studio 0.4.0+) | LLM catalog keys, vision capability, maximum context, loaded instances and runtime context | +| `GET /api/v0/models` (LM Studio 0.3.6+) | LLM/VLM IDs, vision inferred from `vlm`, maximum context | +| `GET /v1/models` | IDs with Zoo Code's existing default metadata | + +HTTP 404 or 405 starts the next discovery endpoint. HTTP 200 with a body of `{"error": ""}` also starts it. Authentication errors, network failures, timeouts, cancellation and other invalid bodies do not. + +All three discovery requests share a 15-second deadline. Explicit model loading has a separate 10-minute timeout. A failed refresh clears cached loaded-detail flags and instance aliases for that server; caller cancellation preserves them. +Older APIs do not expose the native model-load operation; chat/JIT remains responsible for loading on those servers. + +Sources: [native API changelog](https://lmstudio.ai/docs/developer/api-changelog), [legacy API reference](https://lmstudio.ai/docs/developer/rest/endpoints). + +## Saved selections and instances + +Saved `lmStudioModelId` values are preserved. Previous SDK discovery exposed downloaded file paths and loaded model keys, so either may be saved. A catalog miss uses default metadata and sends the original ID to chat. If chat rejects the model selection, Zoo Code asks the user to reselect a model without changing their saved setting. + +Native REST catalog keys and loaded instance IDs are distinct. A separate server-scoped runtime map retains exact instance IDs as aliases; the dropdown receives only catalog keys. These aliases include custom instance names and names created before another quantization was installed. For a selected catalog key, use the instance whose ID matches the key, or the first instance in server order. For an instance alias, use that instance's context length. There is no substring or display-name mapping from legacy paths to catalog keys. + +Native loading checks the selected model's `loaded_instances` first. If none exist, it posts the resolved catalog key to `/api/v1/models/load`. Concurrent callers acquire a shared operation for the normalized server URL and saved ID before discovery; discovery, loading, and cache refresh all run inside it. An unloaded model resolves only by its catalog key; instance aliases already refer to loaded models. The shared operation is cleared on success or failure. This coordinates callers within this extension process, not other clients of the server. + +## Verified migration limits + +A local probe on Linux with llmster **0.0.27+1**, llama.cpp CPU runtime **2.66.0**, and SmolLM2 135M established: + +- Two identical REST load calls created separate instances; loading is not idempotent. +- REST `key` matched the SDK's downloaded `modelKey`, but not its file `path`. +- Installing Q4_K_M alongside Q8_0 added quantization suffixes to catalog keys; already-loaded instances retained their previous keys/IDs. +- REST loading rejected the SDK file path with HTTP 404, but `/v1/chat/completions` accepted that same path with HTTP 200 while the model was loaded. + +The chat result supports preserving legacy selections; it does not establish path compatibility across all LM Studio releases, model formats, or JIT/unloaded configurations. Legacy HTTP fallbacks have focused response-contract tests, not live coverage against historical server binaries. diff --git a/package.json b/package.json index 04922487f6..315bb8ac1a 100644 --- a/package.json +++ b/package.json @@ -126,7 +126,6 @@ "@types/react-dom": "18.3.7", "csstype": "3.2.3", "zod": "4.6.7", - "@lmstudio/sdk>zod": "3.25.76", "katex": "0.19.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a949abf377..904c210d2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,7 +43,6 @@ overrides: '@types/react-dom': 18.3.7 csstype: 3.2.3 zod: 4.6.7 - '@lmstudio/sdk>zod': 3.25.76 katex: 0.19.1 importers: @@ -460,9 +459,6 @@ importers: '@google/genai': specifier: ^1.29.1 version: 1.52.0(@modelcontextprotocol/sdk@1.32.2(zod@4.6.7)) - '@lmstudio/sdk': - specifier: ^1.1.1 - version: 1.5.0 '@mistralai/mistralai': specifier: ^1.9.18 version: 1.15.1 @@ -2029,12 +2025,6 @@ packages: '@kwsites/promise-deferred@1.1.1': resolution: {integrity: sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==} - '@lmstudio/lms-isomorphic@0.4.6': - resolution: {integrity: sha512-v0LIjXKnDe3Ff3XZO5eQjlVxTjleUHXaom14MV7QU9bvwaoo3l5p71+xJ3mmSaqZq370CQ6pTKCn1Bb7Jf+VwQ==} - - '@lmstudio/sdk@1.5.0': - resolution: {integrity: sha512-fdY12x4hb14PEjYijh7YeCqT1ZDY5Ok6VR4l4+E/dI+F6NW8oB+P83Sxed5vqE4XgTzbgyPuSR2ZbMNxxF+6jA==} - '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -6143,9 +6133,6 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} - jsonschema@1.5.0: - resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} - jsonwebtoken@9.0.3: resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} engines: {node: '>=12', npm: '>=6'} @@ -9067,9 +9054,6 @@ packages: peerDependencies: zod: 4.6.7 - zod@3.25.76: - resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.6.7: resolution: {integrity: sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==} @@ -10404,24 +10388,6 @@ snapshots: '@kwsites/promise-deferred@1.1.1': {} - '@lmstudio/lms-isomorphic@0.4.6': - dependencies: - ws: 8.22.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@lmstudio/sdk@1.5.0': - dependencies: - '@lmstudio/lms-isomorphic': 0.4.6 - chalk: 4.1.2 - jsonschema: 1.5.0 - zod: 3.25.76 - zod-to-json-schema: 3.25.2(zod@3.25.76) - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.29.13 @@ -14849,8 +14815,6 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 - jsonschema@1.5.0: {} - jsonwebtoken@9.0.3: dependencies: jws: 4.0.1 @@ -18257,16 +18221,10 @@ snapshots: compress-commons: 4.1.2 readable-stream: 3.6.2 - zod-to-json-schema@3.25.2(zod@3.25.76): - dependencies: - zod: 3.25.76 - zod-to-json-schema@3.25.2(zod@4.6.7): dependencies: zod: 4.6.7 - zod@3.25.76: {} - zod@4.6.7: {} zustand@5.0.16(@types/react@18.3.31)(react@19.3.1): diff --git a/renovate.json b/renovate.json index fbec3053b0..94ddd72b2c 100644 --- a/renovate.json +++ b/renovate.json @@ -36,7 +36,6 @@ "/^@ai-sdk\\//", "/^@anthropic-ai\\//", "/^@aws-sdk\\//", - "/^@lmstudio\\//", "/^@mistralai\\//", "/^@openrouter\\//", "/^@qdrant\\//" diff --git a/src/api/providers/__tests__/lm-studio.spec.ts b/src/api/providers/__tests__/lm-studio.spec.ts index 6a91f59eaf..6903f7c356 100644 --- a/src/api/providers/__tests__/lm-studio.spec.ts +++ b/src/api/providers/__tests__/lm-studio.spec.ts @@ -1,6 +1,7 @@ // npx vitest run api/providers/__tests__/lm-studio.spec.ts import { LmStudioHandler } from "../lm-studio" +import * as lmStudioFetcher from "../fetchers/lmstudio" import { ApiHandlerOptions } from "../../../shared/api" // Mock the timeout config utility @@ -89,6 +90,75 @@ describe("LmStudioHandler abort wiring", () => { } }) + it("resolves saved instance metadata separately from the dropdown catalog", () => { + const aliasInfo = { contextWindow: 4096, supportsImages: false, supportsPromptCache: true } + const aliasSpy = vitest.spyOn(lmStudioFetcher, "getLMStudioModelAlias").mockReturnValue(aliasInfo) + try { + const handler = new LmStudioHandler({ ...options, lmStudioModelId: "custom-instance" }) + expect(handler.getModel()).toEqual({ id: "custom-instance", info: aliasInfo }) + expect(aliasSpy).toHaveBeenCalledWith("custom-instance", options.lmStudioBaseUrl) + } finally { + aliasSpy.mockRestore() + } + }) + + it("preserves a saved path ID when catalog metadata is unavailable", () => { + const savedId = "lmstudio-community/SmolLM2/SmolLM2-Q8_0.gguf" + const handler = new LmStudioHandler({ ...options, lmStudioModelId: savedId }) + expect(handler.getModel().id).toBe(savedId) + expect(handler.getModel().info).toBeDefined() + }) + + describe.each(["stream", "complete"] as const)("%s model selection errors", (mode) => { + it.each([ + { status: 404, message: "Model old/path.gguf not found", code: undefined, targeted: true }, + { status: 400, message: "No models found", code: undefined, targeted: true }, + { status: 404, message: "Unavailable", code: "model_not_found", targeted: true }, + { status: 401, message: "Unauthorized", code: undefined, targeted: false }, + { status: 400, message: "Context length exceeded", code: undefined, targeted: false }, + { status: 404, message: "Endpoint not found", code: undefined, targeted: false }, + ])( + "handles $status $message without rewriting the saved selection", + async ({ status, message, code, targeted }) => { + const handler = new LmStudioHandler(options) + vitest.spyOn(handler, "countTokens").mockResolvedValue(1) + lastCreate().mockRejectedValue(Object.assign(new Error(message), { status, error: { code } })) + const request = + mode === "stream" + ? collectStream(handler.createMessage("system", [])) + : handler.completePrompt("hello") + await expect(request).rejects.toThrow( + targeted ? "Reselect a model" : "Please check the LM Studio developer logs", + ) + expect(options.lmStudioModelId).toBe("llama2") + }, + ) + + it("tells the user to load a model when the server has none loaded", async () => { + const handler = new LmStudioHandler(options) + vitest.spyOn(handler, "countTokens").mockResolvedValue(1) + lastCreate().mockRejectedValue(Object.assign(new Error("No models loaded"), { status: 400 })) + const request = + mode === "stream" ? collectStream(handler.createMessage("system", [])) : handler.completePrompt("hello") + await expect(request).rejects.toThrow("Load a model in LM Studio") + expect(options.lmStudioModelId).toBe("llama2") + }) + + it("passes a saved path to successful chat requests unchanged", async () => { + const savedId = "lmstudio-community/SmolLM2/SmolLM2-Q8_0.gguf" + const handler = new LmStudioHandler({ ...options, lmStudioModelId: savedId }) + vitest.spyOn(handler, "countTokens").mockResolvedValue(1) + lastCreate().mockResolvedValue( + mode === "stream" + ? asyncStreamFrom([{ choices: [{ delta: { content: "hello" } }] }]) + : { choices: [{ message: { content: "hello" } }] }, + ) + if (mode === "stream") await collectStream(handler.createMessage("system", [])) + else expect(await handler.completePrompt("hello")).toBe("hello") + expect(lastCreate().mock.calls[0][0]).toMatchObject({ model: savedId }) + }) + }) + describe("createMessage", () => { it("passes a request-local AbortSignal to the SDK and bridges the external signal", async () => { const handler = new LmStudioHandler(options) diff --git a/src/api/providers/__tests__/lmstudio.spec.ts b/src/api/providers/__tests__/lmstudio.spec.ts index f2901cc7d4..e12d2ee1d7 100644 --- a/src/api/providers/__tests__/lmstudio.spec.ts +++ b/src/api/providers/__tests__/lmstudio.spec.ts @@ -95,6 +95,16 @@ describe("LmStudioHandler", () => { mockGetModelsFromCache.mockReturnValue(undefined) }) + it.each([undefined, "", "http://localhost:1234/"])("normalizes handler URL %s", async (lmStudioBaseUrl) => { + const normalized = new LmStudioHandler({ ...mockOptions, lmStudioBaseUrl }) + mockGetModels.mockResolvedValue({}) + await normalized.ensureModelFetched() + normalized.getModel() + const scope = { provider: providerIdentifiers.lmstudio, baseUrl: "http://localhost:1234" } + expect(mockGetModels).toHaveBeenCalledWith(scope, expect.anything()) + expect(mockGetModelsFromCache).toHaveBeenCalledWith(scope, normalized) + }) + describe("constructor", () => { it("should initialize with provided options", () => { expect(handler).toBeInstanceOf(LmStudioHandler) diff --git a/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json b/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json index 43f5505710..d454304ff2 100644 --- a/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json +++ b/src/api/providers/fetchers/__tests__/fixtures/lmstudio-model-details.json @@ -1,14 +1,61 @@ { - "mistralai/devstral-small-2505": { - "type": "llm", - "modelKey": "mistralai/devstral-small-2505", - "format": "safetensors", - "displayName": "Devstral Small 2505", - "path": "mistralai/devstral-small-2505", - "sizeBytes": 13277565112, - "architecture": "mistral", - "vision": false, - "trainedForToolUse": false, - "maxContextLength": 131072 - } + "models": [ + { + "type": "llm", + "publisher": "lmstudio-community", + "key": "smollm2-135m-instruct@q8_0", + "display_name": "SmolLM2 135M Instruct", + "architecture": "llama", + "quantization": { + "name": "Q8_0", + "bits_per_weight": 8 + }, + "size_bytes": 144811072, + "params_string": "135M", + "loaded_instances": [ + { + "id": "smollm2-135m-instruct", + "config": { + "context_length": 1024, + "eval_batch_size": 2048, + "physical_batch_size": 512, + "parallel": 4, + "flash_attention": false, + "lazy_mode": "auto", + "context_checkpoints": 32, + "reasoning_budget_message": "", + "speculative_draft_mtp": false, + "speculative_draft_simple": false, + "speculative_draft_model": "", + "speculative_draft_max_tokens": 3, + "speculative_draft_min_tokens": 0, + "speculative_draft_min_continue_probability": 0, + "offload_kv_cache_to_gpu": true + } + } + ], + "max_context_length": 8192, + "format": "gguf", + "capabilities": { + "vision": false, + "trained_for_tool_use": false + }, + "description": null + }, + { + "type": "embedding", + "publisher": "nomic-ai", + "key": "text-embedding-nomic-embed-text-v1.5", + "display_name": "Nomic Embed Text v1.5", + "quantization": { + "name": "Q4_K_M", + "bits_per_weight": 4 + }, + "size_bytes": 84106624, + "params_string": null, + "loaded_instances": [], + "max_context_length": 2048, + "format": "gguf" + } + ] } diff --git a/src/api/providers/fetchers/__tests__/lmstudio.test.ts b/src/api/providers/fetchers/__tests__/lmstudio.test.ts index dc86b757ea..810a66c2e1 100644 --- a/src/api/providers/fetchers/__tests__/lmstudio.test.ts +++ b/src/api/providers/fetchers/__tests__/lmstudio.test.ts @@ -1,568 +1,565 @@ -import axios from "axios" -import { LMStudioClient, LLMInstanceInfo, LLMInfo } from "@lmstudio/sdk" - -import { ModelInfo, lMStudioDefaultModelInfo, providerIdentifiers } from "@roo-code/types" - -import { forceFullModelDetailsLoad, getLMStudioModels, hasLoadedFullDetails, parseLMStudioModel } from "../lmstudio" +import axios, { AxiosError, type GenericAbortSignal } from "axios" + +import { lMStudioDefaultModelInfo, providerIdentifiers } from "@roo-code/types" + +import { + forceFullModelDetailsLoad, + getLMStudioModels, + getLMStudioModelAlias, + hasLoadedFullDetails, + LM_STUDIO_MODEL_LOAD_TIMEOUT_MS, +} from "../lmstudio" +import { MODEL_CATALOG_FETCH_TIMEOUT_MS } from "../modelCache" import type { ApiHandler } from "../../../index" -const mockFlushModels = vi.hoisted(() => vi.fn()) +import nativeFixture from "./fixtures/lmstudio-model-details.json" -vi.mock("../modelCache", () => ({ +const mockFlushModels = vi.hoisted(() => vi.fn()) +vi.mock("../modelCache", async (importOriginal) => ({ + ...(await importOriginal()), flushModels: mockFlushModels, - getModels: vi.fn(), })) -// Mock axios -vi.mock("axios") -const mockedAxios = axios as any - -// Mock @lmstudio/sdk -const mockGetModelInfo = vi.fn() -const mockListLoaded = vi.fn() -const mockListDownloadedModels = vi.fn() -const mockLoadModel = vi.fn() -vi.mock("@lmstudio/sdk", () => { - return { - LMStudioClient: vi.fn().mockImplementation(function () { - return { - llm: { - listLoaded: mockListLoaded, - model: mockLoadModel, - }, - system: { - listDownloadedModels: mockListDownloadedModels, - }, - } - }), - } -}) -const MockedLMStudioClientConstructor = LMStudioClient as any - -describe("LMStudio Fetcher", () => { - beforeEach(() => { - vi.clearAllMocks() - MockedLMStudioClientConstructor.mockClear() - mockListLoaded.mockClear() - mockGetModelInfo.mockClear() - mockListDownloadedModels.mockClear() - mockLoadModel.mockClear() - mockFlushModels.mockClear() - }) - - describe("forceFullModelDetailsLoad", () => { - it("loads the selected model before refreshing its server-scoped cache and recording full details", async () => { - const baseUrl = "https://securehost:4321" - const modelId = "mistralai/devstral-small-2505" - await getLMStudioModels("not a valid URL") - vi.clearAllMocks() - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockLoadModel.mockResolvedValueOnce({}) - mockFlushModels.mockResolvedValueOnce(undefined) - - expect(hasLoadedFullDetails(modelId)).toBe(false) - - const handler = { - fetchModels: vi.fn(), - getModelCacheScope: () => ({ urlScoped: true, keyScoped: false, authScoped: false }), - } as unknown as ApiHandler - - await forceFullModelDetailsLoad(baseUrl, modelId, handler) - - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: "wss://securehost:4321" }) - expect(mockLoadModel).toHaveBeenCalledWith(modelId) - expect(mockFlushModels).toHaveBeenCalledWith( - { provider: providerIdentifiers.lmstudio, baseUrl }, - handler, - true, - ) - expect(hasLoadedFullDetails(modelId)).toBe(true) - }) +const get = vi.spyOn(axios, "get") +const post = vi.spyOn(axios, "post") - it("warns with only the origin when the connection is refused for a URL with credentials and a query", async () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(function () {}) - try { - const error = new Error("Connection refused") - ;(error as NodeJS.ErrnoException).code = "ECONNREFUSED" - mockedAxios.get.mockRejectedValueOnce(error) - const handler = { fetchModels: vi.fn() } as unknown as ApiHandler - - await forceFullModelDetailsLoad( - "http://user:pass@localhost:1234/path?token=secret", - "any-model", - handler, - ) +// Only fetchModels and cache scope are used by the refresh double in these tests. +const handler = { + fetchModels: vi.fn(), + getModelCacheScope: () => ({ urlScoped: true, keyScoped: false, authScoped: false }), +} as unknown as ApiHandler - expect(warnSpy).toHaveBeenCalledWith("Error connecting to LMStudio at http://localhost:1234") - } finally { - warnSpy.mockRestore() - } - }) +const baseUrl = "http://localhost:1234" +const modelKey = "smollm2-135m-instruct@q8_0" +const nativeModel = (instances = [{ id: "smollm2-135m-instruct", config: { context_length: 1024 } }]) => ({ + ...nativeFixture.models[0], + loaded_instances: instances, +}) +const nativeResponse = (instances?: ReturnType["loaded_instances"]) => ({ + data: { models: [nativeModel(instances)] }, +}) +const httpError = (status: number) => + new AxiosError("HTTP error", undefined, undefined, undefined, { + status, + statusText: "Error", + data: {}, + headers: {}, + config: { headers: new axios.AxiosHeaders() }, }) - describe("parseLMStudioModel", () => { - it("should correctly parse raw LLMInfo to ModelInfo", () => { - const rawModel: LLMInstanceInfo = { - type: "llm", - modelKey: "mistralai/devstral-small-2505", - format: "safetensors", - displayName: "Devstral Small 2505", - path: "mistralai/devstral-small-2505", - sizeBytes: 13277565112, - architecture: "mistral", - identifier: "mistralai/devstral-small-2505", - instanceReference: "RAP5qbeHVjJgBiGFQ6STCuTJ", - vision: false, - trainedForToolUse: false, - maxContextLength: 131072, - contextLength: 7161, - } +beforeEach(async () => { + get.mockReset() + post.mockReset() + mockFlushModels.mockReset() + get.mockResolvedValue({ data: { models: [] } }) + await getLMStudioModels(baseUrl) + get.mockReset() + vi.spyOn(console, "error").mockImplementation(() => {}) + vi.spyOn(console, "warn").mockImplementation(() => {}) +}) - const expectedModelInfo: ModelInfo = { - ...lMStudioDefaultModelInfo, - description: `${rawModel.displayName} - ${rawModel.path}`, - contextWindow: rawModel.contextLength, - supportsPromptCache: true, - supportsImages: rawModel.vision, - maxTokens: rawModel.contextLength, - inputPrice: 0, - outputPrice: 0, - cacheWritesPrice: 0, - cacheReadsPrice: 0, - } +describe("LM Studio HTTP discovery", () => { + it.each(["Unexpected endpoint or method", "Endpoint not found", "Unrecognized request", ""])( + "falls back for an explicit unsupported-route body: %s", + async (error) => { + get.mockResolvedValueOnce({ data: { error } }) + .mockResolvedValueOnce({ data: { error } }) + .mockResolvedValueOnce({ data: { data: [{ id: "legacy" }] } }) + expect((await getLMStudioModels(baseUrl)).legacy.supportsImages).toBe(false) + expect(get).toHaveBeenCalledTimes(3) + }, + ) + + it.each([undefined, 42, { message: "unsupported" }])( + "does not fall back for a malformed non-string error: %s", + async (error) => { + get.mockResolvedValueOnce({ data: { error } }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledOnce() + }, + ) + + it.each([{ error: 42 }, { unexpected: true }])("rejects an invalid v0 body %j", async (data) => { + get.mockRejectedValueOnce(httpError(404)).mockResolvedValueOnce({ data }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledTimes(2) + }) - const result = parseLMStudioModel(rawModel) - expect(result).toEqual(expectedModelInfo) - }) + it("reports an OpenAI error body after HTTP fallback failures", async () => { + get.mockRejectedValueOnce(httpError(404)) + .mockRejectedValueOnce(httpError(405)) + .mockResolvedValueOnce({ data: { error: "OpenAI endpoint unavailable" } }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(console.error).toHaveBeenCalledWith(expect.any(String), "OpenAI endpoint unavailable") }) - describe("getLMStudioModels", () => { - const baseUrl = "http://localhost:1234" - const lmsUrl = "ws://localhost:1234" - - const mockRawModel: LLMInstanceInfo = { - architecture: "test-arch", - identifier: "mistralai/devstral-small-2505", - instanceReference: "RAP5qbeHVjJgBiGFQ6STCuTJ", - modelKey: "test-model-key-1", - path: "/path/to/test-model-1", - type: "llm", - displayName: "Test Model One", - maxContextLength: 2048, - contextLength: 7161, - paramsString: "1B params, 2k context", - vision: true, - format: "gguf", - sizeBytes: 1000000000, - trainedForToolUse: false, // Added + it("reports discovery deadline expiration as a timeout", async () => { + const deadline = new AbortController() + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal) + try { + get.mockImplementationOnce(async () => { + deadline.abort(new DOMException("deadline", "TimeoutError")) + throw new AxiosError("cancelled", "ERR_CANCELED") + }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(console.error).toHaveBeenCalledWith(expect.any(String), "LM Studio model discovery timed out") + } finally { + timeout.mockRestore() } + }) - it("should fetch downloaded models using system.listDownloadedModels", async () => { - const mockLLMInfo: LLMInfo = { - type: "llm" as const, - modelKey: "mistralai/devstral-small-2505", - format: "safetensors", - displayName: "Devstral Small 2505", - path: "mistralai/devstral-small-2505", - sizeBytes: 13277565112, - architecture: "mistral", - vision: false, - trainedForToolUse: false, - maxContextLength: 131072, - } - - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce([mockLLMInfo]) + it("retains the first server error when every endpoint fails", async () => { + get.mockResolvedValueOnce({ data: { error: "Invalid API key" } }) + .mockResolvedValueOnce({ data: { error: "second error" } }) + .mockResolvedValueOnce({ data: { error: "last error" } }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledTimes(3) + expect(console.error).toHaveBeenCalledWith(expect.any(String), "Invalid API key") + }) - const result = await getLMStudioModels(baseUrl) + it("preserves a healthy refresh when an older flight is cancelled", async () => { + const controller = new AbortController() + let rejectFlight: (error: Error) => void = () => { + throw new Error("flight not started") + } + get.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectFlight = reject + }), + ) + const cancelled = getLMStudioModels(baseUrl, { signal: controller.signal }) + const assertion = expect(cancelled).rejects.toMatchObject({ name: "AbortError" }) + get.mockResolvedValueOnce(nativeResponse()) + await getLMStudioModels(baseUrl) + controller.abort() + rejectFlight(new Error("cancelled")) + await assertion + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(true) + expect(getLMStudioModelAlias("smollm2-135m-instruct", baseUrl)).toBeDefined() + }) - expect(mockedAxios.get).toHaveBeenCalledTimes(1) - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl }) - expect(mockListDownloadedModels).toHaveBeenCalledTimes(1) - expect(mockListDownloadedModels).toHaveBeenCalledWith("llm") - expect(mockListLoaded).toHaveBeenCalled() // we now call it to get context data + it("prefers a valid native catalog over an accompanying error string", async () => { + get.mockResolvedValueOnce({ data: { ...nativeResponse().data, error: "ignored" } }) + expect(Object.keys(await getLMStudioModels(baseUrl))).toEqual([modelKey]) + expect(get).toHaveBeenCalledOnce() + }) - const expectedParsedModel = parseLMStudioModel(mockLLMInfo) - expect(result).toEqual({ [mockLLMInfo.path]: expectedParsedModel }) - }) + it("expires loaded details and aliases after a failed refresh", async () => { + get.mockResolvedValueOnce(nativeResponse()).mockRejectedValueOnce(new Error("server offline")) + await getLMStudioModels(baseUrl) + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(true) + expect(getLMStudioModelAlias("smollm2-135m-instruct", baseUrl)).toBeDefined() + await getLMStudioModels(baseUrl) + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(false) + expect(getLMStudioModelAlias("smollm2-135m-instruct", baseUrl)).toBeUndefined() + }) - it("should fall back to listLoaded when listDownloadedModels fails", async () => { - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockRejectedValueOnce(new Error("Method not available")) - mockListLoaded.mockResolvedValueOnce([{ getModelInfo: mockGetModelInfo }]) - mockGetModelInfo.mockResolvedValueOnce(mockRawModel) + it.each(["__proto__", "constructor"])("preserves catalog keys and aliases named %s", async (id) => { + get.mockResolvedValueOnce(nativeResponse([{ id, config: { context_length: 2048 } }])) + await getLMStudioModels(baseUrl) + expect(getLMStudioModelAlias(id, baseUrl)?.contextWindow).toBe(2048) + get.mockResolvedValueOnce({ data: { models: [{ ...nativeModel([]), key: id }] } }) + const models = await getLMStudioModels(baseUrl) + expect(Object.keys(models)).toEqual([id]) + expect(models[id].supportsImages).toBe(false) + }) - const result = await getLMStudioModels(baseUrl) + it("returns only catalog keys while retaining saved instance aliases separately", async () => { + get.mockResolvedValueOnce({ data: nativeFixture }) + const models = await getLMStudioModels(baseUrl) + expect(get).toHaveBeenCalledExactlyOnceWith(`${baseUrl}/api/v1/models`, { signal: expect.any(AbortSignal) }) + expect(models[modelKey]).toMatchObject({ contextWindow: 1024, maxTokens: 1024, supportsImages: false }) + expect(Object.keys(models)).toEqual([modelKey]) + expect(getLMStudioModelAlias("smollm2-135m-instruct", baseUrl)).toEqual(models[modelKey]) + expect(models["text-embedding-nomic-embed-text-v1.5"]).toBeUndefined() + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(true) + }) - expect(mockedAxios.get).toHaveBeenCalledTimes(1) - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl }) - expect(mockListDownloadedModels).toHaveBeenCalledTimes(1) - expect(mockListLoaded).toHaveBeenCalledTimes(1) + it("prefers an instance matching the catalog key, otherwise server order, and gives aliases their own context", async () => { + get.mockResolvedValueOnce( + nativeResponse([ + { id: "custom", config: { context_length: 2048 } }, + { id: modelKey, config: { context_length: 4096 } }, + ]), + ) + const models = await getLMStudioModels(baseUrl) + expect(models[modelKey].contextWindow).toBe(4096) + expect(models.custom).toBeUndefined() + expect(getLMStudioModelAlias("custom", baseUrl)?.contextWindow).toBe(2048) + }) - const expectedParsedModel = parseLMStudioModel(mockRawModel) - expect(result).toEqual({ [mockRawModel.modelKey]: expectedParsedModel }) + it.each([8192, null])("uses maximum/default context %s for an unloaded model", async (maxContext) => { + get.mockResolvedValueOnce({ + data: { models: [{ ...nativeModel([]), max_context_length: maxContext, capabilities: { vision: true } }] }, }) + const models = await getLMStudioModels(baseUrl) + expect(models[modelKey].contextWindow).toBe(maxContext ?? lMStudioDefaultModelInfo.contextWindow) + expect(models[modelKey].supportsImages).toBe(true) + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(false) + }) - it("should deduplicate models when both downloaded and loaded", async () => { - const mockDownloadedModel: LLMInfo = { - type: "llm" as const, - modelKey: "mistralai/devstral-small-2505", - format: "safetensors", - displayName: "Devstral Small 2505", - path: "mistralai/devstral-small-2505", - sizeBytes: 13277565112, - architecture: "mistral", - vision: false, - trainedForToolUse: false, - maxContextLength: 131072, - } - - const mockLoadedModel: LLMInstanceInfo = { - type: "llm", - modelKey: "devstral-small-2505", // Different key but should match case-insensitively - format: "safetensors", - displayName: "Devstral Small 2505", - path: "mistralai/devstral-small-2505", - sizeBytes: 13277565112, - architecture: "mistral", - identifier: "mistralai/devstral-small-2505", - instanceReference: "RAP5qbeHVjJgBiGFQ6STCuTJ", - vision: false, - trainedForToolUse: false, - maxContextLength: 131072, - contextLength: 7161, // Runtime context info - } - - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce([mockDownloadedModel]) - mockListLoaded.mockResolvedValueOnce([{ getModelInfo: vi.fn().mockResolvedValueOnce(mockLoadedModel) }]) - - const result = await getLMStudioModels(baseUrl) - - // Should only have one model, with the loaded model replacing the downloaded one - expect(Object.keys(result)).toHaveLength(1) - - // The loaded model's key should be used, with loaded model's data - const expectedParsedModel = parseLMStudioModel(mockLoadedModel) - expect(result[mockLoadedModel.modelKey]).toEqual(expectedParsedModel) - - // The downloaded model should have been removed - expect(result[mockDownloadedModel.path]).toBeUndefined() + it.each([404, 405])("falls back to v0 only on unsupported native endpoint %s", async (status) => { + get.mockRejectedValueOnce(httpError(status)).mockResolvedValueOnce({ + data: { + data: [ + { id: "vision", type: "vlm", max_context_length: 8192, state: "loaded" }, + { id: "text", type: "llm", max_context_length: 4096 }, + { id: "embedding", type: "embeddings", max_context_length: 2048 }, + ], + }, }) + const models = await getLMStudioModels(baseUrl) + expect(Object.keys(models)).toEqual(["vision", "text"]) + expect(models.vision).toMatchObject({ supportsImages: true, contextWindow: 8192 }) + expect(models.text).toMatchObject({ supportsImages: false, contextWindow: 4096 }) + expect(hasLoadedFullDetails("vision", baseUrl)).toBe(false) + expect(get.mock.calls.map(([url]) => url)).toEqual([`${baseUrl}/api/v1/models`, `${baseUrl}/api/v0/models`]) + }) - it("should handle deduplication with path-based matching", async () => { - const mockDownloadedModel: LLMInfo = { - type: "llm" as const, - modelKey: "Meta/Llama-3.1/8B-Instruct", - format: "gguf", - displayName: "Llama 3.1 8B Instruct", - path: "Meta/Llama-3.1/8B-Instruct", - sizeBytes: 8000000000, - architecture: "llama", - vision: false, - trainedForToolUse: false, - maxContextLength: 8192, - } - - const mockLoadedModel: LLMInstanceInfo = { - type: "llm", - modelKey: "Llama-3.1", // Should match the path segment - format: "gguf", - displayName: "Llama 3.1", - path: "Meta/Llama-3.1/8B-Instruct", - sizeBytes: 8000000000, - architecture: "llama", - identifier: "Meta/Llama-3.1/8B-Instruct", - instanceReference: "ABC123", - vision: false, - trainedForToolUse: false, - maxContextLength: 8192, - contextLength: 4096, - } - - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce([mockDownloadedModel]) - mockListLoaded.mockResolvedValueOnce([{ getModelInfo: vi.fn().mockResolvedValueOnce(mockLoadedModel) }]) - - const result = await getLMStudioModels(baseUrl) - - expect(Object.keys(result)).toHaveLength(1) - expect(result[mockLoadedModel.modelKey]).toBeDefined() - expect(result[mockDownloadedModel.path]).toBeUndefined() + it("skips a malformed native entry and keeps the valid models", async () => { + get.mockResolvedValueOnce({ + data: { models: [{ type: "llm", key: "broken", loaded_instances: "oops" }, ...nativeFixture.models] }, }) + const models = await getLMStudioModels(baseUrl) + expect(Object.keys(models)).toEqual([modelKey]) + expect(get).toHaveBeenCalledOnce() + }) - it("should not deduplicate models with similar but distinct names", async () => { - const mockDownloadedModels: LLMInfo[] = [ - { - type: "llm" as const, - modelKey: "mistral-7b", - format: "gguf", - displayName: "Mistral 7B", - path: "mistralai/mistral-7b-instruct", - sizeBytes: 7000000000, - architecture: "mistral", - vision: false, - trainedForToolUse: false, - maxContextLength: 4096, - }, - { - type: "llm" as const, - modelKey: "codellama", - format: "gguf", - displayName: "Code Llama", - path: "meta/codellama/7b", - sizeBytes: 7000000000, - architecture: "llama", - vision: false, - trainedForToolUse: false, - maxContextLength: 4096, - }, - ] - - const mockLoadedModel: LLMInstanceInfo = { - type: "llm", - modelKey: "llama", // Should not match "codellama" or "mistral-7b" - format: "gguf", - displayName: "Llama", - path: "meta/llama/7b", - sizeBytes: 7000000000, - architecture: "llama", - identifier: "meta/llama/7b", - instanceReference: "XYZ789", - vision: false, - trainedForToolUse: false, - maxContextLength: 4096, - contextLength: 2048, - } - - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce(mockDownloadedModels) - mockListLoaded.mockResolvedValueOnce([{ getModelInfo: vi.fn().mockResolvedValueOnce(mockLoadedModel) }]) - - const result = await getLMStudioModels(baseUrl) + it("returns no models without falling back when the native body has no model list", async () => { + get.mockResolvedValueOnce({ data: { unexpected: true } }) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledOnce() + }) - // Should have 3 models: mistral-7b (not deduped), codellama (not deduped), and llama (loaded) - expect(Object.keys(result)).toHaveLength(3) - expect(result["mistralai/mistral-7b-instruct"]).toBeDefined() // Should NOT be removed - expect(result["meta/codellama/7b"]).toBeDefined() // Should NOT be removed (codellama != llama) - expect(result[mockLoadedModel.modelKey]).toBeDefined() + it("falls back to OpenAI IDs when both native APIs are unsupported", async () => { + get.mockRejectedValueOnce(httpError(404)) + .mockRejectedValueOnce(httpError(405)) + .mockResolvedValueOnce({ data: { data: [{ id: "legacy" }] } }) + expect(await getLMStudioModels(baseUrl)).toEqual({ + legacy: { ...lMStudioDefaultModelInfo, description: "legacy", supportsImages: false }, }) + expect(get.mock.calls.map(([url]) => url)).toEqual([ + `${baseUrl}/api/v1/models`, + `${baseUrl}/api/v0/models`, + `${baseUrl}/v1/models`, + ]) + }) - it("should handle multiple loaded models with various duplicate scenarios", async () => { - const mockDownloadedModels: LLMInfo[] = [ - { - type: "llm" as const, - modelKey: "mistral-7b", - format: "gguf", - displayName: "Mistral 7B", - path: "mistralai/mistral-7b/instruct", - sizeBytes: 7000000000, - architecture: "mistral", - vision: false, - trainedForToolUse: false, - maxContextLength: 8192, - }, - { - type: "llm" as const, - modelKey: "llama-3.1", - format: "gguf", - displayName: "Llama 3.1", - path: "meta/llama-3.1/8b", - sizeBytes: 8000000000, - architecture: "llama", - vision: false, - trainedForToolUse: false, - maxContextLength: 8192, - }, - ] - - const mockLoadedModels: LLMInstanceInfo[] = [ - { - type: "llm", - modelKey: "mistral-7b", // Exact match with path segment - format: "gguf", - displayName: "Mistral 7B", - path: "mistralai/mistral-7b/instruct", - sizeBytes: 7000000000, - architecture: "mistral", - identifier: "mistralai/mistral-7b/instruct", - instanceReference: "REF1", - vision: false, - trainedForToolUse: false, - maxContextLength: 8192, - contextLength: 4096, - }, - { - type: "llm", - modelKey: "gpt-4", // No match, new model - format: "gguf", - displayName: "GPT-4", - path: "openai/gpt-4", - sizeBytes: 10000000000, - architecture: "gpt", - identifier: "openai/gpt-4", - instanceReference: "REF2", - vision: true, - trainedForToolUse: true, - maxContextLength: 32768, - contextLength: 16384, - }, - ] - - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce(mockDownloadedModels) - mockListLoaded.mockResolvedValueOnce( - mockLoadedModels.map((model) => ({ getModelInfo: vi.fn().mockResolvedValueOnce(model) })), + it.each([401, 403, 429, 500, "timeout", "malformed"])("does not fall back for %s", async (failure) => { + if (failure === "malformed") get.mockResolvedValueOnce({ data: { models: [{}] } }) + else + get.mockRejectedValueOnce( + typeof failure === "number" ? httpError(failure) : new AxiosError("timeout", "ECONNABORTED"), ) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledTimes(1) + }) - const result = await getLMStudioModels(baseUrl) - - // Should have 3 models: llama-3.1 (downloaded), mistral-7b (loaded, replaced), gpt-4 (loaded, new) - expect(Object.keys(result)).toHaveLength(3) - expect(result["meta/llama-3.1/8b"]).toBeDefined() // Downloaded, not replaced - expect(result["mistralai/mistral-7b/instruct"]).toBeUndefined() // Downloaded, replaced - expect(result["mistral-7b"]).toBeDefined() // Loaded, replaced downloaded - expect(result["gpt-4"]).toBeDefined() // Loaded, new - }) + it.each([401, 403, 500])("does not fall through a v0 failure %s", async (status) => { + get.mockRejectedValueOnce(httpError(404)).mockRejectedValueOnce(httpError(status)) + expect(await getLMStudioModels(baseUrl)).toEqual({}) + expect(get).toHaveBeenCalledTimes(2) + }) - it("should use default baseUrl if an empty string is provided", async () => { - const defaultBaseUrl = "http://localhost:1234" - const defaultLmsUrl = "ws://localhost:1234" - mockedAxios.get.mockResolvedValueOnce({ data: {} }) - mockListLoaded.mockResolvedValueOnce([]) + it("keeps loaded detail flags scoped to the server and clears them after unloading", async () => { + get.mockResolvedValueOnce(nativeResponse()) + .mockResolvedValueOnce(nativeResponse([])) + .mockResolvedValueOnce(nativeResponse([])) + await getLMStudioModels(baseUrl) + await getLMStudioModels("http://other:1234") + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(true) + expect(hasLoadedFullDetails(modelKey, "http://other:1234")).toBe(false) + await getLMStudioModels(baseUrl) + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(false) + }) - await getLMStudioModels("") + it("keeps duplicate and custom aliases out of the dropdown and expires them on refresh", async () => { + const instances = [ + { id: "old-unsuffixed-id", config: { context_length: 1024 } }, + { id: "old-unsuffixed-id:2", config: { context_length: 2048 } }, + { id: "custom", config: { context_length: 4096 } }, + ] + get.mockResolvedValueOnce(nativeResponse(instances)) + .mockResolvedValueOnce(nativeResponse([])) + .mockResolvedValueOnce(nativeResponse([])) + expect(Object.keys(await getLMStudioModels(baseUrl))).toEqual([modelKey]) + for (const instance of instances) { + expect(getLMStudioModelAlias(instance.id, baseUrl)?.contextWindow).toBe(instance.config.context_length) + expect(getLMStudioModelAlias(instance.id, "http://other:1234")).toBeUndefined() + } + await getLMStudioModels("http://other:1234") + expect(getLMStudioModelAlias("custom", baseUrl)).toBeDefined() + await getLMStudioModels(baseUrl) + expect(getLMStudioModelAlias("custom", baseUrl)).toBeUndefined() + }) - expect(mockedAxios.get).toHaveBeenCalledWith(`${defaultBaseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: defaultLmsUrl }) - }) + it.each(["", `${baseUrl}/`])("normalizes the base URL %s", async (url) => { + get.mockResolvedValueOnce(nativeResponse()) + await getLMStudioModels(url) + expect(get).toHaveBeenCalledWith(`${baseUrl}/api/v1/models`, { signal: expect.any(AbortSignal) }) + }) - it("should transform https baseUrl to wss for LMStudioClient", async () => { - const httpsBaseUrl = "https://securehost:4321" - const wssLmsUrl = "wss://securehost:4321" - mockedAxios.get.mockResolvedValueOnce({ data: {} }) - mockListLoaded.mockResolvedValueOnce([]) + it("does not request an invalid URL", async () => { + expect(await getLMStudioModels("not a URL")).toEqual({}) + expect(get).not.toHaveBeenCalled() + }) - await getLMStudioModels(httpsBaseUrl) + it("passes the signal through every fallback request", async () => { + const controller = new AbortController() + const { signal } = controller + get.mockRejectedValueOnce(httpError(404)) + .mockRejectedValueOnce(httpError(405)) + .mockResolvedValueOnce({ data: { data: [] } }) + await getLMStudioModels(baseUrl, { signal }) + const mergedSignal = get.mock.calls[0][1]?.signal + expect(mergedSignal).toBeInstanceOf(AbortSignal) + for (const [, config] of get.mock.calls) expect(config?.signal).toBe(mergedSignal) + controller.abort() + expect(mergedSignal?.aborted).toBe(true) + }) - expect(mockedAxios.get).toHaveBeenCalledWith(`${httpsBaseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: wssLmsUrl }) + it("rejects on abort without falling back", async () => { + const controller = new AbortController() + get.mockImplementationOnce(async () => { + controller.abort() + throw httpError(404) }) - - it("should return an empty object if lmsUrl is unparsable", async () => { - const unparsableBaseUrl = "http://localhost:invalid:port" // Leads to ws://localhost:invalid:port - - const result = await getLMStudioModels(unparsableBaseUrl) - - expect(result).toEqual({}) - expect(mockedAxios.get).not.toHaveBeenCalled() - expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() + await expect(getLMStudioModels(baseUrl, { signal: controller.signal })).rejects.toMatchObject({ + name: "AbortError", }) + expect(get).toHaveBeenCalledTimes(1) + }) - it("should return an empty object and log error if axios.get fails with a generic error", async () => { - const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(function () {}) - const networkError = new Error("Network connection failed") - mockedAxios.get.mockRejectedValueOnce(networkError) - - const result = await getLMStudioModels(baseUrl) + it("logs only the server origin on connection refusal", async () => { + get.mockRejectedValueOnce(new AxiosError("refused", "ECONNREFUSED")) + await getLMStudioModels("http://user:pass@localhost:1234/path?token=secret") + expect(console.warn).toHaveBeenCalledWith("Error connecting to LMStudio at http://localhost:1234") + }) +}) - expect(mockedAxios.get).toHaveBeenCalledTimes(1) - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() - expect(mockListLoaded).not.toHaveBeenCalled() - expect(consoleErrorSpy).toHaveBeenCalledWith( - `Error fetching LMStudio models: ${JSON.stringify(networkError, Object.getOwnPropertyNames(networkError), 2)}`, - ) - expect(result).toEqual({}) - consoleErrorSpy.mockRestore() +describe("LM Studio explicit loading", () => { + it("allows a successful model load to run beyond the catalog deadline", async () => { + vi.useFakeTimers() + const timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => { + const controller = new AbortController() + setTimeout(() => controller.abort(), ms) + return controller.signal }) + try { + get.mockResolvedValueOnce(nativeResponse([])) + let finishLoad: () => void = () => { + throw new Error("load not started") + } + let loadSignal: GenericAbortSignal | undefined + post.mockImplementationOnce( + (_url, _body, config) => + new Promise((resolve, reject) => { + loadSignal = config?.signal + loadSignal?.addEventListener?.("abort", () => reject(new Error("load timed out"))) + finishLoad = () => resolve({ data: {} }) + }), + ) + const pending = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await vi.advanceTimersByTimeAsync(30_000) + expect(post).toHaveBeenCalledOnce() + expect(loadSignal?.aborted).toBe(false) + expect(mockFlushModels).not.toHaveBeenCalled() + finishLoad() + await pending + expect(mockFlushModels).toHaveBeenCalledOnce() + } finally { + timeout.mockRestore() + vi.useRealTimers() + } + }) - it("should return an empty object and log info if axios.get fails with ECONNREFUSED", async () => { - const consoleInfoSpy = vi.spyOn(console, "warn").mockImplementation(function () {}) - const econnrefusedError = new Error("Connection refused") - ;(econnrefusedError as any).code = "ECONNREFUSED" - mockedAxios.get.mockRejectedValueOnce(econnrefusedError) - - const result = await getLMStudioModels(baseUrl) - - expect(mockedAxios.get).toHaveBeenCalledTimes(1) - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { signal: undefined }) - expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() - expect(mockListLoaded).not.toHaveBeenCalled() - expect(consoleInfoSpy).toHaveBeenCalledWith(`Error connecting to LMStudio at ${baseUrl}`) - expect(result).toEqual({}) - consoleInfoSpy.mockRestore() - }) + it.each(["discovery", "load"])( + "bounds a hung %s request and releases the pending slot for retry", + async (phase) => { + const deadline = new AbortController() + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal) + try { + get.mockResolvedValue(nativeResponse([])) + post.mockResolvedValue({ data: {} }) + let requestSignal: AbortSignal | undefined + const hang = (_url: string, config?: { signal?: GenericAbortSignal }) => + new Promise((_resolve, reject) => { + requestSignal = config?.signal instanceof AbortSignal ? config.signal : undefined + requestSignal?.addEventListener( + "abort", + () => reject(new AxiosError("timeout", "ERR_CANCELED")), + { once: true }, + ) + }) + if (phase === "discovery") get.mockImplementationOnce(hang) + else post.mockImplementationOnce((url, _body, config) => hang(url, config)) + const pending = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await vi.waitFor(() => expect(requestSignal).toBe(deadline.signal)) + expect(timeout).toHaveBeenLastCalledWith( + phase === "discovery" ? MODEL_CATALOG_FETCH_TIMEOUT_MS : LM_STUDIO_MODEL_LOAD_TIMEOUT_MS, + ) + deadline.abort() + await pending + expect(mockFlushModels).not.toHaveBeenCalled() + timeout.mockRestore() + await forceFullModelDetailsLoad(baseUrl, modelKey, handler) + expect(mockFlushModels).toHaveBeenCalledOnce() + } finally { + timeout.mockRestore() + } + }, + ) + + it.each([modelKey, "custom"])("reuses an existing instance for %s and refreshes metadata", async (savedId) => { + get.mockResolvedValueOnce(nativeResponse([{ id: "custom", config: { context_length: 2048 } }])) + await forceFullModelDetailsLoad(baseUrl, savedId, handler) + expect(post).not.toHaveBeenCalled() + expect(mockFlushModels).toHaveBeenCalledExactlyOnceWith( + { provider: providerIdentifiers.lmstudio, baseUrl }, + handler, + true, + ) + }) - it("should log only the origin of a base URL that holds credentials and a query", async () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(function () {}) - const error = new Error("Connection refused") - ;(error as NodeJS.ErrnoException).code = "ECONNREFUSED" - mockedAxios.get.mockRejectedValueOnce(error) + it("loads by exact catalog key, then refreshes", async () => { + get.mockResolvedValueOnce(nativeResponse([])) + let finishLoad: () => void = () => { + throw new Error("load not started") + } + post.mockImplementationOnce( + () => + new Promise((resolve) => { + finishLoad = () => resolve({ data: { instance_id: modelKey } }) + }), + ) + const pending = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await vi.waitFor(() => expect(post).toHaveBeenCalledOnce()) + expect(mockFlushModels).not.toHaveBeenCalled() + finishLoad() + await pending + expect(mockFlushModels).toHaveBeenCalledTimes(1) + expect(post).toHaveBeenCalledExactlyOnceWith( + `${baseUrl}/api/v1/models/load`, + { model: modelKey }, + { signal: expect.any(AbortSignal) }, + ) + expect(post.mock.invocationCallOrder[0]).toBeLessThan(mockFlushModels.mock.invocationCallOrder[0]) + }) - await getLMStudioModels("http://user:pass@localhost:1234/path?token=secret") + it("shares one load and refresh across concurrent callers, including normalized URLs", async () => { + get.mockResolvedValue(nativeResponse([])) + let finishRefresh: (() => void) | undefined + mockFlushModels.mockImplementation( + () => + new Promise((resolve) => { + finishRefresh = resolve + }), + ) + post.mockResolvedValue({ data: {} }) + const first = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await vi.waitFor(() => expect(mockFlushModels).toHaveBeenCalledTimes(1)) + const second = forceFullModelDetailsLoad(`${baseUrl}/`, modelKey, handler) + expect(get).toHaveBeenCalledTimes(1) + expect(post).toHaveBeenCalledTimes(1) + finishRefresh?.() + await Promise.all([first, second]) + expect(mockFlushModels).toHaveBeenCalledTimes(1) + }) - expect(warnSpy).toHaveBeenCalledWith("Error connecting to LMStudio at http://localhost:1234") - warnSpy.mockRestore() + it("prevents late pre-load discovery from creating a duplicate after the first operation finishes", async () => { + let finishLoad: () => void = () => { + throw new Error("load not started") + } + let finishLateDiscovery: (response: ReturnType) => void = () => { + throw new Error("discovery not set up") + } + const lateDiscovery = new Promise>((resolve) => { + finishLateDiscovery = resolve }) + get.mockResolvedValueOnce(nativeResponse([])).mockReturnValueOnce(lateDiscovery) + post.mockImplementationOnce( + () => + new Promise((resolve) => { + finishLoad = () => resolve({ data: {} }) + }), + ) + const first = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await vi.waitFor(() => expect(post).toHaveBeenCalledTimes(1)) + const second = forceFullModelDetailsLoad(baseUrl, modelKey, handler) + finishLoad() + await first + // The old implementation started this second discovery before the first + // load completed, then acted on its stale response after releasing the slot. + finishLateDiscovery(nativeResponse([])) + await second + expect(get).toHaveBeenCalledTimes(1) + expect(post).toHaveBeenCalledTimes(1) + expect(mockFlushModels).toHaveBeenCalledTimes(1) + }) - it("should return an empty object and log error if listDownloadedModels fails", async () => { - const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(function () {}) - const listError = new Error("LMStudio SDK internal error") - - mockedAxios.get.mockResolvedValueOnce({ data: {} }) - mockListLoaded.mockRejectedValueOnce(listError) - - const result = await getLMStudioModels(baseUrl) + it("keeps independent model keys separate on the same server", async () => { + get.mockResolvedValue({ data: { models: [nativeModel([]), { ...nativeModel([]), key: "another-model" }] } }) + post.mockResolvedValue({ data: {} }) + await Promise.all([ + forceFullModelDetailsLoad(baseUrl, modelKey, handler), + forceFullModelDetailsLoad(baseUrl, "another-model", handler), + ]) + expect(post).toHaveBeenCalledTimes(2) + expect(post).toHaveBeenCalledWith( + `${baseUrl}/api/v1/models/load`, + { model: "another-model" }, + { signal: expect.any(AbortSignal) }, + ) + }) - expect(mockedAxios.get).toHaveBeenCalledTimes(1) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledTimes(1) - expect(MockedLMStudioClientConstructor).toHaveBeenCalledWith({ baseUrl: lmsUrl }) - expect(mockListLoaded).toHaveBeenCalledTimes(1) - expect(consoleErrorSpy).toHaveBeenCalledWith( - `Error fetching LMStudio models: ${JSON.stringify(listError, Object.getOwnPropertyNames(listError), 2)}`, - ) - expect(result).toEqual({}) - consoleErrorSpy.mockRestore() + it("records loaded details from the refreshed catalog rather than an optimistic load flag", async () => { + get.mockResolvedValueOnce(nativeResponse([])).mockResolvedValueOnce(nativeResponse()) + post.mockResolvedValue({ data: {} }) + let loadedBeforeRefresh: boolean | undefined + mockFlushModels.mockImplementationOnce(async () => { + loadedBeforeRefresh = hasLoadedFullDetails(modelKey, baseUrl) + await getLMStudioModels(baseUrl) }) + await forceFullModelDetailsLoad(baseUrl, modelKey, handler) + expect(loadedBeforeRefresh).toBe(false) + expect(hasLoadedFullDetails(modelKey, baseUrl)).toBe(true) + }) - it("should pass the caller's abort signal to the connection probe", async () => { - const controller = new AbortController() - mockedAxios.get.mockResolvedValueOnce({ data: { status: "ok" } }) - mockListDownloadedModels.mockResolvedValueOnce([]) - mockListLoaded.mockResolvedValueOnce([]) - - await getLMStudioModels(baseUrl, { signal: controller.signal }) - - expect(mockedAxios.get).toHaveBeenCalledWith(`${baseUrl}/v1/models`, { signal: controller.signal }) - }) + it("does not combine loads for different servers", async () => { + get.mockResolvedValue(nativeResponse([])) + post.mockResolvedValue({ data: {} }) + await Promise.all([ + forceFullModelDetailsLoad(baseUrl, modelKey, handler), + forceFullModelDetailsLoad("http://other:1234", modelKey, handler), + ]) + expect(post).toHaveBeenCalledTimes(2) + expect(mockFlushModels).toHaveBeenCalledTimes(2) + }) - it("should reject with an AbortError when the signal aborts the probe, without calling the SDK", async () => { - const controller = new AbortController() - mockedAxios.get.mockImplementation((_url: string, config?: { signal?: AbortSignal }) => { - // Mirror the HTTP client: a request rejects when its signal fires, - // including when the signal was already aborted when the request started. - return new Promise((_resolve, reject) => { - if (config?.signal?.aborted) { - reject(new Error("canceled")) - return - } - config?.signal?.addEventListener?.("abort", () => reject(new Error("canceled")), { once: true }) - }) - }) + it.each(["load", "refresh"])("permits a retry after a failed %s", async (failure) => { + get.mockResolvedValue(nativeResponse([])) + post.mockResolvedValue({ data: {} }) + if (failure === "load") post.mockRejectedValueOnce(new Error("load failed")) + else mockFlushModels.mockRejectedValueOnce(new Error("refresh failed")) + await forceFullModelDetailsLoad(baseUrl, modelKey, handler) + await forceFullModelDetailsLoad(baseUrl, modelKey, handler) + expect(post).toHaveBeenCalledTimes(2) + expect(mockFlushModels).toHaveBeenCalledTimes(failure === "load" ? 1 : 2) + }) - const fetchPromise = getLMStudioModels(baseUrl, { signal: controller.signal }) - controller.abort() + it.each(["legacy/path/model.gguf", "unrelated-model"])("does not guess a key for %s", async (savedId) => { + get.mockResolvedValueOnce(nativeResponse([])) + await forceFullModelDetailsLoad(baseUrl, savedId, handler) + expect(post).not.toHaveBeenCalled() + expect(mockFlushModels).not.toHaveBeenCalled() + }) - await expect(fetchPromise).rejects.toMatchObject({ name: "AbortError" }) - expect(MockedLMStudioClientConstructor).not.toHaveBeenCalled() - expect(mockListDownloadedModels).not.toHaveBeenCalled() - expect(mockListLoaded).not.toHaveBeenCalled() + it("does not invent a load endpoint on legacy servers", async () => { + get.mockRejectedValueOnce(httpError(404)).mockResolvedValueOnce({ + data: { data: [{ id: "legacy", type: "llm" }] }, }) + await forceFullModelDetailsLoad(baseUrl, "legacy", handler) + expect(post).not.toHaveBeenCalled() }) }) diff --git a/src/api/providers/fetchers/lmstudio.ts b/src/api/providers/fetchers/lmstudio.ts index bd74c87666..8433d5d3c8 100644 --- a/src/api/providers/fetchers/lmstudio.ts +++ b/src/api/providers/fetchers/lmstudio.ts @@ -1,148 +1,231 @@ import axios from "axios" -import { LLM, LLMInfo, LLMInstanceInfo, LMStudioClient } from "@lmstudio/sdk" +import { z } from "zod" import { type ModelInfo, lMStudioDefaultModelInfo, providerIdentifiers } from "@roo-code/types" -import { throwIfAborted } from "../utils/abort-signal" +import { isTimeoutAbort, mergeAbortSignalAndTimeout, throwIfAborted } from "../utils/abort-signal" import { getLoggableOrigin } from "../utils/endpoint-origin" +import { normalizeLMStudioBaseUrl } from "../utils/lmstudio" import type { ApiHandler } from "../../index" -import { flushModels } from "./modelCache" +import { flushModels, MODEL_CATALOG_FETCH_TIMEOUT_MS } from "./modelCache" + +const contextLengthSchema = z.number().int().positive() +const instanceSchema = z.object({ + id: z.string(), + config: z.object({ context_length: contextLengthSchema.optional() }), +}) +const nativeModelSchema = z.object({ + type: z.string(), + key: z.string(), + display_name: z.string().optional(), + max_context_length: contextLengthSchema.nullish(), + capabilities: z.object({ vision: z.boolean().optional() }).optional(), + loaded_instances: z.array(instanceSchema), +}) +// Entries are validated one by one, so one unexpected model cannot empty the catalog. +const nativeListSchema = z.object({ models: z.array(z.unknown()) }) +const legacyListSchema = z.object({ + data: z.array( + z.object({ + id: z.string(), + type: z.string(), + max_context_length: contextLengthSchema.nullish(), + }), + ), +}) +const openAIListSchema = z.object({ data: z.array(z.object({ id: z.string() })) }) + +type NativeModel = z.infer +type Instance = z.infer + +// Loading large model files can take minutes; discovery keeps its shorter deadline. +export const LM_STUDIO_MODEL_LOAD_TIMEOUT_MS = 10 * 60_000 +const modelsWithLoadedDetails = new Map>() +const pendingLoads = new Map>() +// Runtime aliases support saved selections without adding entries to model dropdowns. +const instanceAliases = new Map>() + +export const getLMStudioModelAlias = (modelId: string, baseUrl?: string): ModelInfo | undefined => + instanceAliases.get(normalizeLMStudioBaseUrl(baseUrl))?.get(modelId) + +export const hasLoadedFullDetails = (modelId: string, baseUrl?: string): boolean => + modelsWithLoadedDetails.get(normalizeLMStudioBaseUrl(baseUrl))?.has(modelId) ?? false + +const isUnsupportedEndpoint = (error: unknown): boolean => + axios.isAxiosError(error) && (error.response?.status === 404 || error.response?.status === 405) + +// Older APIs may return HTTP 200 with an error string for an unsupported route. +const errorBodySchema = z.object({ error: z.string() }) + +async function discoverModels(baseUrl: string, externalSignal?: AbortSignal) { + const signal = mergeAbortSignalAndTimeout(externalSignal, MODEL_CATALOG_FETCH_TIMEOUT_MS) + let firstServerError: Error | undefined + try { + const { data } = await axios.get(`${baseUrl}/api/v1/models`, { signal }) + const parsedList = nativeListSchema.safeParse(data) + if (parsedList.success) { + const models = parsedList.data.models.flatMap((entry) => { + const parsed = nativeModelSchema.safeParse(entry) + return parsed.success ? [parsed.data] : [] + }) + return { version: "v1" as const, models } + } + const serverError = errorBodySchema.safeParse(data) + if (!serverError.success) throw parsedList.error + firstServerError ??= new Error(serverError.data.error) + } catch (error) { + if (isTimeoutAbort(signal)) throw new Error("LM Studio model discovery timed out") + throwIfAborted(signal) + if (!isUnsupportedEndpoint(error)) throw error + } -const modelsWithLoadedDetails = new Set() + try { + const { data } = await axios.get(`${baseUrl}/api/v0/models`, { signal }) + const parsedList = legacyListSchema.safeParse(data) + if (parsedList.success) return { version: "v0" as const, models: parsedList.data.data } + const serverError = errorBodySchema.safeParse(data) + if (!serverError.success) throw parsedList.error + firstServerError ??= new Error(serverError.data.error) + } catch (error) { + if (isTimeoutAbort(signal)) throw new Error("LM Studio model discovery timed out") + throwIfAborted(signal) + if (!isUnsupportedEndpoint(error)) throw error + } -export const hasLoadedFullDetails = (modelId: string): boolean => modelsWithLoadedDetails.has(modelId) + let data: unknown + try { + ;({ data } = await axios.get(`${baseUrl}/v1/models`, { signal })) + } catch (error) { + if (isTimeoutAbort(signal)) throw new Error("LM Studio model discovery timed out") + throwIfAborted(signal) + // Keep the earlier server message only when this endpoint is also unsupported. + throw isUnsupportedEndpoint(error) ? (firstServerError ?? error) : error + } + const parsed = openAIListSchema.safeParse(data) + if (parsed.success) return { version: "openai" as const, models: parsed.data.data } + const serverError = errorBodySchema.safeParse(data) + throw firstServerError ?? (serverError.success ? new Error(serverError.data.error) : parsed.error) +} + +// Prefer the instance explicitly selected by the user; otherwise use server order. +// Catalog keys identify model files, whereas instance IDs identify runtime configurations. +const selectInstance = (model: NativeModel, savedId: string): Instance | undefined => + model.loaded_instances.find(({ id }) => id === savedId) ?? model.loaded_instances[0] + +export const parseLMStudioModel = (model: NativeModel, savedId = model.key): ModelInfo => { + const contextLength = selectInstance(model, savedId)?.config.context_length ?? model.max_context_length + return { + ...lMStudioDefaultModelInfo, + description: `${model.display_name ?? model.key} - ${model.key}`, + supportsImages: model.capabilities?.vision ?? false, + ...(contextLength ? { contextWindow: contextLength, maxTokens: contextLength } : {}), + } +} + +function logModelError(action: string, error: unknown, baseUrl: string) { + if (axios.isAxiosError(error) && error.code === "ECONNREFUSED") { + console.warn(`Error connecting to LMStudio at ${getLoggableOrigin(baseUrl)}`) + } else { + console.error( + `Error ${action} LMStudio models at ${getLoggableOrigin(baseUrl)}:`, + error instanceof Error ? error.message : String(error), + ) + } +} export const forceFullModelDetailsLoad = async ( - baseUrl: string, + baseUrl: string | undefined, modelId: string, handler: ApiHandler, ): Promise => { + baseUrl = normalizeLMStudioBaseUrl(baseUrl) try { - // Test the connection to LM Studio first - // Crrors will be caught further down. - await axios.get(`${baseUrl}/v1/models`) - const lmsUrl = baseUrl.replace(/^http:\/\//, "ws://").replace(/^https:\/\//, "wss://") - - const client = new LMStudioClient({ baseUrl: lmsUrl }) - await client.llm.model(modelId) - // Flush and refresh cache to get updated model details - await flushModels({ provider: providerIdentifiers.lmstudio, baseUrl }, handler, true) - - // Mark this model as having full details loaded. - modelsWithLoadedDetails.add(modelId) - } catch (error) { - if (error.code === "ECONNREFUSED") { - console.warn(`Error connecting to LMStudio at ${getLoggableOrigin(baseUrl)}`) - } else { - console.error( - `Error refreshing LMStudio model details: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`, - ) + // Acquire the slot before discovery so a late catalog response cannot start + // another load after an earlier caller has finished. An unloaded model can + // only resolve by its catalog key; instance aliases already imply a load. + const loadKey = JSON.stringify([baseUrl, modelId]) + let pending = pendingLoads.get(loadKey) + if (!pending) { + pending = (async () => { + const catalog = await discoverModels(baseUrl) + // Older HTTP APIs leave loading to chat/JIT. + if (catalog.version !== "v1") return + const llms = catalog.models.filter(({ type }) => type === "llm") + const model = + llms.find(({ key }) => key === modelId) ?? + llms.find(({ loaded_instances }) => loaded_instances.some(({ id }) => id === modelId)) + // Preserve unmatched SDK paths for chat rather than guessing a key. + if (!model) return + if (!selectInstance(model, modelId)) { + await axios.post( + `${baseUrl}/api/v1/models/load`, + { model: model.key }, + { + signal: mergeAbortSignalAndTimeout(undefined, LM_STUDIO_MODEL_LOAD_TIMEOUT_MS), + }, + ) + } + await flushModels({ provider: providerIdentifiers.lmstudio, baseUrl }, handler, true) + })().finally(() => pendingLoads.delete(loadKey)) + pendingLoads.set(loadKey, pending) } + await pending + } catch (error) { + logModelError("refreshing", error, baseUrl) } } -export const parseLMStudioModel = (rawModel: LLMInstanceInfo | LLMInfo): ModelInfo => { - // Handle both LLMInstanceInfo (from loaded models) and LLMInfo (from downloaded models) - const contextLength = "contextLength" in rawModel ? rawModel.contextLength : rawModel.maxContextLength - - const modelInfo: ModelInfo = Object.assign({}, lMStudioDefaultModelInfo, { - description: `${rawModel.displayName} - ${rawModel.path}`, - contextWindow: contextLength, - supportsPromptCache: true, - supportsImages: rawModel.vision, - maxTokens: contextLength, - }) - - return modelInfo -} - export async function getLMStudioModels( - baseUrl = "http://localhost:1234", + baseUrl?: string, opts?: { signal?: AbortSignal }, ): Promise> { - // clear the set of models that have full details loaded - modelsWithLoadedDetails.clear() - // clearing the input can leave an empty string; use the default in that case - baseUrl = baseUrl === "" ? "http://localhost:1234" : baseUrl - - const models: Record = {} - // ws is required to connect using the LMStudio library - const lmsUrl = baseUrl.replace(/^http:\/\//, "ws://").replace(/^https:\/\//, "wss://") - + baseUrl = normalizeLMStudioBaseUrl(baseUrl) + const models = Object.create(null) as Record + const loadedDetails = new Set() + const aliases = new Map() try { - if (!URL.canParse(lmsUrl)) { - return models - } - - // test the connection to LM Studio first - // errors will be caught further down - await axios.get(`${baseUrl}/v1/models`, { signal: opts?.signal }) - - // The SDK's model-list calls expose no cancellation option, so an abort - // during them cannot reach the network; releasing the shared cache entry - // and stopping the waiters happens at the model-cache layer. - const client = new LMStudioClient({ baseUrl: lmsUrl }) - - // First, try to get all downloaded models - try { - const downloadedModels = await client.system.listDownloadedModels("llm") - for (const model of downloadedModels) { - // Use the model path as the key since that's what users select - models[model.path] = parseLMStudioModel(model) + throwIfAborted(opts?.signal) + if (!URL.canParse(baseUrl)) return models + const catalog = await discoverModels(baseUrl, opts?.signal) + throwIfAborted(opts?.signal) + if (catalog.version === "v1") { + for (const model of catalog.models.filter(({ type }) => type === "llm")) { + models[model.key] = parseLMStudioModel(model) + if (selectInstance(model, model.key)?.config.context_length) loadedDetails.add(model.key) } - } catch { - console.warn("Failed to list downloaded models, falling back to loaded models only") - } - - // Get loaded models for their runtime info (context size) - const loadedModels = (await client.llm.listLoaded().then((models: LLM[]) => { - return Promise.all(models.map((m) => m.getModelInfo())) - })) as Array - - // Deduplicate: For each loaded model, check if any downloaded model path contains the loaded model's key - // This handles cases like loaded "llama-3.1" matching downloaded "Meta/Llama-3.1/Something" - // If found, remove the downloaded version and add the loaded model (prefer loaded over downloaded for accurate runtime info) - for (const lmstudioModel of loadedModels) { - const loadedModelId = lmstudioModel.modelKey.toLowerCase() - - // Find if any downloaded model path contains the loaded model's key as a path segment - // Use word boundaries or path separators to avoid false matches like "llama" matching "codellama" - const existingKey = Object.keys(models).find((key) => { - const keyLower = key.toLowerCase() - // Check if the loaded model ID appears as a distinct segment in the path - // This matches "llama-3.1" in "Meta/Llama-3.1/Something" but not "llama" in "codellama" - return ( - keyLower.includes(`/${loadedModelId}/`) || - keyLower.includes(`/${loadedModelId}`) || - keyLower.startsWith(`${loadedModelId}/`) || - keyLower === loadedModelId - ) - }) - - if (existingKey) { - // Remove the downloaded version - delete models[existingKey] + for (const model of catalog.models.filter(({ type }) => type === "llm")) { + for (const instance of model.loaded_instances) { + // Exact instance aliases preserve saved custom names and pre-variant IDs. + if (!Object.hasOwn(models, instance.id)) + aliases.set(instance.id, parseLMStudioModel(model, instance.id)) + if (instance.config.context_length) loadedDetails.add(instance.id) + } } - - // Add the loaded model (either as replacement or new entry) - models[lmstudioModel.modelKey] = parseLMStudioModel(lmstudioModel) - modelsWithLoadedDetails.add(lmstudioModel.modelKey) + } else if (catalog.version === "v0") { + for (const model of catalog.models.filter(({ type }) => type === "llm" || type === "vlm")) { + models[model.id] = { + ...lMStudioDefaultModelInfo, + description: model.id, + supportsImages: model.type === "vlm", + ...(model.max_context_length + ? { contextWindow: model.max_context_length, maxTokens: model.max_context_length } + : {}), + } + } + } else { + // OpenAI-compatible discovery has no type, vision, or context metadata. + for (const model of catalog.models) + models[model.id] = { ...lMStudioDefaultModelInfo, description: model.id, supportsImages: false } } + modelsWithLoadedDetails.set(baseUrl, loadedDetails) + instanceAliases.set(baseUrl, aliases) } catch (error) { - // An aborted connection probe must reject instead of falling through to - // the empty-catalog handling below. throwIfAborted(opts?.signal) - - if (error.code === "ECONNREFUSED") { - console.warn(`Error connecting to LMStudio at ${getLoggableOrigin(baseUrl)}`) - } else { - console.error( - `Error fetching LMStudio models: ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}`, - ) - } + modelsWithLoadedDetails.delete(baseUrl) + instanceAliases.delete(baseUrl) + logModelError("fetching", error, baseUrl) } - return models } diff --git a/src/api/providers/lm-studio.ts b/src/api/providers/lm-studio.ts index 54b0634d5d..d2591fb24c 100644 --- a/src/api/providers/lm-studio.ts +++ b/src/api/providers/lm-studio.ts @@ -25,7 +25,8 @@ import type { ModelCacheScope, } from "../index" import { getModels, getModelsFromCache } from "./fetchers/modelCache" -import { getLMStudioModels } from "./fetchers/lmstudio" +import { getLMStudioModels, getLMStudioModelAlias } from "./fetchers/lmstudio" +import { normalizeLMStudioBaseUrl } from "./utils/lmstudio" import { mergeAbortSignalAndTimeout, throwIfAborted, @@ -39,6 +40,8 @@ import { import { handleOpenAIError } from "./utils/error-handler" import { extractReasoningFromDelta } from "./utils/extract-reasoning" +class LmStudioModelSelectionError extends Error {} + export class LmStudioHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: OpenAI @@ -52,7 +55,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan const apiKey = "noop" this.client = new OpenAI({ - baseURL: (this.options.lmStudioBaseUrl || "http://localhost:1234") + "/v1", + baseURL: normalizeLMStudioBaseUrl(this.options.lmStudioBaseUrl) + "/v1", apiKey: apiKey, timeout: this.timeoutMs, }) @@ -72,7 +75,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan const models = await getModels( { provider: providerIdentifiers.lmstudio, - baseUrl: this.options.lmStudioBaseUrl, + baseUrl: normalizeLMStudioBaseUrl(this.options.lmStudioBaseUrl), }, this, ) @@ -191,6 +194,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan if (isRequestAborted(error, externalSignal)) { throw createAbortError("LM Studio") } + this.rethrowModelSelectionError(error) throw handleOpenAIError(error, this.providerName) } @@ -289,6 +293,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan if (isRequestAborted(error, externalSignal)) { throw createAbortError("LM Studio") } + this.rethrowModelSelectionError(error) throw new Error( "Please check the LM Studio developer logs to debug what went wrong. You may need to load the model with a larger context length to work with Zoo Code's prompts.", ) @@ -303,24 +308,44 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan } } + private rethrowModelSelectionError(error: unknown): void { + if (error instanceof LmStudioModelSelectionError) throw error + if (!error || typeof error !== "object" || !("status" in error)) return + if (error.status !== 400 && error.status !== 404) return + const detail = "error" in error ? error.error : undefined + const code = detail && typeof detail === "object" && "code" in detail ? detail.code : undefined + const type = detail && typeof detail === "object" && "type" in detail ? detail.type : undefined + const message = "message" in error && typeof error.message === "string" ? error.message : "" + // An empty server needs a model loaded, not a new selection. + if (/no models? loaded/i.test(message)) { + throw new LmStudioModelSelectionError( + "LM Studio has no model loaded. Load a model in LM Studio, or turn on just-in-time loading.", + ) + } + if ( + code === "model_not_found" || + type === "model_not_found" || + /model.*(?:not found|does not exist|not available)|no models? found/i.test(message) + ) { + throw new LmStudioModelSelectionError( + "LM Studio could not find the selected model. Reselect a model in the LM Studio provider settings. Your saved model ID has not been changed.", + ) + } + } + override getModel(): { id: string; info: ModelInfo } { const models = getModelsFromCache( { provider: providerIdentifiers.lmstudio, - baseUrl: this.options.lmStudioBaseUrl, + baseUrl: normalizeLMStudioBaseUrl(this.options.lmStudioBaseUrl), }, this, ) - if (models && this.options.lmStudioModelId && models[this.options.lmStudioModelId]) { - return { - id: this.options.lmStudioModelId, - info: models[this.options.lmStudioModelId], - } - } else { - return { - id: this.options.lmStudioModelId || "", - info: openAiModelInfoSaneDefaults, - } + const id = this.options.lmStudioModelId || "" + return { + id, + info: + models?.[id] ?? getLMStudioModelAlias(id, this.options.lmStudioBaseUrl) ?? openAiModelInfoSaneDefaults, } } @@ -370,6 +395,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan if (isRequestAborted(error, requestSignal)) { throw this.toCompletionAbortError(requestSignal) } + this.rethrowModelSelectionError(error) throw handleOpenAIError(error, this.providerName) } return response.choices[0]?.message.content || "" @@ -377,6 +403,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan if (isRequestAborted(error, requestSignal)) { throw this.toCompletionAbortError(requestSignal) } + this.rethrowModelSelectionError(error) throw new Error( "Please check the LM Studio developer logs to debug what went wrong. You may need to load the model with a larger context length to work with Zoo Code's prompts.", ) diff --git a/src/api/providers/utils/__tests__/lmstudio.spec.ts b/src/api/providers/utils/__tests__/lmstudio.spec.ts new file mode 100644 index 0000000000..e4b921c94c --- /dev/null +++ b/src/api/providers/utils/__tests__/lmstudio.spec.ts @@ -0,0 +1,23 @@ +import { DEFAULT_LM_STUDIO_BASE_URL, normalizeLMStudioBaseUrl } from "../lmstudio" + +describe("normalizeLMStudioBaseUrl", () => { + it.each([undefined, ""])("defaults an unset URL %j", (input) => { + expect(normalizeLMStudioBaseUrl(input)).toBe(DEFAULT_LM_STUDIO_BASE_URL) + }) + + it.each([ + ["http://localhost:1234", "http://localhost:1234"], + ["http://localhost:1234/", "http://localhost:1234"], + ["http://localhost:1234///", "http://localhost:1234"], + ["https://example.com/a//b///", "https://example.com/a//b"], + ["///", ""], + ])("trims only trailing slashes from %j", (input, expected) => { + expect(normalizeLMStudioBaseUrl(input)).toBe(expected) + }) + + it.each(["", "suffix"])("handles a long slash sequence followed by %j", (suffix) => { + const prefix = "http://localhost:1234" + const slashes = "/".repeat(100_000) + expect(normalizeLMStudioBaseUrl(prefix + slashes + suffix)).toBe(suffix ? prefix + slashes + suffix : prefix) + }) +}) diff --git a/src/api/providers/utils/lmstudio.ts b/src/api/providers/utils/lmstudio.ts new file mode 100644 index 0000000000..e903e0a689 --- /dev/null +++ b/src/api/providers/utils/lmstudio.ts @@ -0,0 +1,8 @@ +export const DEFAULT_LM_STUDIO_BASE_URL = "http://localhost:1234" + +export const normalizeLMStudioBaseUrl = (baseUrl?: string): string => { + const url = baseUrl || DEFAULT_LM_STUDIO_BASE_URL + let end = url.length + while (end > 0 && url[end - 1] === "/") end-- + return url.slice(0, end) +} diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 6cec3bdbe7..41e5a85d0e 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -660,9 +660,14 @@ export class ClineProvider // size; we do it now since we're starting a task with that model selected. if (cline.apiConfiguration && cline.apiConfiguration.apiProvider === providerIdentifiers.lmstudio) { try { - if (!hasLoadedFullDetails(cline.apiConfiguration.lmStudioModelId!)) { + if ( + !hasLoadedFullDetails( + cline.apiConfiguration.lmStudioModelId!, + cline.apiConfiguration.lmStudioBaseUrl, + ) + ) { await forceFullModelDetailsLoad( - cline.apiConfiguration.lmStudioBaseUrl ?? "http://localhost:1234", + cline.apiConfiguration.lmStudioBaseUrl, cline.apiConfiguration.lmStudioModelId!, buildApiHandler({ ...cline.apiConfiguration, apiProvider: providerIdentifiers.lmstudio }), ) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index b85a7ff69c..93585d844e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -1057,19 +1057,23 @@ describe("ClineProvider", () => { expect(forceFullModelDetailsLoad).toHaveBeenCalledWith("http://localhost:1234", "test-model", expect.anything()) }) - test("does not reload full model details when the LM Studio model is already loaded", async () => { - vi.mocked(hasLoadedFullDetails).mockReturnValue(true) + test.each([undefined, "http://localhost:1234"])( + "does not reload full model details already loaded at %s", + async (lmStudioBaseUrl) => { + vi.mocked(hasLoadedFullDetails).mockReturnValue(true) - await provider.performPreparationTasks({ - apiConfiguration: { - apiProvider: providerIdentifiers.lmstudio, - lmStudioBaseUrl: "http://localhost:1234", - lmStudioModelId: "test-model", - }, - } as Task) + await provider.performPreparationTasks({ + apiConfiguration: { + apiProvider: providerIdentifiers.lmstudio, + lmStudioBaseUrl, + lmStudioModelId: "test-model", + }, + } as Task) - expect(forceFullModelDetailsLoad).not.toHaveBeenCalled() - }) + expect(hasLoadedFullDetails).toHaveBeenCalledWith("test-model", lmStudioBaseUrl) + expect(forceFullModelDetailsLoad).not.toHaveBeenCalled() + }, + ) test("resolveWebviewView hydrates the saved terminalProfile into the process-wide Terminal state", async () => { const setTerminalProfileSpy = vi.spyOn(Terminal, "setTerminalProfile").mockImplementation(() => {}) diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index 2c01bc44c0..4eb9361132 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -914,6 +914,19 @@ describe("webviewMessageHandler - requestLmStudioModels", () => { }) }) + it.each([undefined, "", "http://localhost:1234/"])( + "uses the default server cache key for persisted URL %s", + async (lmStudioBaseUrl) => { + mockClineProvider.getState = vi.fn().mockResolvedValue({ apiConfiguration: { lmStudioBaseUrl } }) + mockGetModels.mockResolvedValue({}) + await webviewMessageHandler(mockClineProvider, { type: "requestLmStudioModels" }) + expect(mockGetModels).toHaveBeenCalledWith( + { provider: providerIdentifiers.lmstudio, baseUrl: "http://localhost:1234" }, + expect.anything(), + ) + }, + ) + it("prefers the request payload base URL over persisted settings", async () => { mockGetLMStudioModels.mockResolvedValue({}) diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 4adb29264b..d11059486f 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -101,6 +101,7 @@ import { GetModelsOptions } from "../../shared/api" import { generateSystemPrompt } from "./generateSystemPrompt" import { resolveDefaultSaveUri, saveLastExportPath } from "../../utils/export" import { getCommand } from "../../utils/commands" +import { normalizeLMStudioBaseUrl } from "../../api/providers/utils/lmstudio" import { getLMStudioModels } from "../../api/providers/fetchers/lmstudio" const ALLOWED_VSCODE_SETTINGS = new Set(["terminal.integrated.inheritEnv"]) @@ -1540,7 +1541,7 @@ export const webviewMessageHandler = async ( } else { const lmStudioOptions = { provider: providerIdentifiers.lmstudio, - baseUrl: lmStudioApiConfig.lmStudioBaseUrl, + baseUrl: normalizeLMStudioBaseUrl(lmStudioApiConfig.lmStudioBaseUrl), } const handler = handlerFor(lmStudioApiConfig, providerIdentifiers.lmstudio) // Flush cache and refresh to ensure fresh models. diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index ae3d504003..ba0860608b 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -224,11 +224,6 @@ "count": 1 } }, - "api/providers/fetchers/__tests__/lmstudio.test.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 3 - } - }, "api/providers/fetchers/__tests__/modelEndpointCache.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 9 diff --git a/src/package.json b/src/package.json index 6c59828b1d..a0e9264626 100644 --- a/src/package.json +++ b/src/package.json @@ -490,7 +490,6 @@ "@aws-sdk/credential-providers": "^3.922.0", "@smithy/node-http-handler": "^4.0.0", "@google/genai": "^1.29.1", - "@lmstudio/sdk": "^1.1.1", "@mistralai/mistralai": "^1.9.18", "@modelcontextprotocol/sdk": "1.32.2", "@qdrant/js-client-rest": "^1.14.0",