From 3aa633b91abe8b3c92103c9f7aebfa12dab70ac1 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 07:39:17 -0800 Subject: [PATCH 01/42] fix(code-index): port fork reliability fixes (#34, #67, #102, #110) Port the code-index reliability fixes from the simurg79/Roo-Code fork: - Parse files in an isolated worker so a parser crash or hang cannot take down the extension host. - Recover per file: a failing file is recorded and skipped instead of aborting the whole scan. - Share one retry budget across embedders and the vector store. - Pause and pace provider requests when rate-limited. - Report "waiting" separately from "failed" in the indexing status. - Validate the configuration at startup. --- packages/types/src/vscode-extension-host.ts | 23 + pnpm-lock.yaml | 2 +- src/core/task/Task.ts | 8 + .../task/__tests__/Task.run-shadow.spec.ts | 30 + src/core/task/__tests__/Task.spec.ts | 16 + src/core/tools/CodebaseSearchTool.ts | 40 +- .../__tests__/CodebaseSearchTool.spec.ts | 191 +- .../CodebaseSearchTool.workspace.spec.ts | 54 +- .../webview/__tests__/ClineProvider.spec.ts | 265 ++- src/core/webview/webviewMessageHandler.ts | 41 +- src/esbuild.mjs | 2 +- src/eslint-suppressions.json | 42 +- src/i18n/locales/ca/embeddings.json | 7 + src/i18n/locales/de/embeddings.json | 7 + src/i18n/locales/en/embeddings.json | 7 + src/i18n/locales/es/embeddings.json | 7 + src/i18n/locales/fr/embeddings.json | 7 + src/i18n/locales/hi/embeddings.json | 7 + src/i18n/locales/id/embeddings.json | 7 + src/i18n/locales/it/embeddings.json | 7 + src/i18n/locales/ja/embeddings.json | 7 + src/i18n/locales/ko/embeddings.json | 7 + src/i18n/locales/nl/embeddings.json | 7 + src/i18n/locales/pl/embeddings.json | 7 + src/i18n/locales/pt-BR/embeddings.json | 7 + src/i18n/locales/ru/embeddings.json | 7 + src/i18n/locales/tr/embeddings.json | 7 + src/i18n/locales/vi/embeddings.json | 7 + src/i18n/locales/zh-CN/embeddings.json | 7 + src/i18n/locales/zh-TW/embeddings.json | 7 + src/package.json | 2 +- .../__tests__/cache-manager.spec.ts | 400 ++-- .../__tests__/code-index-log.spec.ts | 179 ++ .../code-index-scan-executor.spec.ts | 236 ++- .../code-index-status-manager.spec.ts | 41 + .../__tests__/file-completion.spec.ts | 672 +++++++ .../__tests__/file-failure-registry.spec.ts | 91 + .../__tests__/fixtures/fault-loop.wasm | Bin 0 -> 39 bytes .../__tests__/fixtures/fault-loop.wat | 4 + .../__tests__/generation-allocator.spec.ts | 101 + .../__tests__/index-write-gate.spec.ts | 320 +++ .../code-index/__tests__/manager.spec.ts | 1709 ++++++++++------- .../__tests__/mock-code-index-log.ts | 3 + .../code-index/__tests__/orchestrator.spec.ts | 1300 +++++++------ .../parse-worker-client.real.spec.ts | 159 ++ .../__tests__/parse-worker-client.spec.ts | 255 +++ .../__tests__/parse-worker-imports.spec.ts | 56 + .../__tests__/parser-errors.spec.ts | 28 + .../__tests__/provider-pacer.spec.ts | 392 ++++ .../__tests__/search-service.spec.ts | 367 ++++ .../__tests__/service-factory.spec.ts | 188 ++ .../__tests__/state-manager.spec.ts | 113 ++ ...kspace-indexing-enablement-manager.spec.ts | 51 +- src/services/code-index/cache-manager.ts | 192 +- .../code-index/code-index-log-host.ts | 8 + src/services/code-index/code-index-log.ts | 138 ++ .../code-index/code-index-scan-executor.ts | 139 +- src/services/code-index/constants/index.ts | 9 +- .../__tests__/bedrock-pacing.spec.ts | 70 + .../embedders/__tests__/bedrock.spec.ts | 93 +- .../embedders/__tests__/cancellation.spec.ts | 258 +++ .../embedder-validation-manager.spec.ts | 9 + .../openai-compatible-rate-limit.spec.ts | 250 +-- .../__tests__/openai-compatible.spec.ts | 287 +-- .../embedders/__tests__/openai.spec.ts | 148 +- .../embedders/__tests__/openrouter.spec.ts | 26 +- .../__tests__/paced-transport-writer.ts | 82 + .../__tests__/rate-limit-delay.spec.ts | 130 ++ .../__tests__/rate-limit-scopes.spec.ts | 71 + src/services/code-index/embedders/bedrock.ts | 127 +- .../code-index/embedders/cancellation.ts | 15 + .../embedders/embedder-validation-manager.ts | 2 + src/services/code-index/embedders/gemini.ts | 11 +- src/services/code-index/embedders/mistral.ts | 11 +- src/services/code-index/embedders/ollama.ts | 56 +- .../code-index/embedders/openai-compatible.ts | 254 +-- src/services/code-index/embedders/openai.ts | 107 +- .../code-index/embedders/openrouter.ts | 273 +-- .../code-index/embedders/vercel-ai-gateway.ts | 15 +- src/services/code-index/file-completion.ts | 311 +++ .../code-index/file-failure-registry.ts | 169 ++ .../code-index/generation-allocator.ts | 58 + src/services/code-index/index-write-gate.ts | 176 ++ src/services/code-index/interfaces/cache.ts | 12 + .../interfaces/directory-scanner-factory.ts | 10 + .../code-index/interfaces/embedder.ts | 4 +- .../code-index/interfaces/file-processor.ts | 48 +- .../interfaces/file-watcher-factory.ts | 12 +- src/services/code-index/interfaces/manager.ts | 2 + .../code-index/interfaces/vector-store.ts | 15 +- src/services/code-index/manager.ts | 769 +++++--- src/services/code-index/orchestrator.ts | 572 +++--- .../code-index/parse-worker-client.ts | 189 ++ .../code-index/parse-worker-protocol.ts | 17 + src/services/code-index/parse-worker.ts | 38 + src/services/code-index/parser-errors.ts | 22 + src/services/code-index/parser-types.ts | 25 + .../processors/__tests__/file-watcher.spec.ts | 1005 ++++++---- .../__tests__/parser-constants.spec.ts | 38 + .../processors/__tests__/parser.spec.ts | 10 +- .../__tests__/parser.worker-contract.spec.ts | 368 ++++ .../__tests__/processor-factories.spec.ts | 131 +- .../processors/__tests__/scanner.spec.ts | 992 +++++----- .../processors/directory-scanner-factory.ts | 12 + .../processors/file-watcher-factory.ts | 12 + .../code-index/processors/file-watcher.ts | 679 +++---- .../code-index/processors/parser-constants.ts | 5 + src/services/code-index/processors/parser.ts | 227 ++- src/services/code-index/processors/scanner.ts | 652 ++----- src/services/code-index/provider-pacer.ts | 339 ++++ src/services/code-index/search-service.ts | 119 +- src/services/code-index/service-factory.ts | 31 +- .../shared/__tests__/indexing-errors.spec.ts | 139 ++ .../__tests__/provider-wait-message.spec.ts | 30 + .../shared/__tests__/retry-policy.spec.ts | 206 ++ .../__tests__/validation-helpers.spec.ts | 104 +- .../code-index/shared/indexing-errors.ts | 167 ++ .../shared/provider-wait-message.ts | 18 + .../code-index/shared/retry-policy.ts | 65 + .../code-index/shared/validation-helpers.ts | 28 +- src/services/code-index/state-manager.ts | 40 +- .../qdrant-client.http-pacing.spec.ts | 178 ++ .../qdrant-client.rate-limit.spec.ts | 150 ++ .../qdrant-client.retry-contract.spec.ts | 200 ++ .../code-index/vector-store/qdrant-client.ts | 188 +- .../workspace-indexing-enablement-manager.ts | 6 +- src/workers/codeIndexParse.ts | 1 + .../src/components/chat/CodeIndexPopover.tsx | 107 +- .../components/chat/IndexingStatusBadge.tsx | 32 +- .../chat/__tests__/CodeIndexPopover.test.tsx | 303 ++- .../__tests__/IndexingStatusBadge.spec.tsx | 413 ++-- .../__tests__/provider-wait-locales.spec.ts | 78 + webview-ui/src/i18n/locales/ca/settings.json | 21 + webview-ui/src/i18n/locales/de/settings.json | 21 + webview-ui/src/i18n/locales/en/settings.json | 23 +- webview-ui/src/i18n/locales/es/settings.json | 21 + webview-ui/src/i18n/locales/fr/settings.json | 21 + webview-ui/src/i18n/locales/hi/settings.json | 21 + webview-ui/src/i18n/locales/id/settings.json | 21 + webview-ui/src/i18n/locales/it/settings.json | 21 + webview-ui/src/i18n/locales/ja/settings.json | 21 + webview-ui/src/i18n/locales/ko/settings.json | 21 + webview-ui/src/i18n/locales/nl/settings.json | 21 + webview-ui/src/i18n/locales/pl/settings.json | 21 + .../src/i18n/locales/pt-BR/settings.json | 21 + webview-ui/src/i18n/locales/ru/settings.json | 21 + webview-ui/src/i18n/locales/tr/settings.json | 21 + webview-ui/src/i18n/locales/vi/settings.json | 21 + .../src/i18n/locales/zh-CN/settings.json | 21 + .../src/i18n/locales/zh-TW/settings.json | 21 + 150 files changed, 14342 insertions(+), 5848 deletions(-) create mode 100644 src/services/code-index/__tests__/code-index-log.spec.ts create mode 100644 src/services/code-index/__tests__/file-completion.spec.ts create mode 100644 src/services/code-index/__tests__/file-failure-registry.spec.ts create mode 100644 src/services/code-index/__tests__/fixtures/fault-loop.wasm create mode 100644 src/services/code-index/__tests__/fixtures/fault-loop.wat create mode 100644 src/services/code-index/__tests__/generation-allocator.spec.ts create mode 100644 src/services/code-index/__tests__/index-write-gate.spec.ts create mode 100644 src/services/code-index/__tests__/mock-code-index-log.ts create mode 100644 src/services/code-index/__tests__/parse-worker-client.real.spec.ts create mode 100644 src/services/code-index/__tests__/parse-worker-client.spec.ts create mode 100644 src/services/code-index/__tests__/parse-worker-imports.spec.ts create mode 100644 src/services/code-index/__tests__/parser-errors.spec.ts create mode 100644 src/services/code-index/__tests__/provider-pacer.spec.ts create mode 100644 src/services/code-index/__tests__/search-service.spec.ts create mode 100644 src/services/code-index/__tests__/state-manager.spec.ts create mode 100644 src/services/code-index/code-index-log-host.ts create mode 100644 src/services/code-index/code-index-log.ts create mode 100644 src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts create mode 100644 src/services/code-index/embedders/__tests__/cancellation.spec.ts create mode 100644 src/services/code-index/embedders/__tests__/paced-transport-writer.ts create mode 100644 src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts create mode 100644 src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts create mode 100644 src/services/code-index/embedders/cancellation.ts create mode 100644 src/services/code-index/file-completion.ts create mode 100644 src/services/code-index/file-failure-registry.ts create mode 100644 src/services/code-index/generation-allocator.ts create mode 100644 src/services/code-index/index-write-gate.ts create mode 100644 src/services/code-index/parse-worker-client.ts create mode 100644 src/services/code-index/parse-worker-protocol.ts create mode 100644 src/services/code-index/parse-worker.ts create mode 100644 src/services/code-index/parser-errors.ts create mode 100644 src/services/code-index/parser-types.ts create mode 100644 src/services/code-index/processors/__tests__/parser-constants.spec.ts create mode 100644 src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts create mode 100644 src/services/code-index/processors/parser-constants.ts create mode 100644 src/services/code-index/provider-pacer.ts create mode 100644 src/services/code-index/shared/__tests__/indexing-errors.spec.ts create mode 100644 src/services/code-index/shared/__tests__/provider-wait-message.spec.ts create mode 100644 src/services/code-index/shared/__tests__/retry-policy.spec.ts create mode 100644 src/services/code-index/shared/indexing-errors.ts create mode 100644 src/services/code-index/shared/provider-wait-message.ts create mode 100644 src/services/code-index/shared/retry-policy.ts create mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts create mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts create mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts create mode 100644 src/workers/codeIndexParse.ts create mode 100644 webview-ui/src/components/chat/__tests__/provider-wait-locales.spec.ts diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index 8a006c0639..2b714fefd6 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -857,6 +857,9 @@ export type CheckpointRestorePayload = z.infer } +export interface FailedFileStatus { + path: string + activity: "read" | "parse" | "embed" | "upsert" | "delete" | "verify" + reason: string + state: "retrying" | "exhausted" +} + +export interface FailedFileSummary extends FailedFileStatus { + count: number +} + +export interface ProviderWaitStatus { + service: string + resumeAt?: number + waitingFiles: string[] +} + export interface IndexingStatus { systemStatus: string message?: string @@ -882,6 +902,9 @@ export interface IndexingStatus { workspacePath?: string workspaceEnabled?: boolean autoEnableDefault?: boolean + failedFiles?: FailedFileStatus[] + failedFileSummary?: FailedFileSummary + providerWaits?: ProviderWaitStatus[] } export interface IndexingStatusUpdateMessage { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a949abf377..c792efec3c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -470,7 +470,7 @@ importers: specifier: 1.32.2 version: 1.32.2(zod@4.6.7) '@qdrant/js-client-rest': - specifier: ^1.14.0 + specifier: 1.20.0 version: 1.20.0(typescript@5.9.3) '@roo-code/cloud': specifier: workspace:^ diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 77a1333b39..950801a0c1 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -402,6 +402,12 @@ export class Task extends EventEmitter implements TaskLike { // write was queued with the value at publish time, so a queued write cannot publish // after the task was cancelled or disposed. cancellationGeneration: number = 0 + private readonly lifecycleAbortController = new AbortController() + + public get lifecycleSignal(): AbortSignal { + return this.lifecycleAbortController.signal + } + currentRequestAbortController?: AbortController /** * Controller for the waiter on an in-flight `ensureModelFetched()` call (see @@ -3613,6 +3619,7 @@ export class Task extends EventEmitter implements TaskLike { // the resumed run's, and only if it advances monotonically. this.cancellationGeneration++ this.driveRunEvent({ tag: "abortRequested", abandoned: isAbandoned }) + this.lifecycleAbortController.abort() } requestAbandon(): void { @@ -3626,6 +3633,7 @@ export class Task extends EventEmitter implements TaskLike { // task is gone. this.cancellationGeneration++ this.driveRunEvent({ tag: "disposeRequested" }) + this.lifecycleAbortController.abort() } private driveRunEvent(event: RunEvent): void { diff --git a/src/core/task/__tests__/Task.run-shadow.spec.ts b/src/core/task/__tests__/Task.run-shadow.spec.ts index 5ddc4261a8..b57688b404 100644 --- a/src/core/task/__tests__/Task.run-shadow.spec.ts +++ b/src/core/task/__tests__/Task.run-shadow.spec.ts @@ -97,6 +97,36 @@ describe("Task run-state write helpers", () => { expect(task.runState.stream).toEqual({ tag: "live", generation: 2, cleanupFinished: false }) }) + it.each(["abort", "abandon", "dispose"] as const)("keeps one lifetime signal after %s", (operation) => { + const signal = task.lifecycleSignal + const onAbort = vi.fn() + signal.addEventListener("abort", onAbort) + expect(signal.aborted).toBe(false) + expect(task.lifecycleSignal).toBe(signal) + + if (operation === "dispose") task.requestDispose() + else task.requestAbort(operation === "abandon") + + const reason: unknown = signal.reason + expect(reason).toBeInstanceOf(DOMException) + expect(reason).toHaveProperty("name", "AbortError") + expect(signal.aborted).toBe(true) + task.requestAbort(false) + task.requestDispose() + expect(task.lifecycleSignal).toBe(signal) + expect(task.lifecycleSignal.reason).toBe(reason) + expect(onAbort).toHaveBeenCalledOnce() + const resumed = makeTask() + expect(resumed.lifecycleSignal).not.toBe(signal) + expect(resumed.lifecycleSignal.aborted).toBe(false) + }) + + it("does not end the lifetime signal when cancelling an individual request", () => { + task.currentRequestAbortController = new AbortController() + task.cancelCurrentRequest() + expect(task.lifecycleSignal.aborted).toBe(false) + }) + it("requestAbort(false) sets abort only", () => { task.requestAbort(false) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 3fea8b61bc..3f79bc5223 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -4784,9 +4784,15 @@ describe("Cline", () => { }) vi.spyOn(task, "dispose").mockResolvedValue(undefined) + const lifetimeSignal = task.lifecycleSignal + expect(lifetimeSignal.aborted).toBe(false) expect(task.cancellationGeneration).toBe(0) await task.abortTask() + const lifetimeReason: unknown = lifetimeSignal.reason + expect(lifetimeSignal.aborted).toBe(true) + expect(lifetimeReason).toBeInstanceOf(DOMException) + expect(lifetimeReason).toHaveProperty("name", "AbortError") expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(1) @@ -4795,10 +4801,14 @@ describe("Cline", () => { // on resume: that is exactly why the generation exists. Only the // generation keeps the cancelled run and the resumed run apart. expect(task.cancellationGeneration).toBe(1) + expect(task.lifecycleSignal).toBe(lifetimeSignal) + expect(task.lifecycleSignal.aborted).toBe(true) + expect(task.lifecycleSignal.reason).toBe(lifetimeReason) await task.abortTask() expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(2) + expect(task.lifecycleSignal.reason).toBe(lifetimeReason) }) it("advances the cancellation generation on disposal, without any explicit cancel", async () => { @@ -4812,10 +4822,16 @@ describe("Cline", () => { startTask: false, }) + const lifetimeSignal = task.lifecycleSignal + expect(lifetimeSignal.aborted).toBe(false) expect(task.cancellationGeneration).toBe(0) await task.dispose() + expect(task.lifecycleSignal).toBe(lifetimeSignal) + expect(lifetimeSignal.aborted).toBe(true) + expect(lifetimeSignal.reason).toBeInstanceOf(DOMException) + expect(lifetimeSignal.reason).toHaveProperty("name", "AbortError") expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(1) }) diff --git a/src/core/tools/CodebaseSearchTool.ts b/src/core/tools/CodebaseSearchTool.ts index 2d1228794b..1a69b9153c 100644 --- a/src/core/tools/CodebaseSearchTool.ts +++ b/src/core/tools/CodebaseSearchTool.ts @@ -4,7 +4,6 @@ import { Task } from "../task/Task" import { CodeIndexManagerRegistry } from "../../services/code-index/code-index-manager-registry" import { getWorkspacePath } from "../../utils/path" import { formatResponse } from "../prompts/responses" -import { VectorStoreSearchResult } from "../../services/code-index/interfaces" import type { ToolUse } from "../../shared/tools" import { BaseTool, ToolCallbacks } from "./BaseTool" @@ -78,18 +77,36 @@ export class CodebaseSearchTool extends BaseTool<"codebase_search"> { throw new Error("Code Indexing is not initialized for this workspace.") } - const searchResults: VectorStoreSearchResult[] = await manager.searchIndex(query, directoryPrefix) - - if (!searchResults || searchResults.length === 0) { - pushToolResult(`No relevant code snippets found for the query: "${query}"`) - return - } + const { + results: searchResults, + incomplete, + failedPaths, + waitingPaths, + } = await manager.searchIndex(query, directoryPrefix, task.lifecycleSignal) + const incompleteNotice = incomplete + ? [ + "Partial index: results may be incomplete.", + ...(failedPaths.length ? ["Failed files:", ...failedPaths.map((file) => `- ${file}`)] : []), + ...(waitingPaths.length + ? [ + "Waiting files (indexing resumes automatically):", + ...waitingPaths.map((file) => `- ${file}`), + ] + : []), + ].join("\n") + "\n\n" + : "" const jsonResult = { query, results: [], + incomplete, + failedPaths, + waitingPaths, } as { query: string + incomplete: boolean + failedPaths: string[] + waitingPaths: string[] results: Array<{ filePath: string score: number @@ -117,7 +134,12 @@ export class CodebaseSearchTool extends BaseTool<"codebase_search"> { const payload = { tool: "codebaseSearch", content: jsonResult } await task.say("codebase_search_result", JSON.stringify(payload)) - const output = `Query: ${query} + if (jsonResult.results.length === 0) { + pushToolResult(`${incompleteNotice}No relevant code snippets found for the query: "${query}"`) + return + } + + const output = `${incompleteNotice}Query: ${query} Results: ${jsonResult.results @@ -131,7 +153,7 @@ Code Chunk: ${result.codeChunk} .join("\n")}` pushToolResult(output) - } catch (error: any) { + } catch (error) { await handleError("codebase_search", error) } } diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts index eb0c27c1ae..9d6b4ab616 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -46,6 +46,7 @@ describe("CodebaseSearchTool", () => { Task, | "cwd" | "providerRef" + | "lifecycleSignal" | "consecutiveMistakeCount" | "didToolFailInCurrentTurn" | "sayAndCreateMissingParamError" @@ -54,6 +55,7 @@ describe("CodebaseSearchTool", () => { > = { cwd: "/task", providerRef: { deref, [Symbol.toStringTag]: "WeakRef" }, + lifecycleSignal: new AbortController().signal, consecutiveMistakeCount: 3, didToolFailInCurrentTurn: false, sayAndCreateMissingParamError: vi @@ -75,7 +77,7 @@ describe("CodebaseSearchTool", () => { isFeatureConfigured: true, isInitialized: true, initialize: vi.fn().mockResolvedValue({ requiresRestart: false }), - searchIndex: vi.fn().mockResolvedValue([]), + searchIndex: vi.fn().mockResolvedValue(searchResponse()), } vi.mocked(CodeIndexManagerRegistry.getOrCreate).mockReturnValue(manager as CodeIndexManager) vi.mocked(getWorkspacePath).mockReturnValue("/fallback") @@ -96,6 +98,14 @@ describe("CodebaseSearchTool", () => { } } + function searchResponse( + results: VectorStoreSearchResult[] = [], + failedPaths: string[] = [], + waitingPaths: string[] = [], + ): Awaited> { + return { results, incomplete: failedPaths.length > 0 || waitingPaths.length > 0, failedPaths, waitingPaths } + } + it("exports a named tool instance", () => { expect(codebaseSearchTool).toBeInstanceOf(CodebaseSearchTool) expect(codebaseSearchTool.name).toBe(toolNamesSchema.enum.codebase_search) @@ -264,51 +274,86 @@ describe("CodebaseSearchTool", () => { async (path) => { vi.mocked(manager.searchIndex).mockImplementation(async () => { expect(task.consecutiveMistakeCount).toBe(0) - return [] + return searchResponse() }) await tool.execute({ query, path }, task, callbacks) expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, path) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, path, task.lifecycleSignal) expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( `No relevant code snippets found for the query: "${query}"`, ) expect(callbacks.handleError).not.toHaveBeenCalled() - expect(task.say).not.toHaveBeenCalled() + expect(task.say).toHaveBeenCalledExactlyOnceWith( + "codebase_search_result", + JSON.stringify({ + tool: "codebaseSearch", + content: { query, results: [], incomplete: false, failedPaths: [], waitingPaths: [] }, + }), + ) + expect(task.say).toHaveBeenCalledBefore(vi.mocked(callbacks.pushToolResult)) expect(vscode.workspace.asRelativePath).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() }, ) - it.each([null, undefined, false, 0, ""])( - "defensively handles a runtime-invalid falsy search response %j", - async (value) => { - // The manager promises an array. Deliberately violate that boundary to exercise the existing falsy guard. - vi.mocked(manager.searchIndex).mockResolvedValue(value as unknown as VectorStoreSearchResult[]) - await tool.execute({ query }, task, callbacks) - expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( - `No relevant code snippets found for the query: "${query}"`, - ) - expect(callbacks.handleError).not.toHaveBeenCalled() - expect(task.say).not.toHaveBeenCalled() - expect(vscode.workspace.asRelativePath).not.toHaveBeenCalled() - }, - ) + it.each([false, true])("warns about an incomplete index with matches: %s", async (hasMatches) => { + const failedPaths = ["src/failed.ts", "other/unreadable.ts"] + vi.mocked(manager.searchIndex).mockResolvedValue(searchResponse(hasMatches ? [result()] : [], failedPaths)) - it("preserves result order and metadata, relativizes paths without workspace prefixes and trims chunks", async () => { - vi.mocked(manager.searchIndex).mockResolvedValue([ - result(), - result({ - id: "second", - score: 0.5, - payload: { filePath: "/task/lib/other.ts", startLine: 10, endLine: 10, codeChunk: " \t " }, + await tool.execute({ query, path: "src" }, task, callbacks) + + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", task.lifecycleSignal) + expect(task.say).toHaveBeenCalledExactlyOnceWith( + "codebase_search_result", + JSON.stringify({ + tool: "codebaseSearch", + content: { + query, + results: hasMatches + ? [ + { + filePath: "src/result.ts", + score: 0.9, + startLine: 2, + endLine: 4, + codeChunk: "first\n second", + }, + ] + : [], + incomplete: true, + failedPaths, + waitingPaths: [], + }, }), - ]) + ) + const notice = + "Partial index: results may be incomplete.\nFailed files:\n- src/failed.ts\n- other/unreadable.ts\n\n" + const output = hasMatches + ? `Query: ${query}\nResults:\n\nFile path: src/result.ts\nScore: 0.9\nLines: 2-4\nCode Chunk: first\n second\n` + : `No relevant code snippets found for the query: "${query}"` + expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith(notice + output) + expect(task.say).toHaveBeenCalledBefore(vi.mocked(callbacks.pushToolResult)) + expect(vscode.workspace.asRelativePath).toHaveBeenCalledTimes(hasMatches ? 1 : 0) + expect(callbacks.handleError).not.toHaveBeenCalled() + }) + + it("preserves result order and metadata, relativizes paths without workspace prefixes and trims chunks", async () => { + vi.mocked(manager.searchIndex).mockResolvedValue( + searchResponse([ + result(), + result({ + id: "second", + score: 0.5, + payload: { filePath: "/task/lib/other.ts", startLine: 10, endLine: 10, codeChunk: " \t " }, + }), + ]), + ) vi.mocked(vscode.workspace.asRelativePath) .mockReturnValueOnce("src/result.ts") .mockReturnValueOnce("lib/other.ts") await tool.execute({ query }, task, callbacks) expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined, task.lifecycleSignal) expect(vscode.workspace.asRelativePath).toHaveBeenCalledTimes(2) expect(vscode.workspace.asRelativePath).toHaveBeenNthCalledWith(1, "/task/src/result.ts", false) expect(vscode.workspace.asRelativePath).toHaveBeenNthCalledWith(2, "/task/lib/other.ts", false) @@ -328,6 +373,9 @@ describe("CodebaseSearchTool", () => { }, { filePath: "lib/other.ts", score: 0.5, startLine: 10, endLine: 10, codeChunk: "" }, ], + incomplete: false, + failedPaths: [], + waitingPaths: [], }, }), ) @@ -341,12 +389,9 @@ describe("CodebaseSearchTool", () => { it("skips malformed entries while preserving a valid result", async () => { // Missing filePath is invalid under Payload's type but explicitly guarded against at runtime. const missingPath = { id: "malformed", score: 1, payload: { codeChunk: "ignored" } } as VectorStoreSearchResult - vi.mocked(manager.searchIndex).mockResolvedValue([ - result({ payload: undefined }), - result({ payload: null }), - missingPath, - result(), - ]) + vi.mocked(manager.searchIndex).mockResolvedValue( + searchResponse([result({ payload: undefined }), result({ payload: null }), missingPath, result()]), + ) await tool.execute({ query }, task, callbacks) expect(vscode.workspace.asRelativePath).toHaveBeenCalledExactlyOnceWith("/task/src/result.ts", false) expect(task.say).toHaveBeenCalledExactlyOnceWith( @@ -364,6 +409,9 @@ describe("CodebaseSearchTool", () => { codeChunk: "first\n second", }, ], + incomplete: false, + failedPaths: [], + waitingPaths: [], }, }), ) @@ -373,23 +421,26 @@ describe("CodebaseSearchTool", () => { expect(callbacks.handleError).not.toHaveBeenCalled() }) - it("preserves the existing empty-header response when every entry is skipped", async () => { + it("reports no snippets with complete metadata when every entry is skipped", async () => { // Missing filePath deliberately violates the backend payload contract. const missingPath = { id: "malformed", score: 1, payload: { codeChunk: "ignored" } } as VectorStoreSearchResult - vi.mocked(manager.searchIndex).mockResolvedValue([ - result({ payload: undefined }), - result({ payload: null }), - missingPath, - ]) + vi.mocked(manager.searchIndex).mockResolvedValue( + searchResponse([result({ payload: undefined }), result({ payload: null }), missingPath]), + ) await tool.execute({ query }, task, callbacks) expect(vscode.workspace.asRelativePath).not.toHaveBeenCalled() expect(task.say).toHaveBeenCalledExactlyOnceWith( "codebase_search_result", - JSON.stringify({ tool: "codebaseSearch", content: { query, results: [] } }), + JSON.stringify({ + tool: "codebaseSearch", + content: { query, results: [], incomplete: false, failedPaths: [], waitingPaths: [] }, + }), + ) + expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( + `No relevant code snippets found for the query: "${query}"`, ) - expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith(`Query: ${query}\nResults:\n\n`) expect(callbacks.handleError).not.toHaveBeenCalled() }) @@ -415,7 +466,7 @@ describe("CodebaseSearchTool", () => { await tool.execute({ query }, task, callbacks) - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined, task.lifecycleSignal) expect(callbacks.handleError).toHaveBeenCalledExactlyOnceWith(toolNamesSchema.enum.codebase_search, error) expect(vi.mocked(callbacks.handleError).mock.calls[0][1]).toBe(error) expect(callbacks.pushToolResult).not.toHaveBeenCalled() @@ -423,9 +474,43 @@ describe("CodebaseSearchTool", () => { expect(task.say).not.toHaveBeenCalled() }) + it.each(["already aborted", "midflight"] as const)("preserves the exact %s cancellation reason", async (timing) => { + const controller = new AbortController() + const reason = Object.freeze({ cancelled: timing }) + Object.defineProperty(task, "lifecycleSignal", { value: controller.signal }) + if (timing === "already aborted") controller.abort(reason) + vi.mocked(manager.searchIndex).mockImplementation(async (_query, _path, signal) => { + expect(signal).toBe(controller.signal) + if (!signal) throw new Error("Missing lifecycle signal") + signal.throwIfAborted() + return new Promise>>((_resolve, reject) => { + const onAbort = () => { + clearTimeout(deadline) + reject(signal.reason) + } + const deadline = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + reject(new Error("Search cancellation was not observed")) + }, 1_000) + signal.addEventListener("abort", onAbort, { once: true }) + queueMicrotask(() => controller.abort(reason)) + }) + }) + + await tool.execute({ query, path: "src" }, task, callbacks) + + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", controller.signal) + expect(controller.signal.reason).toBe(reason) + expect(callbacks.handleError).toHaveBeenCalledExactlyOnceWith(toolNamesSchema.enum.codebase_search, reason) + expect(vi.mocked(callbacks.handleError).mock.calls[0][1]).toBe(reason) + expect(callbacks.pushToolResult).not.toHaveBeenCalled() + expect(vscode.workspace.asRelativePath).not.toHaveBeenCalled() + expect(task.say).not.toHaveBeenCalled() + }) + it("forwards a result-message rejection without publishing a text result", async () => { const error = new Error("say failed") - vi.mocked(manager.searchIndex).mockResolvedValue([result()]) + vi.mocked(manager.searchIndex).mockResolvedValue(searchResponse([result()])) vi.mocked(task.say).mockRejectedValue(error) await tool.execute({ query }, task, callbacks) @@ -462,7 +547,7 @@ describe("CodebaseSearchTool", () => { } expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src") + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", task.lifecycleSignal) expect(task.consecutiveMistakeCount).toBe(0) expect(callbacks.handleError).not.toHaveBeenCalled() }) @@ -521,4 +606,22 @@ describe("CodebaseSearchTool", () => { expect(task.say).not.toHaveBeenCalled() }) }) + + it.each([false, true])( + "prints separate waiting and failed warnings before output, matches=%s", + async (hasMatches) => { + vi.mocked(manager.searchIndex).mockResolvedValue( + searchResponse(hasMatches ? [result()] : [], ["failed.ts"], ["waiting.ts"]), + ) + await tool.execute({ query }, task, callbacks) + const output = vi.mocked(callbacks.pushToolResult).mock.calls[0][0] + expect(output).toEqual( + expect.stringContaining( + "Partial index: results may be incomplete.\nFailed files:\n- failed.ts\n" + + "Waiting files (indexing resumes automatically):\n- waiting.ts\n\n", + ), + ) + expect(output).toEqual(expect.stringContaining(hasMatches ? "Query:" : "No relevant code snippets")) + }, + ) }) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts index f6f6c0f5cb..b4ae44fd8f 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts @@ -37,17 +37,23 @@ vi.mock("../../../services/code-index/manager", () => ({ initialized = true return { requiresRestart: false } }), - searchIndex: vi.fn().mockResolvedValue([ - { - score: 0.9, - payload: { - filePath: `${workspacePath}/src/result.ts`, - startLine: 2, - endLine: 4, - codeChunk: " match ", + searchIndex: vi.fn().mockResolvedValue({ + results: [ + { + id: "workspace-result", + score: 0.9, + payload: { + filePath: `${workspacePath}/src/result.ts`, + startLine: 2, + endLine: 4, + codeChunk: " match ", + }, }, - }, - ]), + ], + incomplete: false, + failedPaths: [], + waitingPaths: [], + }), dispose: vi.fn(), } }), @@ -75,8 +81,9 @@ describe("CodebaseSearchTool workspace selection", () => { vi.mocked(getWorkspacePath).mockReturnValue(first.uri.fsPath) // Only the provider context and task members consumed by this tool are needed. provider = { context: makeExtensionContext(), contextProxy: {} } as ClineProvider - const taskStub: Pick = { + const taskStub: Pick = { cwd: second.uri.fsPath, + lifecycleSignal: new AbortController().signal, providerRef: new WeakRef(provider), consecutiveMistakeCount: 0, say: vi.fn().mockResolvedValue(undefined), @@ -106,7 +113,7 @@ describe("CodebaseSearchTool workspace selection", () => { await new CodebaseSearchTool().execute({ query: "find match", path: "src" }, task, callbacks) expect(callbacks.handleError).not.toHaveBeenCalled() - expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("find match", "src") + expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("find match", "src", task.lifecycleSignal) expect(taskManager.initialize).not.toHaveBeenCalled() expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() @@ -122,6 +129,9 @@ describe("CodebaseSearchTool workspace selection", () => { results: [ { filePath: "/second/src/result.ts", score: 0.9, startLine: 2, endLine: 4, codeChunk: "match" }, ], + incomplete: false, + failedPaths: [], + waitingPaths: [], }, }), ) @@ -137,7 +147,7 @@ describe("CodebaseSearchTool workspace selection", () => { await new CodebaseSearchTool().execute({ query: "fallback" }, task, callbacks) expect(getWorkspacePath).toHaveBeenCalledExactlyOnceWith() - expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("fallback", undefined) + expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("fallback", undefined, task.lifecycleSignal) expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(callbacks.handleError).not.toHaveBeenCalled() expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( @@ -181,9 +191,19 @@ describe("CodebaseSearchTool workspace selection", () => { await tool.execute({ query: "without an editor" }, task, callbacks) expect(taskManager.searchIndex).toHaveBeenCalledTimes(3) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith(1, "before editor switch", undefined) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith(2, "after editor switch", undefined) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith(3, "without an editor", undefined) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith( + 1, + "before editor switch", + undefined, + task.lifecycleSignal, + ) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith( + 2, + "after editor switch", + undefined, + task.lifecycleSignal, + ) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith(3, "without an editor", undefined, task.lifecycleSignal) expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() expect(callbacks.handleError).not.toHaveBeenCalled() @@ -225,7 +245,7 @@ describe("CodebaseSearchTool workspace selection", () => { expect(manager.state).toBe("Indexing") expect(manager.initialize).not.toHaveBeenCalled() - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith("external match", undefined) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith("external match", undefined, task.lifecycleSignal) expect(callbacks.handleError).not.toHaveBeenCalled() }) }) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 464889418e..a639abdad7 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -31,6 +31,7 @@ import { experimentDefault } from "../../../shared/experiments" import { setTtsEnabled } from "../../../utils/tts" import { ContextProxy } from "../../config/ContextProxy" import { WorkspaceIndexingEnablementManager } from "../../../services/code-index/workspace-indexing-enablement-manager" +import type { CodeIndexManager } from "../../../services/code-index/manager" import { Task, TaskOptions } from "../../task/Task" import { safeWriteJson } from "../../../utils/safeWriteJson" @@ -5813,10 +5814,23 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { initialize: vi.fn().mockResolvedValue(undefined), state: "Standby", isInitialized: true, - startIndexing: vi.fn().mockResolvedValue(undefined), - stopIndexing: vi.fn(), + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + recoverFromError: vi.fn().mockResolvedValue(undefined), clearIndexData: vi.fn().mockResolvedValue(undefined), - getCurrentStatus: vi.fn().mockReturnValue({ systemStatus: "Standby" }), + getCurrentStatus: vi.fn().mockReturnValue({ + systemStatus: "Standby", + message: "Ready", + processedItems: 0, + totalItems: 0, + currentItemUnit: "blocks", + workspacePath: "/test/workspace", + workspaceEnabled: true, + autoEnableDefault: false, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], + }), }, overrides, ) @@ -5825,27 +5839,51 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { vi.clearAllMocks() }) - it("logs a detached indexing rejection without rejecting the handler", async () => { - let rejectIndexing!: (error: Error) => void - const indexingPromise = new Promise((_, reject) => { - rejectIndexing = reject + const deferred = () => { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise }) - const manager = createIndexManager({ - startIndexing: vi.fn().mockReturnValue(indexingPromise), + return { promise, resolve, reject } + } + + it.each(["resolve", "reject"] as const)("awaits explicit indexing startup before %s", async (outcome) => { + const startup = deferred() + const entered = deferred() + const manager = createIndexManager() + manager.startIndexing.mockImplementation(() => { + entered.resolve() + return startup.promise }) const provider = createProvider({ getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), }) - - await expect(webviewMessageHandler(provider, { type: "startIndexing" })).resolves.toBeUndefined() - expect(provider.getCurrentWorkspaceCodeIndexScope).toHaveBeenCalledOnce() - expect(manager.startIndexing).toHaveBeenCalledOnce() - - rejectIndexing(new Error("boom")) + let completed = false + const pending = webviewMessageHandler(provider, { type: "startIndexing" }).then(() => { + completed = true + }) + await entered.promise await Promise.resolve() await Promise.resolve() - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: boom") + expect(completed).toBe(false) + expect(provider.getCurrentWorkspaceCodeIndexScope).toHaveBeenCalledOnce() + expect(manager.setWorkspaceEnabled).toHaveBeenCalledExactlyOnceWith(true) + expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.startIndexing) + expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) + expect(manager.initialize).not.toHaveBeenCalled() + expect(manager.recoverFromError).not.toHaveBeenCalled() + expect(provider.log).not.toHaveBeenCalled() + if (outcome === "reject") startup.reject(new Error("boom")) + else startup.resolve() + await pending + + expect(completed).toBe(true) + expect(manager.startIndexing).toHaveBeenCalledOnce() + if (outcome === "reject") expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error starting indexing: boom") + else expect(provider.log).not.toHaveBeenCalled() }) it("covers the changed task-operation happy paths", async () => { @@ -6001,7 +6039,15 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(manager.getCurrentStatus).not.toHaveBeenCalled() expect(provider.postMessageToWebview).toHaveBeenCalledWith({ type: "indexingStatusUpdate", - values: expect.objectContaining({ systemStatus: "Error", message: expect.any(String) }), + values: { + systemStatus: "Error", + message: expect.any(String), + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", + workspacePath: undefined, + failedFiles: [], + }, }) await webviewMessageHandler(provider, { type: "requestIndexingStatus" }) expect(getScope).toHaveBeenCalledTimes(2) @@ -6022,27 +6068,108 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.log).toHaveBeenCalledWith("Cannot start indexing: No workspace folder open") }) - it("catches both start-indexing calls during error recovery", async () => { - const manager = createIndexManager({ - isInitialized: false, - startIndexing: vi - .fn() - .mockRejectedValueOnce(new Error("first failure")) - .mockRejectedValueOnce(new Error("second failure")), + it.each(["requestIndexingStatus", "stopIndexing"] as const)( + "forwards the whole computed status snapshot for %s", + async (type) => { + const status: ReturnType = { + systemStatus: "Standby", + message: "Partial index: 2 files failed", + processedItems: 7, + totalItems: 9, + currentItemUnit: "files", + workspacePath: "/snapshot/workspace", + workspaceEnabled: false, + autoEnableDefault: true, + failedFiles: [ + { path: "src/first.ts", activity: "embed", reason: "Provider unavailable", state: "exhausted" }, + { path: "src/second.ts", activity: "verify", reason: "Verification failed", state: "exhausted" }, + ], + failedFileSummary: { + path: "src/first.ts", + activity: "embed", + reason: "Provider unavailable", + state: "exhausted", + count: 2, + }, + providerWaits: [{ service: "embedding", resumeAt: 123_000, waitingFiles: ["src/waiting.ts"] }], + } + const manager = createIndexManager() + manager.getCurrentStatus.mockReturnValue(status) + const provider = createProvider({ + getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), + }) + + await webviewMessageHandler(provider, { type }) + + expect(manager.getCurrentStatus).toHaveBeenCalledOnce() + expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ + type: "indexingStatusUpdate", + values: status, + }) + const [message] = vi.mocked(provider.postMessageToWebview).mock.calls[0] + expect(message.values).toBe(status) + }, + ) + + it.each(["resolve", "reject"] as const)("awaits stop completion before %s", async (outcome) => { + const stopping = deferred() + const entered = deferred() + const manager = createIndexManager() + manager.stopIndexing.mockImplementation(() => { + entered.resolve() + return stopping.promise + }) + const provider = createProvider({ + getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), + }) + let completed = false + const pending = webviewMessageHandler(provider, { type: "stopIndexing" }).then(() => { + completed = true }) + await entered.promise + await Promise.resolve() + await Promise.resolve() + + expect(completed).toBe(false) + expect(manager.stopIndexing).toHaveBeenCalledExactlyOnceWith() + expect(manager.getCurrentStatus).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + expect(provider.log).not.toHaveBeenCalled() + if (outcome === "reject") stopping.reject(new Error("stop failure")) + else stopping.resolve() + await pending + + expect(completed).toBe(true) + if (outcome === "reject") { + expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error stopping indexing: stop failure") + expect(manager.getCurrentStatus).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + } else { + expect(manager.getCurrentStatus).toHaveBeenCalledOnce() + expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ + type: "indexingStatusUpdate", + values: manager.getCurrentStatus.mock.results[0].value, + }) + expect(provider.log).not.toHaveBeenCalled() + } + }) + + it("starts once from an uninitialized error without separate initialization or recovery", async () => { + const manager = createIndexManager({ isInitialized: false, state: "Error" }) + manager.startIndexing.mockRejectedValue(new Error("startup failure")) const provider = createProvider({ getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), }) await webviewMessageHandler(provider, { type: "startIndexing" }) - await Promise.resolve() - expect(manager.startIndexing).toHaveBeenCalledTimes(2) - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: first failure") - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: second failure") + expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) + expect(manager.initialize).not.toHaveBeenCalled() + expect(manager.recoverFromError).not.toHaveBeenCalled() + expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error starting indexing: startup failure") }) - it("covers changed stop, toggle, and detached toggle rejection paths", async () => { + it("logs workspace-toggle startup failures without publishing a successful toggle status", async () => { const manager = createIndexManager({ startIndexing: vi.fn().mockRejectedValue(new Error("toggle failure")), }) @@ -6060,13 +6187,11 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { values: manager.getCurrentStatus(), }) await webviewMessageHandler(provider, { type: "toggleWorkspaceIndexing", bool: true }) - await Promise.resolve() - expect(manager.stopIndexing).toHaveBeenCalledOnce() - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: toggle failure") - expect(provider.postMessageToWebview).toHaveBeenCalledWith( - expect.objectContaining({ type: "indexingStatusUpdate" }), - ) + expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) + expect(manager.initialize).not.toHaveBeenCalled() + expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error toggling workspace indexing: toggle failure") + expect(provider.postMessageToWebview).toHaveBeenCalledOnce() }) it("does not toggle indexing or publish status without a workspace scope", async () => { @@ -6148,16 +6273,28 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.postMessageToWebview).not.toHaveBeenCalled() }) - it("catches auto-enabled indexing failures and posts the resulting status", async () => { + it.each([ + { enabled: true, outcome: "resolve" }, + { enabled: true, outcome: "reject" }, + { enabled: false, outcome: "resolve" }, + { enabled: false, outcome: "reject" }, + ] as const)("awaits auto-enable enabled=$enabled with $outcome", async ({ enabled, outcome }) => { const { CodeIndexManagerRegistry } = await import("../../../services/code-index/code-index-manager-registry") - let workspaceEnabled = false + const operation = deferred() + const entered = deferred() + let workspaceEnabled = !enabled const manager = createIndexManager({ setAutoEnableDefault: vi.fn().mockImplementation(async () => { - workspaceEnabled = true + workspaceEnabled = enabled }), - startIndexing: vi.fn().mockRejectedValue(new Error("auto-enable failure")), + }) + const indexingOperation = enabled ? manager.startIndexing : manager.stopIndexing + indexingOperation.mockImplementation(() => { + entered.resolve() + return operation.promise }) Object.defineProperty(manager, "isWorkspaceEnabled", { get: () => workspaceEnabled }) + // The registry double provides the enablement, startup, stop, and status members used by this handler. const getAllInstances = vi .spyOn(CodeIndexManagerRegistry, "getAllInstances") .mockReturnValue([manager] as unknown as ReturnType) @@ -6166,15 +6303,53 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { }) try { - await webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: true }) + let completed = false + const pending = webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: enabled }).then( + () => { + completed = true + }, + ) + await entered.promise + await Promise.resolve() await Promise.resolve() + expect(completed).toBe(false) expect(provider.getCurrentWorkspaceCodeIndexScope).toHaveBeenCalledOnce() - expect(manager.startIndexing).toHaveBeenCalledOnce() - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: auto-enable failure") - expect(provider.postMessageToWebview).toHaveBeenCalledWith( - expect.objectContaining({ type: "indexingStatusUpdate" }), - ) + expect(manager.setAutoEnableDefault).toHaveBeenCalledExactlyOnceWith(enabled) + expect(manager.initialize).not.toHaveBeenCalled() + expect(manager.recoverFromError).not.toHaveBeenCalled() + expect(manager.getCurrentStatus).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + if (enabled) { + expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) + expect(manager.stopIndexing).not.toHaveBeenCalled() + } else { + expect(manager.stopIndexing).toHaveBeenCalledExactlyOnceWith() + expect(manager.startIndexing).not.toHaveBeenCalled() + } + if (outcome === "reject") operation.reject(new Error("auto-enable failure")) + else operation.resolve() + await pending + + expect(completed).toBe(true) + expect(indexingOperation).toHaveBeenCalledOnce() + if (outcome === "reject") { + expect(provider.log).toHaveBeenCalledExactlyOnceWith( + enabled + ? "Error starting auto-enabled indexing: auto-enable failure" + : "Error setting auto-enable default: auto-enable failure", + ) + } else expect(provider.log).not.toHaveBeenCalled() + if (!enabled && outcome === "reject") { + expect(manager.getCurrentStatus).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + } else { + expect(manager.getCurrentStatus).toHaveBeenCalledOnce() + expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ + type: "indexingStatusUpdate", + values: manager.getCurrentStatus.mock.results[0].value, + }) + } } finally { getAllInstances.mockRestore() } diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index c5abbc0a12..943a298339 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3452,22 +3452,14 @@ export const webviewMessageHandler = async ( processedItems: 0, totalItems: 0, currentItemUnit: "items", - workerspacePath: undefined, + workspacePath: undefined, + failedFiles: [], }, }) return } - const status = manager - ? manager.getCurrentStatus() - : { - systemStatus: "Standby", - message: "No workspace folder open", - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", - workspacePath: undefined, - } + const status = manager.getCurrentStatus() await provider.postMessageToWebview({ type: "indexingStatusUpdate", @@ -3526,19 +3518,7 @@ export const webviewMessageHandler = async ( await manager.setWorkspaceEnabled(true) if (manager.isFeatureEnabled && manager.isFeatureConfigured) { - await manager.initialize(provider.contextProxy) - - const currentState = manager.state - if (currentState === "Standby" || currentState === "Error") { - void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) - - if (!manager.isInitialized) { - await manager.initialize(provider.contextProxy) - if (manager.state === "Standby" || manager.state === "Error") { - void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) - } - } - } + await manager.startIndexing(provider.contextProxy) } } catch (error) { provider.log(`Error starting indexing: ${error instanceof Error ? error.message : String(error)}`) @@ -3552,7 +3532,7 @@ export const webviewMessageHandler = async ( provider.log("Cannot stop indexing: No workspace folder open") return } - scope.codeIndexManager.stopIndexing() + await scope.codeIndexManager.stopIndexing() await provider.postMessageToWebview({ type: "indexingStatusUpdate", values: scope.codeIndexManager.getCurrentStatus(), @@ -3594,10 +3574,15 @@ export const webviewMessageHandler = async ( const wasEnabled = priorStates.get(m)! const isNowEnabled = m.isWorkspaceEnabled if (wasEnabled && !isNowEnabled) { - m.stopIndexing() + await m.stopIndexing() } else if (!wasEnabled && isNowEnabled && m.isFeatureEnabled && m.isFeatureConfigured) { - await m.initialize(provider.contextProxy) - void m.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + try { + await m.startIndexing(provider.contextProxy) + } catch (error) { + provider.log( + `Error starting auto-enabled indexing: ${error instanceof Error ? error.message : String(error)}`, + ) + } } } await provider.postMessageToWebview({ diff --git a/src/esbuild.mjs b/src/esbuild.mjs index 8159581f36..c824ef9c2d 100644 --- a/src/esbuild.mjs +++ b/src/esbuild.mjs @@ -134,7 +134,7 @@ async function main() { */ const workerConfig = { ...buildOptions, - entryPoints: ["workers/countTokens.ts"], + entryPoints: ["workers/countTokens.ts", "workers/codeIndexParse.ts"], outdir: "dist/workers", } diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index ae3d504003..7307fc29af 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1039,11 +1039,6 @@ "count": 43 } }, - "services/code-index/embedders/__tests__/bedrock.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 7 - } - }, "services/code-index/embedders/__tests__/gemini.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 4 @@ -1059,26 +1054,6 @@ "count": 1 } }, - "services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 15 - } - }, - "services/code-index/embedders/__tests__/openai-compatible.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 28 - } - }, - "services/code-index/embedders/__tests__/openai.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 10 - } - }, - "services/code-index/embedders/__tests__/openrouter.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 4 - } - }, "services/code-index/embedders/__tests__/vercel-ai-gateway.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 1 @@ -1086,7 +1061,7 @@ }, "services/code-index/embedders/bedrock.ts": { "@typescript-eslint/no-explicit-any": { - "count": 3 + "count": 2 } }, "services/code-index/embedders/ollama.ts": { @@ -1094,21 +1069,6 @@ "count": 4 } }, - "services/code-index/embedders/openai-compatible.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 1 - } - }, - "services/code-index/embedders/openai.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 1 - } - }, - "services/code-index/embedders/openrouter.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 3 - } - }, "services/code-index/interfaces/vector-store.ts": { "@typescript-eslint/no-explicit-any": { "count": 2 diff --git a/src/i18n/locales/ca/embeddings.json b/src/i18n/locales/ca/embeddings.json index 6955d53b46..6e856bfd19 100644 --- a/src/i18n/locales/ca/embeddings.json +++ b/src/i18n/locales/ca/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "El proveïdor de cerca ha limitat les sol·licituds. La indexació es reprèn automàticament quan acaba l’espera del proveïdor.", "authenticationFailed": "No s'han pogut crear les incrustacions: ha fallat l'autenticació. Comproveu la vostra clau d'API.", "failedWithStatus": "No s'han pogut crear les incrustacions després de {{attempts}} intents: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "No s'han pogut crear les incrustacions després de {{attempts}} intents: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "No es poden crear serveis: La indexació de codi no està configurada correctament" }, "orchestrator": { + "providerWaiting": "Esperant {{service}}; la indexació es reprèn automàticament", + "providerWaitingUntil": "Esperant {{service}}; la indexació es reprèn automàticament a les {{time}}", + "parserFailed": "La indexació del codi s'ha aturat en analitzar {{filePath}} ({{category}}). Corregeix el fitxer i inicia explícitament la indexació per tornar-ho a provar, o desactiva-la. No és un error del model ni del proveïdor.", + "indexingFilesRetrying": "S'estan reintentant els fitxers amb errors. Hi ha un índex parcial disponible.", + "indexingFilesFailed": "Encara hi ha errors en fitxers. No hi ha reintents automàtics pendents. Hi ha un índex parcial disponible.", + "indexingStopping": "S'està aturant la indexació...", "indexingFailedNoBlocks": "Indexació fallida: No s'ha indexat cap bloc de codi amb èxit. Això normalment indica un problema de configuració de l'embedder.", "fileWatcherStarted": "Monitor de fitxers iniciat.", "fileWatcherStopped": "Monitor de fitxers aturat.", diff --git a/src/i18n/locales/de/embeddings.json b/src/i18n/locales/de/embeddings.json index 8572e02863..87b9462f2d 100644 --- a/src/i18n/locales/de/embeddings.json +++ b/src/i18n/locales/de/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Der Suchanbieter hat die Anfragerate begrenzt. Die Indexierung wird nach der Wartezeit des Anbieters automatisch fortgesetzt.", "authenticationFailed": "Erstellung von Einbettungen fehlgeschlagen: Authentifizierung fehlgeschlagen. Bitte überprüfe deinen API-Schlüssel.", "failedWithStatus": "Erstellung von Einbettungen nach {{attempts}} Versuchen fehlgeschlagen: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Erstellung von Einbettungen nach {{attempts}} Versuchen fehlgeschlagen: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Kann keine Dienste erstellen: Code-Indizierung ist nicht richtig konfiguriert" }, "orchestrator": { + "providerWaiting": "Warten auf {{service}}; die Indexierung wird automatisch fortgesetzt", + "providerWaitingUntil": "Warten auf {{service}}; die Indexierung wird um {{time}} automatisch fortgesetzt", + "parserFailed": "Die Code-Indexierung wurde beim Parsen von {{filePath}} ({{category}}) gestoppt. Behebe den Fehler in der Datei und starte die Indexierung ausdrücklich erneut, oder deaktiviere sie. Dies ist kein Modell- oder Anbieterfehler.", + "indexingFilesRetrying": "Fehlgeschlagene Dateien werden erneut versucht. Ein Teilindex ist verfügbar.", + "indexingFilesFailed": "Es bestehen weiterhin Dateifehler. Keine automatischen Wiederholungen ausstehend. Ein Teilindex ist verfügbar.", + "indexingStopping": "Indexierung wird gestoppt...", "indexingFailedNoBlocks": "Indizierung fehlgeschlagen: Keine Code-Blöcke wurden erfolgreich indiziert. Dies deutet normalerweise auf ein Embedder-Konfigurationsproblem hin.", "fileWatcherStarted": "Datei-Watcher gestartet.", "fileWatcherStopped": "Datei-Watcher gestoppt.", diff --git a/src/i18n/locales/en/embeddings.json b/src/i18n/locales/en/embeddings.json index accd71fc7d..215e8a1334 100644 --- a/src/i18n/locales/en/embeddings.json +++ b/src/i18n/locales/en/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Search provider rate-limited. Indexing resumes automatically after the provider wait.", "authenticationFailed": "Failed to create embeddings: Authentication failed. Please check your API key.", "failedWithStatus": "Failed to create embeddings after {{attempts}} attempts: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Failed to create embeddings after {{attempts}} attempts: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Cannot create services: Code indexing is not properly configured" }, "orchestrator": { + "providerWaiting": "Waiting for {{service}}; indexing resumes automatically", + "providerWaitingUntil": "Waiting for {{service}}; indexing resumes automatically at {{time}}", + "parserFailed": "Code indexing stopped while parsing {{filePath}} ({{category}}). Fix the file and explicitly start indexing to retry, or disable indexing. This is not a model/provider error.", + "indexingFilesRetrying": "Retrying failed files. Partial index available.", + "indexingFilesFailed": "File failures remain. No automatic retries pending. Partial index available.", + "indexingStopping": "Stopping indexing...", "indexingFailedNoBlocks": "Indexing failed: No code blocks were successfully indexed. This usually indicates an embedder configuration issue.", "fileWatcherStarted": "File watcher started.", "fileWatcherStopped": "File watcher stopped.", diff --git a/src/i18n/locales/es/embeddings.json b/src/i18n/locales/es/embeddings.json index ea77d2d410..b4e49cc838 100644 --- a/src/i18n/locales/es/embeddings.json +++ b/src/i18n/locales/es/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "El proveedor de búsqueda ha limitado las solicitudes. La indexación se reanuda automáticamente cuando termina la espera del proveedor.", "authenticationFailed": "No se pudieron crear las incrustaciones: Error de autenticación. Comprueba tu clave de API.", "failedWithStatus": "No se pudieron crear las incrustaciones después de {{attempts}} intentos: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "No se pudieron crear las incrustaciones después de {{attempts}} intentos: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "No se pueden crear servicios: La indexación de código no está configurada correctamente" }, "orchestrator": { + "providerWaiting": "Esperando a {{service}}; la indexación se reanuda automáticamente", + "providerWaitingUntil": "Esperando a {{service}}; la indexación se reanuda automáticamente a las {{time}}", + "parserFailed": "La indexación del código se detuvo al analizar {{filePath}} ({{category}}). Corrige el archivo e inicia explícitamente la indexación para reintentarlo, o desactívala. No es un error del modelo ni del proveedor.", + "indexingFilesRetrying": "Reintentando los archivos con errores. Hay un índice parcial disponible.", + "indexingFilesFailed": "Siguen existiendo errores en archivos. No hay reintentos automáticos pendientes. Hay un índice parcial disponible.", + "indexingStopping": "Deteniendo la indexación...", "indexingFailedNoBlocks": "Indexación fallida: No se indexaron exitosamente bloques de código. Esto usualmente indica un problema de configuración del incrustador.", "fileWatcherStarted": "Monitor de archivos iniciado.", "fileWatcherStopped": "Monitor de archivos detenido.", diff --git a/src/i18n/locales/fr/embeddings.json b/src/i18n/locales/fr/embeddings.json index 187c7bac93..88e98e0411 100644 --- a/src/i18n/locales/fr/embeddings.json +++ b/src/i18n/locales/fr/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Le fournisseur de recherche a limité la fréquence des requêtes. L’indexation reprend automatiquement après le délai d’attente du fournisseur.", "authenticationFailed": "Échec de la création des embeddings : Échec de l'authentification. Veuillez vérifier votre clé API.", "failedWithStatus": "Échec de la création des embeddings après {{attempts}} tentatives : HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Échec de la création des embeddings après {{attempts}} tentatives : {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Impossible de créer les services : L'indexation du code n'est pas correctement configurée" }, "orchestrator": { + "providerWaiting": "En attente de {{service}} ; l’indexation reprend automatiquement", + "providerWaitingUntil": "En attente de {{service}} ; l’indexation reprend automatiquement à {{time}}", + "parserFailed": "L'indexation du code s'est arrêtée pendant l'analyse de {{filePath}} ({{category}}). Corrige le fichier et lance explicitement l'indexation pour réessayer, ou désactive-la. Ce n'est pas une erreur du modèle ni du fournisseur.", + "indexingFilesRetrying": "Nouvelle tentative pour les fichiers en échec. Un index partiel est disponible.", + "indexingFilesFailed": "Des fichiers restent en échec. Aucune nouvelle tentative automatique en attente. Un index partiel est disponible.", + "indexingStopping": "Arrêt de l'indexation en cours...", "indexingFailedNoBlocks": "Échec de l'indexation : Aucun bloc de code n'a été indexé avec succès. Cela indique généralement un problème de configuration de l'embedder.", "fileWatcherStarted": "Surveillant de fichiers démarré.", "fileWatcherStopped": "Surveillant de fichiers arrêté.", diff --git a/src/i18n/locales/hi/embeddings.json b/src/i18n/locales/hi/embeddings.json index aea009242e..f8883d91aa 100644 --- a/src/i18n/locales/hi/embeddings.json +++ b/src/i18n/locales/hi/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "खोज प्रदाता ने अनुरोधों की दर सीमित कर दी है। प्रदाता की प्रतीक्षा अवधि पूरी होने के बाद इंडेक्सिंग अपने आप फिर शुरू होगी।", "authenticationFailed": "एम्बेडिंग बनाने में विफल: प्रमाणीकरण विफल। कृपया अपनी एपीआई कुंजी जांचें।", "failedWithStatus": "{{attempts}} प्रयासों के बाद एम्बेडिंग बनाने में विफल: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "{{attempts}} प्रयासों के बाद एम्बेडिंग बनाने में विफल: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "सेवाएं नहीं बना सकते: कोड इंडेक्सिंग ठीक से कॉन्फ़िगर नहीं है" }, "orchestrator": { + "providerWaiting": "{{service}} की प्रतीक्षा है; इंडेक्सिंग अपने आप फिर शुरू होगी", + "providerWaitingUntil": "{{service}} की प्रतीक्षा है; इंडेक्सिंग {{time}} पर अपने आप फिर शुरू होगी", + "parserFailed": "{{filePath}} को पार्स करते समय कोड इंडेक्सिंग रुक गई ({{category}})। फ़ाइल ठीक करो और फिर से कोशिश करने के लिए खुद इंडेक्सिंग शुरू करो, या इंडेक्सिंग बंद कर दो। यह मॉडल या प्रदाता की त्रुटि नहीं है।", + "indexingFilesRetrying": "विफल फ़ाइलों के लिए फिर से कोशिश की जा रही है। आंशिक इंडेक्स उपलब्ध है।", + "indexingFilesFailed": "फ़ाइलों में विफलताएँ अभी भी हैं। कोई स्वचालित पुनः प्रयास लंबित नहीं है। आंशिक इंडेक्स उपलब्ध है।", + "indexingStopping": "इंडेक्सिंग रोकी जा रही है...", "indexingFailedNoBlocks": "इंडेक्सिंग असफल: कोई भी कोड ब्लॉक सफलतापूर्वक इंडेक्स नहीं हुआ। यह आमतौर पर एम्बेडर कॉन्फ़िगरेशन समस्या को दर्शाता है।", "fileWatcherStarted": "फाइल वॉचर शुरू हुआ।", "fileWatcherStopped": "फाइल वॉचर रुक गया।", diff --git a/src/i18n/locales/id/embeddings.json b/src/i18n/locales/id/embeddings.json index bba797d3e5..ada4ea3f68 100644 --- a/src/i18n/locales/id/embeddings.json +++ b/src/i18n/locales/id/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Penyedia pencarian membatasi laju permintaan. Pengindeksan dilanjutkan secara otomatis setelah waktu tunggu penyedia berakhir.", "authenticationFailed": "Gagal membuat embeddings: Autentikasi gagal. Silakan periksa API key Anda.", "failedWithStatus": "Gagal membuat embeddings setelah {{attempts}} percobaan: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Gagal membuat embeddings setelah {{attempts}} percobaan: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Tidak dapat membuat layanan: Pengindeksan kode tidak dikonfigurasi dengan benar" }, "orchestrator": { + "providerWaiting": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis", + "providerWaitingUntil": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis pada {{time}}", + "parserFailed": "Pengindeksan kode berhenti saat mengurai {{filePath}} ({{category}}). Perbaiki file dan mulai pengindeksan secara manual untuk mencoba lagi, atau nonaktifkan pengindeksan. Ini bukan kesalahan model atau provider.", + "indexingFilesRetrying": "Mencoba ulang file yang gagal. Indeks parsial tersedia.", + "indexingFilesFailed": "Masih ada kegagalan file. Tidak ada percobaan ulang otomatis yang tertunda. Indeks parsial tersedia.", + "indexingStopping": "Menghentikan pengindeksan...", "indexingFailedNoBlocks": "Pengindeksan gagal: Tidak ada blok kode yang berhasil diindeks. Ini biasanya menunjukkan masalah konfigurasi embedder.", "fileWatcherStarted": "Pemantau file dimulai.", "fileWatcherStopped": "Pemantau file dihentikan.", diff --git a/src/i18n/locales/it/embeddings.json b/src/i18n/locales/it/embeddings.json index 5b517df528..7e6505e834 100644 --- a/src/i18n/locales/it/embeddings.json +++ b/src/i18n/locales/it/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Il provider di ricerca ha limitato la frequenza delle richieste. L’indicizzazione riprende automaticamente al termine dell’attesa del provider.", "authenticationFailed": "Creazione degli embedding non riuscita: Autenticazione fallita. Controlla la tua chiave API.", "failedWithStatus": "Creazione degli embedding non riuscita dopo {{attempts}} tentativi: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Creazione degli embedding non riuscita dopo {{attempts}} tentativi: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Impossibile creare i servizi: L'indicizzazione del codice non è configurata correttamente" }, "orchestrator": { + "providerWaiting": "In attesa di {{service}}; l’indicizzazione riprende automaticamente", + "providerWaitingUntil": "In attesa di {{service}}; l’indicizzazione riprende automaticamente alle {{time}}", + "parserFailed": "L'indicizzazione del codice si è fermata durante l'analisi di {{filePath}} ({{category}}). Correggi il file e avvia esplicitamente l'indicizzazione per riprovare, oppure disattivala. Non è un errore del modello o del provider.", + "indexingFilesRetrying": "Nuovo tentativo per i file non riusciti. È disponibile un indice parziale.", + "indexingFilesFailed": "Persistono errori nei file. Nessun nuovo tentativo automatico in attesa. È disponibile un indice parziale.", + "indexingStopping": "Arresto dell'indicizzazione in corso...", "indexingFailedNoBlocks": "Indicizzazione fallita: Nessun blocco di codice è stato indicizzato con successo. Questo di solito indica un problema di configurazione dell'embedder.", "fileWatcherStarted": "Monitoraggio file avviato.", "fileWatcherStopped": "Monitoraggio file fermato.", diff --git a/src/i18n/locales/ja/embeddings.json b/src/i18n/locales/ja/embeddings.json index b90d9a0d52..d8bcf9f32b 100644 --- a/src/i18n/locales/ja/embeddings.json +++ b/src/i18n/locales/ja/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "検索プロバイダーのレート制限に達しました。プロバイダーの待機時間が終了すると、インデックス作成が自動的に再開されます。", "authenticationFailed": "埋め込みの作成に失敗しました：認証に失敗しました。APIキーを確認してください。", "failedWithStatus": "{{attempts}}回試行しましたが、埋め込みの作成に失敗しました：HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "{{attempts}}回試行しましたが、埋め込みの作成に失敗しました：{{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "サービスを作成できません: コードインデックスが正しく設定されていません" }, "orchestrator": { + "providerWaiting": "{{service}} を待機中です。インデックス作成は自動的に再開されます", + "providerWaitingUntil": "{{service}} を待機中です。インデックス作成は {{time}} に自動的に再開されます", + "parserFailed": "{{filePath}} の解析中にコードのインデックス作成が停止しました（{{category}}）。ファイルを修正し、インデックス作成を明示的に開始して再試行するか、インデックス作成を無効にしてください。モデルやプロバイダーのエラーではありません。", + "indexingFilesRetrying": "失敗したファイルを再試行しています。部分的なインデックスを利用できます。", + "indexingFilesFailed": "ファイルのエラーが残っています。自動再試行の予定はありません。部分的なインデックスを利用できます。", + "indexingStopping": "インデックス作成を停止中...", "indexingFailedNoBlocks": "インデックス作成に失敗しました：コードブロックが正常にインデックス化されませんでした。これは通常、エンベッダーの設定問題を示しています。", "fileWatcherStarted": "ファイルウォッチャーが開始されました。", "fileWatcherStopped": "ファイルウォッチャーが停止されました。", diff --git a/src/i18n/locales/ko/embeddings.json b/src/i18n/locales/ko/embeddings.json index 7307c91c82..de01158384 100644 --- a/src/i18n/locales/ko/embeddings.json +++ b/src/i18n/locales/ko/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "검색 공급자의 요청 속도 제한에 도달했습니다. 공급자의 대기 시간이 끝나면 인덱싱이 자동으로 재개됩니다.", "authenticationFailed": "임베딩 생성 실패: 인증에 실패했습니다. API 키를 확인하세요.", "failedWithStatus": "{{attempts}}번 시도 후 임베딩 생성 실패: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "{{attempts}}번 시도 후 임베딩 생성 실패: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "서비스를 생성할 수 없습니다: 코드 인덱싱이 올바르게 구성되지 않았습니다" }, "orchestrator": { + "providerWaiting": "{{service}} 대기 중; 인덱싱이 자동으로 재개됩니다", + "providerWaitingUntil": "{{service}} 대기 중; {{time}}에 인덱싱이 자동으로 재개됩니다", + "parserFailed": "{{filePath}} 파싱 중 코드 인덱싱이 중지되었습니다({{category}}). 파일을 수정하고 인덱싱을 직접 시작해 다시 시도하거나 인덱싱을 비활성화하세요. 모델 또는 제공자 오류가 아닙니다.", + "indexingFilesRetrying": "실패한 파일을 다시 시도하고 있습니다. 부분 인덱스를 사용할 수 있습니다.", + "indexingFilesFailed": "파일 오류가 남아 있습니다. 예정된 자동 재시도가 없습니다. 부분 인덱스를 사용할 수 있습니다.", + "indexingStopping": "인덱싱 중지 중...", "indexingFailedNoBlocks": "인덱싱 실패: 코드 블록이 성공적으로 인덱싱되지 않았습니다. 이는 일반적으로 임베더 구성 문제를 나타냅니다.", "fileWatcherStarted": "파일 감시자가 시작되었습니다.", "fileWatcherStopped": "파일 감시자가 중지되었습니다.", diff --git a/src/i18n/locales/nl/embeddings.json b/src/i18n/locales/nl/embeddings.json index f78c927e3a..4beceff5ba 100644 --- a/src/i18n/locales/nl/embeddings.json +++ b/src/i18n/locales/nl/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "De zoekprovider heeft het aantal aanvragen beperkt. De indexering wordt automatisch hervat na de wachttijd van de provider.", "authenticationFailed": "Insluitingen maken mislukt: Authenticatie mislukt. Controleer je API-sleutel.", "failedWithStatus": "Insluitingen maken mislukt na {{attempts}} pogingen: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Insluitingen maken mislukt na {{attempts}} pogingen: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Kan geen services maken: Code-indexering is niet correct geconfigureerd" }, "orchestrator": { + "providerWaiting": "Wachten op {{service}}; de indexering wordt automatisch hervat", + "providerWaitingUntil": "Wachten op {{service}}; de indexering wordt automatisch hervat om {{time}}", + "parserFailed": "Code-indexering is gestopt tijdens het parseren van {{filePath}} ({{category}}). Herstel het bestand en start de indexering expliciet om het opnieuw te proberen, of schakel indexering uit. Dit is geen model- of providerfout.", + "indexingFilesRetrying": "Mislukte bestanden worden opnieuw geprobeerd. Een gedeeltelijke index is beschikbaar.", + "indexingFilesFailed": "Er zijn nog bestandsfouten. Geen automatische nieuwe pogingen in afwachting. Een gedeeltelijke index is beschikbaar.", + "indexingStopping": "Indexering wordt gestopt...", "indexingFailedNoBlocks": "Indexering mislukt: Geen codeblokken werden succesvol geïndexeerd. Dit duidt meestal op een embedder configuratieprobleem.", "fileWatcherStarted": "Bestandsmonitor gestart.", "fileWatcherStopped": "Bestandsmonitor gestopt.", diff --git a/src/i18n/locales/pl/embeddings.json b/src/i18n/locales/pl/embeddings.json index 3f81081238..1ab81cbafe 100644 --- a/src/i18n/locales/pl/embeddings.json +++ b/src/i18n/locales/pl/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Dostawca wyszukiwania ograniczył częstotliwość żądań. Indeksowanie zostanie automatycznie wznowione po upływie czasu oczekiwania dostawcy.", "authenticationFailed": "Nie udało się utworzyć osadzeń: Uwierzytelnianie nie powiodło się. Sprawdź swój klucz API.", "failedWithStatus": "Nie udało się utworzyć osadzeń po {{attempts}} próbach: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Nie udało się utworzyć osadzeń po {{attempts}} próbach: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Nie można utworzyć usług: Indeksowanie kodu nie jest poprawnie skonfigurowane" }, "orchestrator": { + "providerWaiting": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione", + "providerWaitingUntil": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione o {{time}}", + "parserFailed": "Indeksowanie kodu zatrzymało się podczas parsowania {{filePath}} ({{category}}). Popraw plik i ręcznie uruchom indeksowanie, aby spróbować ponownie, lub wyłącz indeksowanie. To nie jest błąd modelu ani dostawcy.", + "indexingFilesRetrying": "Ponawianie prób dla plików z błędami. Dostępny jest częściowy indeks.", + "indexingFilesFailed": "Nadal występują błędy plików. Brak oczekujących automatycznych ponowień. Dostępny jest częściowy indeks.", + "indexingStopping": "Zatrzymywanie indeksowania...", "indexingFailedNoBlocks": "Indeksowanie nie powiodło się: Żadne bloki kodu nie zostały pomyślnie zaindeksowane. To zwykle wskazuje na problem z konfiguracją embeddera.", "fileWatcherStarted": "Monitor plików uruchomiony.", "fileWatcherStopped": "Monitor plików zatrzymany.", diff --git a/src/i18n/locales/pt-BR/embeddings.json b/src/i18n/locales/pt-BR/embeddings.json index fa9be50a42..aeba6c2f34 100644 --- a/src/i18n/locales/pt-BR/embeddings.json +++ b/src/i18n/locales/pt-BR/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "O provedor de pesquisa limitou a taxa de solicitações. A indexação é retomada automaticamente após o tempo de espera do provedor.", "authenticationFailed": "Falha ao criar embeddings: Falha na autenticação. Verifique sua chave de API.", "failedWithStatus": "Falha ao criar embeddings após {{attempts}} tentativas: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Falha ao criar embeddings após {{attempts}} tentativas: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Não é possível criar serviços: A indexação de código não está configurada corretamente" }, "orchestrator": { + "providerWaiting": "Aguardando {{service}}; a indexação é retomada automaticamente", + "providerWaitingUntil": "Aguardando {{service}}; a indexação é retomada automaticamente às {{time}}", + "parserFailed": "A indexação do código parou durante a análise de {{filePath}} ({{category}}). Corrige o arquivo e inicia explicitamente a indexação para tentar de novo, ou desativa a indexação. Não é um erro do modelo nem do provedor.", + "indexingFilesRetrying": "Tentando novamente os arquivos com falhas. Um índice parcial está disponível.", + "indexingFilesFailed": "Ainda há falhas em arquivos. Nenhuma nova tentativa automática pendente. Um índice parcial está disponível.", + "indexingStopping": "Parando a indexação...", "indexingFailedNoBlocks": "Indexação falhou: Nenhum bloco de código foi indexado com sucesso. Isso geralmente indica um problema de configuração do embedder.", "fileWatcherStarted": "Monitor de arquivos iniciado.", "fileWatcherStopped": "Monitor de arquivos parado.", diff --git a/src/i18n/locales/ru/embeddings.json b/src/i18n/locales/ru/embeddings.json index 0a25f7cfda..ee1b7aed0f 100644 --- a/src/i18n/locales/ru/embeddings.json +++ b/src/i18n/locales/ru/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Поисковый провайдер ограничил частоту запросов. Индексация возобновится автоматически после окончания периода ожидания провайдера.", "authenticationFailed": "Не удалось создать вложения: Ошибка аутентификации. Проверьте свой ключ API.", "failedWithStatus": "Не удалось создать вложения после {{attempts}} попыток: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Не удалось создать вложения после {{attempts}} попыток: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Невозможно создать сервисы: Индексация кода не настроена должным образом" }, "orchestrator": { + "providerWaiting": "Ожидание {{service}}; индексация возобновится автоматически", + "providerWaitingUntil": "Ожидание {{service}}; индексация возобновится автоматически в {{time}}", + "parserFailed": "Индексация кода остановлена при разборе {{filePath}} ({{category}}). Исправь файл и явно запусти индексацию для повторной попытки или отключи её. Это не ошибка модели или провайдера.", + "indexingFilesRetrying": "Повторные попытки для файлов с ошибками. Доступен частичный индекс.", + "indexingFilesFailed": "Остались ошибки файлов. Автоматических повторных попыток не ожидается. Доступен частичный индекс.", + "indexingStopping": "Остановка индексации...", "indexingFailedNoBlocks": "Индексация не удалась: Ни один блок кода не был успешно проиндексирован. Это обычно указывает на проблему конфигурации эмбеддера.", "fileWatcherStarted": "Наблюдатель файлов запущен.", "fileWatcherStopped": "Наблюдатель файлов остановлен.", diff --git a/src/i18n/locales/tr/embeddings.json b/src/i18n/locales/tr/embeddings.json index 5202ccd220..0c62e2f6a4 100644 --- a/src/i18n/locales/tr/embeddings.json +++ b/src/i18n/locales/tr/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Arama sağlayıcısının istek hızı sınırına ulaşıldı. Sağlayıcının bekleme süresi dolunca dizinleme otomatik olarak devam eder.", "authenticationFailed": "Gömülmeler oluşturulamadı: Kimlik doğrulama başarısız oldu. Lütfen API anahtarınızı kontrol edin.", "failedWithStatus": "{{attempts}} denemeden sonra gömülmeler oluşturulamadı: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "{{attempts}} denemeden sonra gömülmeler oluşturulamadı: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Hizmetler oluşturulamıyor: Kod indeksleme düzgün yapılandırılmamış" }, "orchestrator": { + "providerWaiting": "{{service}} bekleniyor; dizinleme otomatik olarak devam eder", + "providerWaitingUntil": "{{service}} bekleniyor; dizinleme {{time}} zamanında otomatik olarak devam eder", + "parserFailed": "{{filePath}} ayrıştırılırken kod indeksleme durdu ({{category}}). Dosyayı düzelt ve yeniden denemek için indekslemeyi açıkça başlat ya da indekslemeyi devre dışı bırak. Bu bir model veya sağlayıcı hatası değil.", + "indexingFilesRetrying": "Başarısız dosyalar yeniden deneniyor. Kısmi indeks kullanılabilir.", + "indexingFilesFailed": "Dosya hataları devam ediyor. Bekleyen otomatik yeniden deneme yok. Kısmi indeks kullanılabilir.", + "indexingStopping": "İndeksleme durduruluyor...", "indexingFailedNoBlocks": "İndeksleme başarısız: Hiçbir kod bloğu başarıyla indekslenemedi. Bu genellikle bir embedder yapılandırma sorunu olduğunu gösterir.", "fileWatcherStarted": "Dosya izleyici başlatıldı.", "fileWatcherStopped": "Dosya izleyici durduruldu.", diff --git a/src/i18n/locales/vi/embeddings.json b/src/i18n/locales/vi/embeddings.json index e33413b023..44f624778c 100644 --- a/src/i18n/locales/vi/embeddings.json +++ b/src/i18n/locales/vi/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "Nhà cung cấp tìm kiếm đã giới hạn tốc độ yêu cầu. Việc lập chỉ mục tự động tiếp tục sau khi hết thời gian chờ của nhà cung cấp.", "authenticationFailed": "Không thể tạo nhúng: Xác thực không thành công. Vui lòng kiểm tra khóa API của bạn.", "failedWithStatus": "Không thể tạo nhúng sau {{attempts}} lần thử: HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "Không thể tạo nhúng sau {{attempts}} lần thử: {{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "Không thể tạo dịch vụ: Lập chỉ mục mã không được cấu hình đúng cách" }, "orchestrator": { + "providerWaiting": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục", + "providerWaitingUntil": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục lúc {{time}}", + "parserFailed": "Đã dừng lập chỉ mục mã khi phân tích {{filePath}} ({{category}}). Sửa tệp rồi chủ động bắt đầu lập chỉ mục để thử lại, hoặc tắt lập chỉ mục. Đây không phải lỗi mô hình hay nhà cung cấp.", + "indexingFilesRetrying": "Đang thử lại các tệp bị lỗi. Có thể dùng chỉ mục một phần.", + "indexingFilesFailed": "Vẫn còn tệp bị lỗi. Không có lần thử lại tự động nào đang chờ. Có thể dùng chỉ mục một phần.", + "indexingStopping": "Đang dừng lập chỉ mục...", "indexingFailedNoBlocks": "Lập chỉ mục thất bại: Không có khối mã nào được lập chỉ mục thành công. Điều này thường cho thấy vấn đề cấu hình embedder.", "fileWatcherStarted": "Trình theo dõi tệp đã khởi động.", "fileWatcherStopped": "Trình theo dõi tệp đã dừng.", diff --git a/src/i18n/locales/zh-CN/embeddings.json b/src/i18n/locales/zh-CN/embeddings.json index f511bc67c4..053256c17f 100644 --- a/src/i18n/locales/zh-CN/embeddings.json +++ b/src/i18n/locales/zh-CN/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "搜索提供商已限制请求速率。提供商的等待时间结束后，索引将自动恢复。", "authenticationFailed": "创建嵌入失败：身份验证失败。请检查您的 API 密钥。", "failedWithStatus": "尝试 {{attempts}} 次后创建嵌入失败：HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "尝试 {{attempts}} 次后创建嵌入失败：{{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "无法创建服务：代码索引未正确配置" }, "orchestrator": { + "providerWaiting": "正在等待 {{service}}；索引将自动恢复", + "providerWaitingUntil": "正在等待 {{service}}；索引将于 {{time}} 自动恢复", + "parserFailed": "解析 {{filePath}} 时，代码索引已停止（{{category}}）。修复文件后，手动启动索引以重试，或禁用索引。这不是模型或提供商错误。", + "indexingFilesRetrying": "正在重试失败的文件。部分索引可用。", + "indexingFilesFailed": "仍有文件失败。没有待执行的自动重试。部分索引可用。", + "indexingStopping": "正在停止索引...", "indexingFailedNoBlocks": "索引失败：没有代码块被成功索引。这通常表示 Embedder 配置问题。", "fileWatcherStarted": "文件监控已启动。", "fileWatcherStopped": "文件监控已停止。", diff --git a/src/i18n/locales/zh-TW/embeddings.json b/src/i18n/locales/zh-TW/embeddings.json index 6adef3c9de..d308f1f3f7 100644 --- a/src/i18n/locales/zh-TW/embeddings.json +++ b/src/i18n/locales/zh-TW/embeddings.json @@ -1,4 +1,5 @@ { + "searchProviderRateLimited": "搜尋供應商已限制請求速率。供應商的等待時間結束後，索引將自動恢復。", "authenticationFailed": "建立內嵌失敗：驗證失敗。請檢查您的 API 金鑰。", "failedWithStatus": "嘗試 {{attempts}} 次後建立內嵌失敗：HTTP {{statusCode}} - {{errorMessage}}", "failedWithError": "嘗試 {{attempts}} 次後建立內嵌失敗：{{errorMessage}}", @@ -57,6 +58,12 @@ "codeIndexingNotConfigured": "無法建立服務：程式碼索引未正確設定" }, "orchestrator": { + "providerWaiting": "正在等待 {{service}}；索引將自動恢復", + "providerWaitingUntil": "正在等待 {{service}}；索引將於 {{time}} 自動恢復", + "parserFailed": "剖析 {{filePath}} 時，程式碼索引已停止（{{category}}）。修正檔案後，手動啟動索引以重試，或停用索引。這不是模型或供應商錯誤。", + "indexingFilesRetrying": "正在重試失敗的檔案。部分索引可供使用。", + "indexingFilesFailed": "仍有檔案失敗。沒有待執行的自動重試。部分索引可供使用。", + "indexingStopping": "正在停止索引...", "indexingFailedNoBlocks": "索引失敗：沒有程式碼區塊被成功索引。這通常表示 Embedder 設定問題。", "fileWatcherStarted": "檔案監控已啟動。", "fileWatcherStopped": "檔案監控已停止。", diff --git a/src/package.json b/src/package.json index 6c59828b1d..98578ff92a 100644 --- a/src/package.json +++ b/src/package.json @@ -493,7 +493,7 @@ "@lmstudio/sdk": "^1.1.1", "@mistralai/mistralai": "^1.9.18", "@modelcontextprotocol/sdk": "1.32.2", - "@qdrant/js-client-rest": "^1.14.0", + "@qdrant/js-client-rest": "1.20.0", "@roo-code/cloud": "workspace:^", "@roo-code/core": "workspace:^", "@roo-code/ipc": "workspace:^", diff --git a/src/services/code-index/__tests__/cache-manager.spec.ts b/src/services/code-index/__tests__/cache-manager.spec.ts index 77bd5be198..6831400424 100644 --- a/src/services/code-index/__tests__/cache-manager.spec.ts +++ b/src/services/code-index/__tests__/cache-manager.spec.ts @@ -1,202 +1,278 @@ -import type { Mock } from "vitest" import * as vscode from "vscode" import { createHash } from "crypto" -import debounce from "lodash.debounce" import { CacheManager } from "../cache-manager" - -import { clearAllMocks } from "../../../test-utils/reset" - -// Mock safeWriteJson utility -vitest.mock("../../../utils/safeWriteJson", () => ({ - safeWriteJson: vitest.fn().mockResolvedValue(undefined), -})) - -// Import the mocked version import { safeWriteJson } from "../../../utils/safeWriteJson" +import { codeIndexLog } from "../code-index-log" -// Mock vscode -vitest.mock("vscode", () => ({ - Uri: { - joinPath: vitest.fn(), - }, - workspace: { - fs: { - readFile: vitest.fn(), - writeFile: vitest.fn(), - delete: vitest.fn(), - }, - }, -})) - -// Mock debounce to execute immediately -vitest.mock("lodash.debounce", () => ({ - default: vitest.fn(function (fn) { - return fn - }), +vi.mock("../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn() })) +vi.mock("../code-index-log", () => ({ codeIndexLog: { error: vi.fn() } })) +vi.mock("vscode", () => ({ + Uri: { joinPath: vi.fn() }, + workspace: { fs: { readFile: vi.fn() } }, })) -// Mock TelemetryService -vitest.mock("@roo-code/telemetry", () => ({ - TelemetryService: { - instance: { - captureEvent: vitest.fn(), - }, - }, -})) +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} describe("CacheManager", () => { - let mockContext: vscode.ExtensionContext - let mockWorkspacePath: string - let mockCachePath: vscode.Uri - let cacheManager: CacheManager + const cachePath = { fsPath: "/storage/cache.json" } as vscode.Uri + // CacheManager uses only globalStorageUri from the extension context. + const context = { globalStorageUri: { fsPath: "/storage" } } as vscode.ExtensionContext + let cache: CacheManager + let persisted: unknown - beforeEach(() => { - // Reset all mocks - clearAllMocks() + function freshCache() { + return new CacheManager(context, "/workspace") + } - // Mock context - mockWorkspacePath = "/mock/workspace" - mockCachePath = { fsPath: "/mock/storage/cache.json" } as vscode.Uri - mockContext = { - globalStorageUri: { fsPath: "/mock/storage" } as vscode.Uri, - } as vscode.ExtensionContext - - // Mock Uri.joinPath - ;(vscode.Uri.joinPath as Mock).mockReturnValue(mockCachePath) - - // Create cache manager instance - cacheManager = new CacheManager(mockContext, mockWorkspacePath) - }) - - describe("constructor", () => { - it("should correctly set up cachePath using Uri.joinPath and crypto.createHash", () => { - const expectedHash = createHash("sha256").update(mockWorkspacePath).digest("hex") - - expect(vscode.Uri.joinPath).toHaveBeenCalledWith( - mockContext.globalStorageUri, - `roo-index-cache-${expectedHash}.json`, - ) + beforeEach(() => { + vi.useFakeTimers() + vi.resetAllMocks() + persisted = undefined + vi.mocked(vscode.Uri.joinPath).mockReturnValue(cachePath) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async () => { + if (persisted === undefined) throw new Error("Missing cache") + return Buffer.from(JSON.stringify(persisted)) }) - - it("should set up debounced save function", () => { - expect(debounce).toHaveBeenCalledWith(expect.any(Function), 1500) + vi.mocked(safeWriteJson).mockImplementation(async (_path, data: unknown) => { + persisted = structuredClone(data) }) + cache = freshCache() }) - describe("initialize", () => { - it("should load existing cache file successfully", async () => { - const mockCache = { "file1.ts": "hash1", "file2.ts": "hash2" } - const mockBuffer = Buffer.from(JSON.stringify(mockCache)) - ;(vscode.workspace.fs.readFile as Mock).mockResolvedValue(mockBuffer) - - await cacheManager.initialize() - - expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(mockCachePath) - expect(cacheManager.getAllHashes()).toEqual(mockCache) - }) - - it("should handle missing cache file by creating empty cache", async () => { - ;(vscode.workspace.fs.readFile as Mock).mockRejectedValue(new Error("File not found")) - - await cacheManager.initialize() - - expect(cacheManager.getAllHashes()).toEqual({}) - }) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() }) - describe("hash management", () => { - it("should update hash and trigger save", () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - - expect(cacheManager.getHash(filePath)).toBe(hash) - expect(safeWriteJson).toHaveBeenCalled() - }) - - it("should delete hash and trigger save", () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - cacheManager.deleteHash(filePath) + it("uses Zoo's workspace-specific hashed cache path", () => { + const workspaceHash = createHash("sha256").update("/workspace").digest("hex") + expect(vscode.Uri.joinPath).toHaveBeenCalledWith( + context.globalStorageUri, + `roo-index-cache-${workspaceHash}.json`, + ) + }) - expect(cacheManager.getHash(filePath)).toBeUndefined() - expect(safeWriteJson).toHaveBeenCalled() - }) + it("loads legacy hashes without inventing a generation watermark or changing path keys", async () => { + persisted = { "/workspace/file.ts": "hash", "relative.ts": "relative-hash" } + await cache.initialize() + expect(cache.loadSucceeded).toBe(true) + expect(cache.getAllHashes()).toEqual(persisted) + expect(cache.getHash("/workspace/file.ts")).toBe("hash") + expect(cache.getHash("file.ts")).toBeUndefined() + expect(cache.generationWatermark).toBeUndefined() + }) - it("should return shallow copy of hashes", () => { - const filePath = "test.ts" - const hash = "testhash" + it("loads open records ahead of stale hashes and returns independent snapshots", async () => { + persisted = { + version: 2, + fileHashes: { "open.ts": "stale", "complete.ts": "hash" }, + openFiles: { "open.ts": true, "deleted.ts": true }, + generationWatermark: 2048, + } + vi.setSystemTime(12345) + await cache.initialize() + expect(cache.loadedAt).toBe(12345) + expect(cache.loadSucceeded).toBe(true) + expect(cache.generationWatermark).toBe(2048) + expect(cache.getHash("open.ts")).toBeUndefined() + expect(cache.getTrackedPaths().sort()).toEqual(["complete.ts", "deleted.ts", "open.ts"]) + const openFiles = cache.getOpenFiles() + delete openFiles["deleted.ts"] + const hashes = cache.getAllHashes() + hashes["complete.ts"] = "changed" + expect(cache.getOpenFiles()).toEqual({ "open.ts": true, "deleted.ts": true }) + expect(cache.getHash("complete.ts")).toBe("hash") + }) - cacheManager.updateHash(filePath, hash) - const hashes = cacheManager.getAllHashes() + it.each([ + undefined, + "invalid JSON", + null, + [], + { version: 2, fileHashes: { "file.ts": 4 }, openFiles: {} }, + { version: 2, fileHashes: {}, openFiles: { "file.ts": false } }, + { version: 2, fileHashes: [], openFiles: {} }, + { version: 2, fileHashes: {}, openFiles: [] }, + { version: 2, fileHashes: {} }, + { "legacy.ts": 42 }, + ])("resets unreadable or invalid cache state (%j)", async (data: unknown) => { + cache.completeFile("previous.ts", "old") + persisted = data + if (data === "invalid JSON") { + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValueOnce(Buffer.from(data)) + } + vi.setSystemTime(54321) + await cache.initialize() + expect(cache.loadedAt).toBe(54321) + expect(cache.loadSucceeded).toBe(false) + expect(cache.getAllHashes()).toEqual({}) + expect(cache.getOpenFiles()).toEqual({}) + expect(cache.generationWatermark).toBeUndefined() + }) - // Modify the returned object - hashes[filePath] = "modified" + it.each([0, 2048, Number.MAX_SAFE_INTEGER])("loads a valid watermark %s", async (watermark) => { + persisted = { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: watermark } + await cache.initialize() + expect(cache.loadSucceeded).toBe(true) + expect(cache.generationWatermark).toBe(watermark) + }) - // Original should remain unchanged - expect(cacheManager.getHash(filePath)).toBe(hash) + it.each([-1, 0.5, "2048", Number.MAX_SAFE_INTEGER + 1, null])( + "does not trust an invalid watermark %j", + async (watermark) => { + persisted = { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: watermark } + await cache.initialize() + expect(cache.loadSucceeded).toBe(true) + expect(cache.generationWatermark).toBeUndefined() + }, + ) + + it("awaits durable invalidation and reloads an unfinished file without a hash", async () => { + cache.completeFile("file.ts", "hash") + await cache.flush() + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { + await held.promise + persisted = structuredClone(data) }) + const saved = vi.fn() + const invalidation = cache.invalidateFile("file.ts").then(saved) + await vi.advanceTimersByTimeAsync(0) + expect(saved).not.toHaveBeenCalled() + expect(cache.getHash("file.ts")).toBeUndefined() + held.resolve() + await invalidation + const reloaded = freshCache() + await reloaded.initialize() + expect(reloaded.getHash("file.ts")).toBeUndefined() + expect(reloaded.getOpenFiles()).toEqual({ "file.ts": true }) }) - describe("saving", () => { - it("should save cache to disk with correct data", async () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - - expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, expect.any(Object)) - - // Verify the saved data - const savedData = (safeWriteJson as Mock).mock.calls[0][1] - expect(savedData).toEqual({ [filePath]: hash }) + it("serializes an already-issued older snapshot before durable invalidation", async () => { + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { + await held.promise + persisted = structuredClone(data) }) + cache.completeFile("file.ts", "old") + await vi.advanceTimersByTimeAsync(1500) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + const invalidation = cache.invalidateFile("file.ts") + await vi.advanceTimersByTimeAsync(0) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + held.resolve() + await invalidation + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) + await vi.advanceTimersByTimeAsync(1500) + expect(safeWriteJson).toHaveBeenCalledTimes(2) + }) - it("should handle save errors gracefully", async () => { - const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(function () {}) - ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) - - cacheManager.updateHash("test.ts", "hash") - - // Wait for any pending promises - await new Promise((resolve) => setTimeout(resolve, 0)) - - expect(consoleErrorSpy).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) + it("propagates durable-save failure, retains the open record, and allows a later save", async () => { + cache.completeFile("file.ts", "hash") + const failure = new Error("Disk full") + vi.mocked(safeWriteJson).mockRejectedValueOnce(failure) + await expect(cache.invalidateFile("file.ts")).rejects.toBe(failure) + expect(cache.getHash("file.ts")).toBeUndefined() + expect(cache.getOpenFiles()).toEqual({ "file.ts": true }) + await cache.flush() + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) + }) - consoleErrorSpy.mockRestore() + it("debounces completion and removes the open record in the same snapshot as its hash", async () => { + await cache.invalidateFile("file.ts") + vi.mocked(safeWriteJson).mockClear() + cache.completeFile("file.ts", "current") + expect(safeWriteJson).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1499) + expect(safeWriteJson).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(safeWriteJson).toHaveBeenCalledWith(cachePath.fsPath, { + version: 2, + fileHashes: { "file.ts": "current" }, + openFiles: {}, + generationWatermark: undefined, }) }) - describe("clearCacheFile", () => { - it("should clear cache file and reset state", async () => { - cacheManager.updateHash("test.ts", "hash") - - // Reset the mock to ensure safeWriteJson succeeds for clearCacheFile - ;(safeWriteJson as Mock).mockClear() - ;(safeWriteJson as Mock).mockResolvedValue(undefined) + it("tracks completed and open files until removal without enumerating paths", async () => { + const enumerate = vi.spyOn(cache, "getTrackedPaths") + cache.completeFile("file.ts", "hash") + expect(cache.isTracked("file.ts")).toBe(true) + expect(cache.isTracked("missing.ts")).toBe(false) + expect(cache.isTracked("toString")).toBe(false) + expect(cache.getHash("toString")).toBeUndefined() + await cache.invalidateFile("file.ts") + expect(cache.isTracked("file.ts")).toBe(true) + expect(cache.getHash("file.ts")).toBeUndefined() + expect(enumerate).not.toHaveBeenCalled() + expect(cache.getTrackedPaths()).toEqual(["file.ts"]) + cache.completeRemoval("file.ts") + await cache.flush() + expect(cache.isTracked("file.ts")).toBe(false) + expect(cache.getTrackedPaths()).toEqual([]) + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: {} }) + }) - await cacheManager.clearCacheFile() + it("persists the reserved watermark through hash and open-record clears", async () => { + await cache.reserveGenerations(4096) + cache.completeFile("complete.ts", "hash") + await cache.invalidateFile("open.ts") + await cache.clearCacheFile() + expect(cache.generationWatermark).toBe(4096) + expect(persisted).toEqual({ version: 2, fileHashes: {}, openFiles: {}, generationWatermark: 4096 }) + const reloaded = freshCache() + await reloaded.initialize() + expect(reloaded.generationWatermark).toBe(4096) + }) - expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, {}) - expect(cacheManager.getAllHashes()).toEqual({}) + it("does not acknowledge a reservation before it is durable", async () => { + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { + await held.promise + persisted = structuredClone(data) }) + const saved = vi.fn() + const reservation = cache.reserveGenerations(1024).then(saved) + await vi.advanceTimersByTimeAsync(0) + expect(saved).not.toHaveBeenCalled() + held.resolve() + await reservation + expect(persisted).toMatchObject({ generationWatermark: 1024 }) + }) - it("should handle clear errors gracefully", async () => { - const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(function () {}) - ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) + it.each(["clearCacheFile", "flush", "saveNow"] as const)("propagates %s write errors", async (method) => { + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) + await expect(cache[method]()).rejects.toThrow("Write denied") + }) - await cacheManager.clearCacheFile() + it("propagates generation reservation failure and permits a later reservation", async () => { + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) + await expect(cache.reserveGenerations(1024)).rejects.toThrow("Write denied") + await cache.reserveGenerations(2048) + expect(persisted).toMatchObject({ generationWatermark: 2048 }) + }) - expect(consoleErrorSpy).toHaveBeenCalledWith( - "Failed to clear cache file:", - expect.any(Error), - mockCachePath, - ) + it("keeps legacy hash mutation aliases compatible with Zoo callers", async () => { + cache.updateHash("file.ts", "hash") + expect(cache.getHash("file.ts")).toBe("hash") + await cache.flush() + cache.deleteHash("file.ts") + await cache.flush() + expect(cache.getHash("file.ts")).toBeUndefined() + expect(cache.isTracked("file.ts")).toBe(false) + }) - consoleErrorSpy.mockRestore() - }) + it("logs debounced-save errors without an unhandled rejection", async () => { + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) + cache.completeFile("file.ts", "hash") + await vi.advanceTimersByTimeAsync(1500) + expect(codeIndexLog.error).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) }) }) diff --git a/src/services/code-index/__tests__/code-index-log.spec.ts b/src/services/code-index/__tests__/code-index-log.spec.ts new file mode 100644 index 0000000000..78466c37d6 --- /dev/null +++ b/src/services/code-index/__tests__/code-index-log.spec.ts @@ -0,0 +1,179 @@ +const mocks = vi.hoisted(() => ({ + channel: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), dispose: vi.fn() }, + createOutputChannel: vi.fn(), + postMessage: vi.fn(), + thread: { isMainThread: true }, +})) +vi.mock("vscode", () => ({ window: { createOutputChannel: mocks.createOutputChannel } })) +vi.mock("node:worker_threads", () => ({ + get isMainThread() { + return mocks.thread.isMainThread + }, + parentPort: { postMessage: mocks.postMessage }, +})) + +import { codeIndexLog } from "../code-index-log" +import { initializeCodeIndexHostLogging } from "../code-index-log-host" +import { clearAllMocks } from "../../../test-utils/reset" + +beforeEach(() => { + codeIndexLog.dispose() + clearAllMocks() + mocks.thread.isMainThread = true + mocks.createOutputChannel.mockReturnValue(mocks.channel) + codeIndexLog.configure({ workspacePaths: [], secrets: [] }) + initializeCodeIndexHostLogging() +}) + +afterEach(() => codeIndexLog.dispose()) + +it("lazily uses one named log channel and prefixes every severity", () => { + expect(mocks.createOutputChannel).not.toHaveBeenCalled() + for (const level of ["info", "warn", "error", "debug"] as const) { + codeIndexLog[level]("message", { source: "scan", attempt: 2 }) + expect(mocks.channel[level]).toHaveBeenCalledWith('[CodeIndex] message {"source":"scan","attempt":2}') + } + expect(mocks.createOutputChannel).toHaveBeenCalledExactlyOnceWith("Zoo Code Indexing", { log: true }) +}) + +it("redacts configured secrets, headers, URL credentials and workspace roots in all fields", () => { + codeIndexLog.configure({ + workspacePaths: ["C:\\work\\project", "/home/user/project"], + secrets: ["private-key", "a+b"], + }) + codeIndexLog.error("https://user:password@host/test?api-key=private-key", { + path: "c:/work/project/src/file.ts", + other: "C:\\work\\project\\src\\other.ts", + error: new Error('/home/user/project/src/file.ts Authorization: Bearer private-key\napi-key="a+b"'), + nested: { Authorization: "secret", apiKey: "secret", content: "never log file contents" }, + }) + const output = mocks.channel.error.mock.calls[0][0] + for (const secret of [ + "user:password", + "private-key", + "a+b", + "home/user", + "work", + "never log file contents", + '"secret"', + ]) { + expect(output).not.toContain(secret) + } + expect(output).toContain("src/file.ts") + expect(output).toContain("[REDACTED]") +}) + +it("does not log SDK request bodies or arbitrary Error properties", () => { + const error = Object.assign(new Error("request failed"), { request: { body: "file contents" }, status: 503 }) + codeIndexLog.error("failed", error) + expect(mocks.channel.error.mock.calls[0][0]).toContain('"status":503') + expect(mocks.channel.error.mock.calls[0][0]).not.toContain("file contents") +}) + +it("keeps every retry and final failure in one warning per explicit flush", () => { + const batch = codeIndexLog.createFailureBatch({ source: "watcher", batch: 7 }) + batch.flush() + for (const attempt of [1, 2, 3]) + batch.record({ path: "file.ts", activity: "embed", reason: "503", attempt, willRetry: attempt < 3 }) + expect(mocks.channel.warn).not.toHaveBeenCalled() + batch.flush() + batch.flush() + expect(mocks.channel.warn).toHaveBeenCalledOnce() + const output = mocks.channel.warn.mock.calls[0][0] + const fields = JSON.parse(output.slice(output.indexOf("{"))) + expect(fields.failureCount).toBe(3) + expect(fields.failures.map((entry: { attempt: number }) => entry.attempt)).toEqual([1, 2, 3]) + expect(fields.failures[2].willRetry).toBe(false) + batch.record({ path: "other.ts", activity: "verify", reason: "timeout", attempt: 1, willRetry: true }) + batch.flush() + expect(mocks.channel.warn).toHaveBeenCalledTimes(2) +}) + +it.each(["info", "warn", "error", "debug"] as const)( + "forwards redacted worker %s diagnostics without invoking the host sink", + (level) => { + mocks.thread.isMainThread = false + codeIndexLog.configure({ workspacePaths: ["/workspace"], secrets: ["configured-secret"] }) + codeIndexLog[level]("api-key=hidden", { + path: "/workspace/file.ts", + reason: "configured-secret", + content: "file contents", + nested: { token: "hidden" }, + }) + expect(mocks.createOutputChannel).not.toHaveBeenCalled() + expect(mocks.channel[level]).not.toHaveBeenCalled() + expect(mocks.postMessage).toHaveBeenCalledExactlyOnceWith({ + id: 0, + diagnostic: { + level, + message: "api-key=[REDACTED]", + fields: { + path: "file.ts", + reason: "[REDACTED]", + content: "[REDACTED]", + nested: { token: "[REDACTED]" }, + }, + }, + }) + }, +) + +it("initializes idempotently and recreates the channel after disposal", () => { + codeIndexLog.info("first") + initializeCodeIndexHostLogging() + codeIndexLog.info("second") + expect(mocks.createOutputChannel).toHaveBeenCalledTimes(1) + expect(mocks.channel.dispose).not.toHaveBeenCalled() + codeIndexLog.dispose() + expect(mocks.channel.dispose).toHaveBeenCalledTimes(1) + codeIndexLog.info("third") + expect(mocks.createOutputChannel).toHaveBeenCalledTimes(2) +}) + +it("disposes the previous channel when replacing the host sink", () => { + codeIndexLog.info("first") + const replacement = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), dispose: vi.fn() } + codeIndexLog.setHostSinkFactory(() => replacement) + expect(mocks.channel.dispose).toHaveBeenCalledOnce() + codeIndexLog.info("second") + expect(replacement.info).toHaveBeenCalledExactlyOnceWith("[CodeIndex] second") +}) + +it("handles cycles and configuration replacement", () => { + const fields: Record = {} + fields.circular = fields + codeIndexLog.info("cycle", fields) + expect(mocks.channel.info.mock.calls[0][0]).toContain("[Circular]") + codeIndexLog.configure({ secrets: ["old"] }) + codeIndexLog.configure({ secrets: ["new"] }) + codeIndexLog.info("old new") + expect(mocks.channel.info).toHaveBeenLastCalledWith("[CodeIndex] old [REDACTED]") +}) + +it("matches a reference redactor and scales across a 1/2/4/8 input ladder", () => { + codeIndexLog.configure({ workspacePaths: ["/workspace/project"], secrets: ["configured-secret"] }) + const unit = + 'failed /workspace/project/src/file.ts https://user:password@host/path api-key="token" configured-secret\n' + const expectedUnit = 'failed src/file.ts https://[REDACTED]@host/path api-key="[REDACTED]" [REDACTED]\n' + let captured = "" + mocks.channel.debug.mockImplementation((message: string) => { + captured = message + }) + const samples: number[] = [] + for (const multiplier of [1, 2, 4, 8]) { + const input = unit.repeat(500 * multiplier) + const expected = `[CodeIndex] ${expectedUnit.repeat(500 * multiplier)}` + codeIndexLog.debug(input) + expect(captured).toBe(expected) + const timings: number[] = [] + for (let sampleNumber = 0; sampleNumber < 5; sampleNumber++) { + const started = performance.now() + for (let iteration = 0; iteration < 15; iteration++) codeIndexLog.debug(input) + timings.push(performance.now() - started) + } + samples.push(timings.sort((left, right) => left - right)[2]) + } + console.log(`codeIndexLog redaction ladder ms: ${samples.map((sample) => sample.toFixed(2)).join(" / ")}`) + expect(samples[2] / samples[0]).toBeLessThan(8) + expect(samples[3] / samples[0]).toBeLessThan(16) +}) diff --git a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts index bcc63f7c00..e69ba39579 100644 --- a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts +++ b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts @@ -1,158 +1,142 @@ import { CodeIndexScanExecutor } from "../code-index-scan-executor" import type { IDirectoryScanner } from "../interfaces" +import { ParserFatalError } from "../parser-errors" +import { EmbeddingError, IndexingRetryError } from "../shared/indexing-errors" -vi.mock("../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) function setup() { const scanner = { scanDirectory: vi.fn() } - const vectorStore = { markIndexingIncomplete: vi.fn().mockResolvedValue(undefined) } const stateManager = { setSystemState: vi.fn(), reportBlockIndexingProgress: vi.fn() } + let current = true return { scanner, - vectorStore, stateManager, - executor: new CodeIndexScanExecutor("/workspace", scanner, vectorStore, stateManager), + supersede: () => { + current = false + }, + executor: new CodeIndexScanExecutor("/workspace", scanner, stateManager, () => current), } } +const methods = ["runFullScan", "runIncrementalScan"] as const + describe("CodeIndexScanExecutor", () => { - it.each(["runFullScan", "runIncrementalScan"] as const)( - "%s skips failure validation when cancellation occurs during the scan", - async (method) => { - const { executor, scanner } = setup() + it.each(methods)("%s preserves progress and signal wiring", async (method) => { + const { executor, scanner, stateManager } = setup() + const signal = new AbortController().signal + scanner.scanDirectory.mockImplementation(async (directory, _onError, onIndexed, onParsed, receivedSignal) => { + expect(directory).toBe("/workspace") + expect(receivedSignal).toBe(signal) + onParsed?.(2) + onIndexed?.(1) + onParsed?.(1) + onIndexed?.(2) + return { stats: { processed: 2, skipped: 0 }, totalBlockCount: 3 } + }) + await expect(executor[method](signal)).resolves.toBe(true) + expect(stateManager.reportBlockIndexingProgress.mock.calls).toEqual([ + [0, 2, "scan"], + [1, 2, "scan"], + [1, 3, "scan"], + [3, 3, "scan"], + ]) + expect(executor.partial).toBe(false) + }) + + it.each(methods)("%s rejects a missing scanner result", async (method) => { + const { executor } = setup() + await expect(executor[method](new AbortController().signal)).rejects.toThrow("is scanner initialized?") + }) + + it.each(methods)("%s propagates genuine scanner failures unchanged", async (method) => { + const { executor, scanner } = setup() + const failure = new Error("collection failed") + scanner.scanDirectory.mockRejectedValue(failure) + await expect(executor[method](new AbortController().signal)).rejects.toBe(failure) + }) + + it.each(methods)("%s refuses dispatch after cancellation", async (method) => { + const { executor, scanner, stateManager } = setup() + const controller = new AbortController() + controller.abort() + await expect(executor[method](controller.signal)).resolves.toBe(false) + expect(scanner.scanDirectory).not.toHaveBeenCalled() + expect(stateManager.setSystemState).not.toHaveBeenCalled() + }) + + it.each(methods)("%s ignores stopped and superseded callbacks", async (method) => { + for (const ending of ["stop", "supersede"]) { + const { executor, scanner, stateManager, supersede } = setup() const controller = new AbortController() - scanner.scanDirectory.mockImplementation(async (_path, onError, _onIndexed, onParsed) => { + scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { + if (ending === "stop") controller.abort() + else supersede() onParsed?.(3) - onError?.(new Error("batch failure")) - controller.abort() + onIndexed?.(1) + onError?.(new ParserFatalError("parser-abort", "bad.ts")) return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 3 } }) await expect(executor[method](controller.signal)).resolves.toBe(false) - }, - ) - - it("completes an incremental scan with no changed files", async () => { - const { executor, scanner } = setup() - scanner.scanDirectory.mockResolvedValue({ stats: { processed: 0, skipped: 2 }, totalBlockCount: 0 }) - await expect(executor.runIncrementalScan(new AbortController().signal)).resolves.toBe(true) + expect(stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() + } }) - it.each(["runFullScan", "runIncrementalScan"] as const)( - "%s preserves progress and signal wiring", - async (method) => { - const { executor, scanner, vectorStore, stateManager } = setup() - const signal = new AbortController().signal - scanner.scanDirectory.mockImplementation(async (path, _onError, onIndexed, onParsed, receivedSignal) => { - expect(path).toBe("/workspace") - expect(receivedSignal).toBe(signal) - expect(vectorStore.markIndexingIncomplete).toHaveBeenCalledOnce() - onParsed?.(2) - onIndexed?.(1) - onParsed?.(1) - onIndexed?.(2) - return { stats: { processed: 2, skipped: 0 }, totalBlockCount: 3 } + it.each(methods)("%s retains file failures as partial regardless of percentage", async (method) => { + for (const indexed of [0, 8, 9]) { + const { executor, scanner } = setup() + scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { + onParsed?.(10) + onIndexed?.(indexed) + onError?.(new Error("file failed")) + return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 10, incompleteFiles: ["bad.ts"] } }) - await expect(executor[method](signal)).resolves.toBe(true) - expect(stateManager.reportBlockIndexingProgress.mock.calls).toEqual([ - [0, 2], - [1, 2], - [1, 3], - [3, 3], - ]) - expect(stateManager.setSystemState).toHaveBeenCalledExactlyOnceWith( - "Indexing", - method === "runFullScan" - ? "Services ready. Starting workspace scan..." - : "Checking for new or modified files...", - ) - }, - ) - - it.each(["runFullScan", "runIncrementalScan"] as const)("%s rejects missing results", async (method) => { - const { executor } = setup() - // An unconfigured mock simulates a scanner violating its return contract. - await expect(executor[method](new AbortController().signal)).rejects.toThrow( - method === "runFullScan" - ? "Scan failed, is scanner initialized?" - : "Incremental scan failed, is scanner initialized?", - ) + await expect(executor[method](new AbortController().signal)).resolves.toBe(true) + expect(executor.partial).toBe(true) + } }) - it.each(["runFullScan", "runIncrementalScan"] as const)( - "%s returns cancellation before validating results", - async (method) => { - const { executor, scanner } = setup() - const controller = new AbortController() - // Abort before the unconfigured scanner returns a missing result. - controller.abort() - await expect(executor[method](controller.signal)).resolves.toBe(false) - expect(scanner.scanDirectory).toHaveBeenCalledOnce() - }, - ) + it.each(methods)("%s accepts an empty workspace", async (method) => { + const { executor, scanner } = setup() + scanner.scanDirectory.mockResolvedValue({ stats: { processed: 0, skipped: 2 }, totalBlockCount: 0 }) + await expect(executor[method](new AbortController().signal)).resolves.toBe(true) + expect(executor.partial).toBe(false) + }) - it.each(["runFullScan", "runIncrementalScan"] as const)( - "%s propagates scanner and marker failures unchanged", - async (method) => { - const { executor, scanner, vectorStore } = setup() - const signal = new AbortController().signal - const failure = new Error("scan failed") - scanner.scanDirectory.mockRejectedValueOnce(failure) - await expect(executor[method](signal)).rejects.toBe(failure) - const abort = new DOMException("stopped", "AbortError") - scanner.scanDirectory.mockRejectedValueOnce(abort) - await expect(executor[method](signal)).rejects.toBe(abort) - vectorStore.markIndexingIncomplete.mockRejectedValueOnce(failure) - await expect(executor[method](signal)).rejects.toBe(failure) - expect(scanner.scanDirectory).toHaveBeenCalledTimes(2) - }, - ) + it.each(methods)("%s propagates a typed fatal callback", async (method) => { + const { executor, scanner } = setup() + const fatal = new ParserFatalError("parse-timeout", "bad.ts") + scanner.scanDirectory.mockImplementation(async (_directory, onError) => { + onError?.(fatal) + return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } + }) + await expect(executor[method](new AbortController().signal)).rejects.toBe(fatal) + }) - it.each([ - { found: 0, indexed: 0, error: false, message: undefined }, - { found: 3, indexed: 0, error: false, message: "embeddings:orchestrator.indexingFailedNoBlocks" }, - { found: 3, indexed: 0, error: true, message: "Indexing failed: batch failure" }, - { found: 0, indexed: 0, error: true, message: "Indexing failed completely: batch failure" }, - { found: 10, indexed: 9, error: true, message: undefined }, - { - found: 10, - indexed: 8, - error: true, - message: "Indexing partially failed: Only 8 of 10 blocks were indexed. batch failure", - }, - { found: 10, indexed: 8, error: false, message: undefined }, - ])( - "preserves full-scan policy for $indexed/$found blocks, error=$error", - async ({ found, indexed, error, message }) => { + it.each([400, 401, 403, 404, 422, 429, 500, undefined])( + "escalates only all-embedding authentication/configuration HTTP %s failures in a full scan", + async (status) => { const { executor, scanner } = setup() - scanner.scanDirectory.mockImplementation(async (_path, onError, onIndexed, onParsed) => { - onParsed?.(found) - onIndexed?.(indexed) - if (error) onError?.(new Error("batch failure")) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: found } + const error = new Error("file embedding failed", { + cause: new IndexingRetryError( + new EmbeddingError("provider failed", { status }), + 1, + "permanent", + "embed", + ), + }) + scanner.scanDirectory.mockImplementation(async (_directory, onError, _onIndexed, onParsed) => { + onParsed?.(1) + onError?.(error) + return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 1 } }) const result = executor.runFullScan(new AbortController().signal) - if (message) await expect(result).rejects.toThrow(message) - else await expect(result).resolves.toBe(true) + if (status !== undefined && [400, 401, 403, 404, 422].includes(status)) + await expect(result).rejects.toBe(error) + else { + await expect(result).resolves.toBe(true) + expect(executor.partial).toBe(true) + } }, ) - - it.each([0, 2])("rejects incremental batch errors with %s indexed blocks", async (indexed) => { - const { executor, scanner } = setup() - const failures = [new Error("embedding failure"), new Error("upsert failure"), new Error("embedding failure")] - scanner.scanDirectory.mockImplementation(async (_path, onError, onIndexed, onParsed) => { - onParsed?.(3) - onIndexed?.(indexed) - for (const failure of failures) onError?.(failure) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 3 } - }) - const result = executor.runIncrementalScan(new AbortController().signal) - await expect(result).rejects.toBeInstanceOf(AggregateError) - await expect(result).rejects.toMatchObject({ - errors: failures, - message: "Incremental scan failed with 3 errors:\nembedding failure\nupsert failure", - }) - await result.catch((error: AggregateError) => { - for (const [index, failure] of failures.entries()) expect(error.errors[index]).toBe(failure) - }) - }) }) diff --git a/src/services/code-index/__tests__/code-index-status-manager.spec.ts b/src/services/code-index/__tests__/code-index-status-manager.spec.ts index 6d003194a6..2a0d454a60 100644 --- a/src/services/code-index/__tests__/code-index-status-manager.spec.ts +++ b/src/services/code-index/__tests__/code-index-status-manager.spec.ts @@ -39,6 +39,9 @@ function makeSource(workspacePath: string) { workspacePath, workspaceEnabled: true, autoEnableDefault: true, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], } let emit = () => {} const subscription = { dispose: vi.fn() } @@ -168,6 +171,43 @@ describe("CodeIndexStatusManager", () => { manager.dispose() }) + it.each(["Error", "Stopping", "Standby", "Indexed"] as const)( + "publishes computed %s and failures rather than the event payload", + (systemStatus) => { + const source = makeSource("/first") + const publish = vi.fn() + const manager = new CodeIndexStatusManager(() => source) + manager["publishStatus"] = publish + const computed = { + ...source.status, + systemStatus, + message: "Computed manager status", + failedFiles: [ + { + path: "failed.ts", + revision: "version", + activity: "embed" as const, + reason: "unavailable", + classification: "server" as const, + attempt: 3, + state: "exhausted" as const, + }, + ], + } + source.getCurrentStatus.mockReturnValue(computed) + try { + manager.init() + expect(publish).toHaveBeenCalledExactlyOnceWith(computed) + source.status.systemStatus = "Indexing" + source.emit() + expect(publish).toHaveBeenLastCalledWith(computed) + expect(publish).toHaveBeenCalledTimes(2) + } finally { + manager.dispose() + } + }, + ) + it("publishes initial and full updated status without resubscribing in the same workspace", () => { const source = makeSource("/first") const publish = vi.fn() @@ -177,6 +217,7 @@ describe("CodeIndexStatusManager", () => { manager.init() expect(publish).toHaveBeenCalledExactlyOnceWith(source.status) source.status.message = "Updated" + source.status.providerWaits = [{ service: "embedding", resumeAt: 123_000, waitingFiles: ["waiting.ts"] }] source.emit() expect(publish).toHaveBeenLastCalledWith(source.status) expect(publish).toHaveBeenCalledTimes(2) diff --git a/src/services/code-index/__tests__/file-completion.spec.ts b/src/services/code-index/__tests__/file-completion.spec.ts new file mode 100644 index 0000000000..606506ab1e --- /dev/null +++ b/src/services/code-index/__tests__/file-completion.spec.ts @@ -0,0 +1,672 @@ +import "./mock-code-index-log" +import { v5 as uuidv5 } from "uuid" +import { FileCompletion, FileIndexWriter } from "../file-completion" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" +import { QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" +import type { CodeBlock, IVectorStore } from "../interfaces" +import type { CacheManager } from "../cache-manager" +import type { GenerationAllocator } from "../generation-allocator" +import { makeRateLimitScope, ProviderPacerRegistry } from "../provider-pacer" +import { FileFailureRegistry } from "../file-failure-registry" + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} + +describe("FileCompletion", () => { + it("does not finish when an early batch settles before the final batch is registered", async () => { + const finish = vi.fn().mockResolvedValue(undefined) + const completion = new FileCompletion(finish) + completion.register() + await completion.settled(true) + expect(completion.outstanding).toBe(0) + expect(finish).not.toHaveBeenCalled() + completion.register() + await completion.registered() + expect(finish).not.toHaveBeenCalled() + await completion.settled(true) + expect(finish).toHaveBeenCalledOnce() + }) + + it("finishes an empty file once and rejects registration after the file is closed", async () => { + const finish = vi.fn().mockResolvedValue(undefined) + const completion = new FileCompletion(finish) + await completion.registered() + await completion.registered() + expect(finish).toHaveBeenCalledOnce() + expect(() => completion.register()).toThrow("registration already complete") + }) + + it.each(["before registration", "after registration"])("keeps a failed file incomplete %s", async (timing) => { + const finish = vi.fn() + const completion = new FileCompletion(finish) + completion.register() + completion.register() + if (timing === "after registration") await completion.registered() + await completion.settled(false) + if (timing === "before registration") await completion.registered() + await completion.settled(true) + expect(completion.failed).toBe(true) + expect(completion.outstanding).toBe(0) + expect(finish).not.toHaveBeenCalled() + }) + + it("shares a pending finish and never retries a failed finish", async () => { + const held = deferred() + const finish = vi.fn(() => held.promise) + const completion = new FileCompletion(finish) + const first = completion.registered() + const second = completion.registered() + const firstRejected = expect(first).rejects.toThrow("Verification failed") + const secondRejected = expect(second).rejects.toThrow("Verification failed") + expect(finish).toHaveBeenCalledOnce() + held.reject(new Error("Verification failed")) + await Promise.all([firstRejected, secondRejected]) + expect(completion.failed).toBe(true) + await completion.registered() + expect(finish).toHaveBeenCalledOnce() + }) +}) + +function createWriterDoubles() { + return { + cache: { + invalidateFile: vi.fn<(filePath: string) => Promise>().mockResolvedValue(undefined), + completeFile: vi.fn<(filePath: string, hash: string) => void>(), + completeRemoval: vi.fn<(filePath: string) => void>(), + }, + store: { + rateLimitScope: "qdrant:writer-test", + upsertPoints: vi.fn().mockResolvedValue(undefined), + deletePointsByFilePath: vi + .fn<(filePath: string, generation: number) => Promise>() + .mockResolvedValue(undefined), + countPointsForFile: vi + .fn<(filePath: string, generation: number) => Promise<{ current: number; other: number }>>() + .mockResolvedValue({ current: 2, other: 0 }), + }, + generations: { allocate: vi.fn().mockResolvedValue(17) }, + embedder: { + createEmbeddings: vi.fn(async (texts: string[]) => ({ embeddings: texts.map(() => [0.1, 0.2]) })), + rateLimitScope: "openai:writer-test", + validateConfiguration: vi.fn(async () => ({ valid: true })), + embedderInfo: { name: "openai" as const }, + }, + } +} + +function createWriter( + doubles: Pick, "cache" | "store" | "generations">, + gate: IndexWriteGate, + failures: FileFailureRegistry, +) { + // Doubles expose invalidation/completion, vector mutations/counts, and allocation without unrelated dependencies. + return new FileIndexWriter( + doubles.cache as unknown as CacheManager, + doubles.store as unknown as IVectorStore, + gate, + doubles.generations as unknown as GenerationAllocator, + "/workspace", + failures, + ) +} + +describe("FileIndexWriter", () => { + const filePath = "/workspace/file.ts" + let cache: ReturnType["cache"] + let store: ReturnType["store"] + let generations: ReturnType["generations"] + let embedder: ReturnType["embedder"] + let gate: IndexWriteGate + let attempt: AbortController + let writer: FileIndexWriter + let failures: FileFailureRegistry + + function blocks(count: number): CodeBlock[] { + return Array.from({ length: count }, (_, offset) => ({ + file_path: filePath, + identifier: null, + type: "function", + start_line: 1, + end_line: 1, + content: ` block ${offset} `, + fileHash: "hash", + segmentHash: `segment-${offset}`, + })) + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + ;({ cache, store, generations, embedder } = createWriterDoubles()) + gate = new IndexWriteGate() + attempt = new AbortController() + failures = new FileFailureRegistry() + writer = createWriter({ cache, store, generations }, gate, failures) + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it.each(["embed", "upsert", "delete", "verify", "remove"])( + "retries only %s at 500/1000 ms and preserves its generation", + async (activity) => { + const request = + activity === "embed" + ? embedder.createEmbeddings + : activity === "upsert" + ? store.upsertPoints + : activity === "verify" + ? store.countPointsForFile + : store.deletePointsByFilePath + request.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) + request.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) + + const records = vi.fn() + failures.onDidRecord(records) + const pending = + activity === "remove" + ? writer.remove(filePath, attempt.signal) + : writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(1) + expect(failures.list()[0]).toMatchObject({ attempt: 1, state: "retrying", classification: "server" }) + await vi.advanceTimersByTimeAsync(499) + expect(request).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(request).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(999) + expect(request).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1) + expect(request).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await pending + expect(generations.allocate).toHaveBeenCalledOnce() + expect(embedder.createEmbeddings).toHaveBeenCalledTimes( + activity === "embed" ? 3 : activity === "remove" ? 0 : 1, + ) + expect(records).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + }, + ) + + it.each(["embed", "upsert"])("shares exactly two %s retries across batches", async (activity) => { + const request = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints + request.mockRejectedValueOnce({ status: 503 }).mockRejectedValueOnce({ status: 503 }) + if (activity === "embed") embedder.createEmbeddings.mockResolvedValueOnce({ embeddings: [[0.1]] }) + else store.upsertPoints.mockResolvedValueOnce(undefined) + request.mockRejectedValue({ status: 503 }) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1).catch((error) => error) + await vi.advanceTimersByTimeAsync(1500) + expect(await pending).toMatchObject({ activity, attempts: 3 }) + expect(request).toHaveBeenCalledTimes(4) + expect(failures.list()[0]).toMatchObject({ activity, attempt: 3, state: "exhausted" }) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow() + expect(request).toHaveBeenCalledTimes(4) + }) + + it("does not charge successful batch first calls against retries", async () => { + await writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + }) + + it("keeps exhausted store budgets across writer recreation and unrelated renewal", async () => { + store.upsertPoints.mockRejectedValue(Object.assign(new Error("unavailable"), { status: 503 })) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(1500) + expect(await pending).toMatchObject({ activity: "upsert", attempts: 3, classification: "server" }) + failures.renew("other.ts", "new") + const recreated = createWriter({ cache, store, generations }, gate, failures) + await expect(recreated.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "unavailable", + ) + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(failures.list()[0]).toMatchObject({ attempt: 3, state: "exhausted" }) + }) + + it.each(["allocation", "invalidation"])( + "exhausts a new revision after %s fails despite an old entry", + async (stage) => { + store.upsertPoints.mockRejectedValue(Object.assign(new Error("old server failure"), { status: 503 })) + const old = writer.replace(filePath, "r1", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(1500) + expect(await old).toMatchObject({ attempts: 3 }) + const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile + failingOperation.mockRejectedValue(new Error(`new ${stage} failure`)) + const records = vi.fn() + failures.onDidRecord(records) + + await expect(writer.replace(filePath, "r2", blocks(2), embedder, attempt.signal)).rejects.toThrow( + `new ${stage} failure`, + ) + const calls = failingOperation.mock.calls.length + await expect(writer.replace(filePath, "r2", blocks(2), embedder, attempt.signal)).rejects.toThrow( + `new ${stage} failure`, + ) + expect(failingOperation).toHaveBeenCalledTimes(calls) + expect(records).toHaveBeenCalledOnce() + expect(failures.list()[0]).toMatchObject({ + revision: "r2", + reason: `new ${stage} failure`, + attempt: 1, + state: "exhausted", + }) + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + }, + ) + + it.each(["allocation", "invalidation"])( + "exhausts a retained successful retry when the same revision later fails %s", + async (stage) => { + failures.renew("file.ts", "hash") + const request = vi + .fn() + .mockRejectedValueOnce(Object.assign(new Error("temporary failure"), { status: 503 })) + const retry = writer.retryTransient(request, { + signal: attempt.signal, + file: filePath, + revision: "hash", + activity: "embed", + }) + await vi.advanceTimersByTimeAsync(500) + await retry + expect(failures.list()[0]).toMatchObject({ state: "retrying", attempt: 1 }) + const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile + failingOperation.mockRejectedValue(new Error(`${stage} failed`)) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + `${stage} failed`, + ) + expect(failures.list()[0]).toMatchObject({ + activity: "embed", + reason: "temporary failure", + attempt: 1, + classification: "server", + state: "exhausted", + }) + expect(cache.completeFile).not.toHaveBeenCalled() + }, + ) + + it.each(["allocation", "invalidation"])( + "exhausts recovered retries when completion fails before a later %s failure", + async (stage) => { + embedder.createEmbeddings.mockRejectedValueOnce( + Object.assign(new Error("temporary embedding failure"), { status: 503 }), + ) + cache.completeFile.mockImplementation(() => { + throw new Error("completion failed") + }) + const pending = writer + .replace(filePath, "hash", blocks(2), embedder, attempt.signal) + .catch((error) => error) + await vi.advanceTimersByTimeAsync(500) + expect(await pending).toMatchObject({ message: "completion failed" }) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) + expect(store.countPointsForFile).toHaveBeenCalledOnce() + const failure = { + activity: "embed", + reason: "temporary embedding failure", + attempt: 1, + classification: "server", + state: "exhausted", + } + expect(failures.list()[0]).toMatchObject(failure) + expect(failures.getFailure("file.ts", "hash", "embed")).toMatchObject(failure) + + const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile + failingOperation.mockRejectedValue(new Error(`later ${stage} failure`)) + const calls = failingOperation.mock.calls.length + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "temporary embedding failure", + ) + expect(failingOperation).toHaveBeenCalledTimes(calls) + expect(failures.list()[0]).toMatchObject(failure) + }, + ) + + it("T35 a different API key completes through the writer while the original allowance remains paused", async () => { + const pacers = new ProviderPacerRegistry() + const originalScope = makeRateLimitScope("openai", "https://api.example.test/v1", "first-test-key") + const independentScope = makeRateLimitScope("openai", "https://api.example.test/v1", "second-test-key") + const paused = pacers.get(originalScope) + paused.reportRateLimit({ status: 429, retryAfterMs: 30_000 }, paused.markIssued()) + const resumeAt = paused.state().pausedUntil + embedder.rateLimitScope = independentScope + // These doubles provide cache completion, store writes/counts, and generation allocation. + writer = new FileIndexWriter( + cache as unknown as CacheManager, + store as unknown as IVectorStore, + gate, + generations as unknown as GenerationAllocator, + "/workspace", + failures, + pacers, + ) + await writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + expect(paused.state().pausedUntil).toBe(resumeAt) + expect(pacers.get(independentScope).state().pausedUntil).toBeUndefined() + expect(failures.list()).toEqual([]) + }) + + it("T41 an upsert aborted inside a paused pacer never enters the gate or incurs grace", async () => { + const pacers = new ProviderPacerRegistry() + // These doubles provide cache completion, store writes/counts, and generation allocation. + writer = new FileIndexWriter( + cache as unknown as CacheManager, + store as unknown as IVectorStore, + gate, + generations as unknown as GenerationAllocator, + "/workspace", + failures, + pacers, + ) + const pacer = pacers.get(store.rateLimitScope) + pacer.reportRateLimit({ status: 429, retryAfterMs: 30_000 }, pacer.markIssued()) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(0) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(pacer.state().waitingPaths).toEqual(["file.ts"]) + expect(gate.hasPending(attempt.signal)).toBe(false) + expect(store.upsertPoints).not.toHaveBeenCalled() + attempt.abort() + expect(await pending).toBe(attempt.signal.reason) + expect(gate.uncertainFiles(attempt.signal)).toEqual([]) + expect(gate.graceDeadline(filePath)).toBe(0) + expect(gate.needsReconciliation(filePath)).toBe(false) + await gate.settled(attempt.signal) + await vi.advanceTimersByTimeAsync(60_000) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }) + + it.each(["permanent", "abort", "revision"])("ends automatic attempts for %s", async (ending) => { + embedder.createEmbeddings.mockRejectedValue( + Object.assign(new Error("failed"), { + status: ending === "permanent" ? 401 : 503, + }), + ) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(0) + if (ending === "abort") attempt.abort() + if (ending === "revision") { + failures.renew("file.ts", "new") + failures.record("file.ts", "new", "read", "new failure", 1, "exhausted") + } + await vi.advanceTimersByTimeAsync(1500) + await pending + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(failures.list()[0].state).toBe("exhausted") + if (ending === "revision") expect(failures.list()[0].reason).toBe("new failure") + }) + + it("discards issued old-revision embeddings without mutating the new failure", async () => { + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockReturnValueOnce(held.promise) + const pending = writer.replace(filePath, "old", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(0) + failures.renew("file.ts", "new") + failures.record("file.ts", "new", "parse", "new failure", 1, "exhausted") + held.resolve({ embeddings: [[1], [1]] }) + expect(await pending).toMatchObject({ name: "AbortError" }) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(failures.list()[0].reason).toBe("new failure") + }) + + it("waits for an issued superseded revision save before completing the newer revision", async () => { + const held = deferred() + store.upsertPoints.mockReturnValueOnce(held.promise) + const old = writer.replace(filePath, "old", blocks(2), embedder, attempt.signal).catch((error) => error) + await vi.advanceTimersByTimeAsync(0) + const current = writer.replace(filePath, "new", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(cache.completeFile).not.toHaveBeenCalled() + held.resolve() + await current + expect(await old).toMatchObject({ name: "AbortError" }) + expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(filePath, "new") + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + }) + + it("awaits durable invalidation before any embedding or vector mutation", async () => { + const held = deferred() + cache.invalidateFile.mockReturnValue(held.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + held.resolve() + await pending + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it.each(["replace", "remove"] as const)( + "issues no vector mutation when %s invalidation fails", + async (operation) => { + cache.invalidateFile.mockRejectedValue(new Error("Disk full")) + const pending = + operation === "replace" + ? writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + : writer.remove(filePath, attempt.signal) + await expect(pending).rejects.toThrow("Disk full") + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(false) + expect(failures.list()).toEqual([ + expect.objectContaining({ + path: "file.ts", + revision: operation === "replace" ? "hash" : "removed", + activity: operation === "replace" ? "upsert" : "delete", + reason: "Disk full", + state: "exhausted", + }), + ]) + }, + ) + + it.each(["replace", "remove"] as const)("records %s allocation failures before any mutation", async (operation) => { + generations.allocate.mockRejectedValue(new Error("watermark save failed")) + const pending = + operation === "replace" + ? writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + : writer.remove(filePath, attempt.signal) + await expect(pending).rejects.toThrow("watermark save failed") + expect(cache.invalidateFile).not.toHaveBeenCalled() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(failures.list()[0]).toMatchObject({ + path: "file.ts", + activity: operation === "replace" ? "upsert" : "delete", + reason: "watermark save failed", + state: "exhausted", + classification: "permanent", + }) + }) + + it("upserts every registered batch before fenced cleanup, verification, hash and progress", async () => { + const finalBatch = deferred() + store.upsertPoints.mockResolvedValueOnce(undefined).mockReturnValueOnce(finalBatch.promise) + const onComplete = vi.fn() + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) + await vi.advanceTimersByTimeAsync(0) + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + finalBatch.resolve() + await pending + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(store.countPointsForFile).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + expect(onComplete).toHaveBeenCalledWith(2) + expect(store.upsertPoints.mock.invocationCallOrder[1]).toBeLessThan( + store.deletePointsByFilePath.mock.invocationCallOrder[0], + ) + expect(store.countPointsForFile.mock.invocationCallOrder[0]).toBeLessThan( + cache.completeFile.mock.invocationCallOrder[0], + ) + expect(embedder.createEmbeddings).toHaveBeenNthCalledWith(1, ["block 0"], undefined, expect.any(AbortSignal)) + const points = store.upsertPoints.mock.calls.flatMap(([batch]) => batch) + expect(points.map((point) => point.id)).toEqual([ + uuidv5("segment-0:17", QDRANT_CODE_BLOCK_NAMESPACE), + uuidv5("segment-1:17", QDRANT_CODE_BLOCK_NAMESPACE), + ]) + expect(points[0].payload).toMatchObject({ filePath: "file.ts", generation: 17, segmentHash: "segment-0" }) + expect(new Set(points.map((point) => point.id)).size).toBe(2) + }) + + it.each([ + { current: 1, other: 0 }, + { current: 2, other: 1 }, + ])("does not complete a file with missing or stale points (%j)", async (counts) => { + store.countPointsForFile.mockResolvedValue(counts) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "verification failed", + ) + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it("cleans and verifies a zero-content file without embedding", async () => { + store.countPointsForFile.mockResolvedValue({ current: 0, other: 0 }) + const whitespace = blocks(1) + whitespace[0].content = " \n\t" + await writer.replace(filePath, "hash", whitespace, embedder, attempt.signal) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it("does not issue cleanup or completion after a failed batch", async () => { + store.upsertPoints.mockRejectedValueOnce(new Error("Upsert failed")) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow( + "Upsert failed", + ) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + }) + + it("rejects incomplete embeddings without issuing an upsert", async () => { + embedder.createEmbeddings.mockResolvedValue({ embeddings: [[0.1]] }) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "Incomplete embeddings", + ) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it("discards late embedding results after cancellation without issuing subsequent batches", async () => { + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockReturnValueOnce(held.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(0) + attempt.abort() + held.resolve({ embeddings: [[0.1]] }) + await rejected + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it.each(["complete", "stale", "cancel"])( + "keeps an uncertain file incomplete through grace (%s)", + async (outcome) => { + gate.recover([filePath]) + const onComplete = vi.fn() + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) + const observed = pending.catch((error) => error) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(store.countPointsForFile).toHaveBeenCalledTimes(1) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(onComplete).not.toHaveBeenCalled() + if (outcome === "cancel") attempt.abort() + if (outcome === "stale") store.countPointsForFile.mockResolvedValue({ current: 2, other: 1 }) + await vi.advanceTimersByTimeAsync(1) + const result = await observed + if (outcome === "complete") { + expect(result).toBeUndefined() + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + expect(onComplete).toHaveBeenCalledOnce() + expect(gate.needsReconciliation(filePath)).toBe(false) + } else { + expect(result).toMatchObject( + outcome === "cancel" + ? { name: "AbortError" } + : { message: expect.stringContaining("verification failed") }, + ) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(onComplete).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + } + }, + ) + + it("extends grace when another uncertainty arrives during final verification", async () => { + gate.recover([filePath]) + const heldCounts = deferred<{ current: number; other: number }>() + store.countPointsForFile.mockResolvedValueOnce({ current: 2, other: 0 }).mockReturnValueOnce(heldCounts.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + gate.recover([filePath]) + heldCounts.resolve({ current: 2, other: 0 }) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(cache.completeFile).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await pending + expect(store.countPointsForFile).toHaveBeenCalledTimes(3) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it("invalidates a completed file before removal and drops its open record only after success", async () => { + const invalidation = deferred() + const deletion = deferred() + cache.invalidateFile.mockReturnValue(invalidation.promise) + store.deletePointsByFilePath.mockReturnValue(deletion.promise) + const pending = writer.remove(filePath, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + invalidation.resolve() + await vi.advanceTimersByTimeAsync(0) + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeRemoval).not.toHaveBeenCalled() + deletion.resolve() + await pending + expect(cache.completeRemoval).toHaveBeenCalledWith(filePath) + }) + + it("retains the open record after an uncertain removal failure", async () => { + store.deletePointsByFilePath.mockRejectedValue(new DOMException("Lost result", "AbortError")) + await expect(writer.remove(filePath, attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + }) +}) diff --git a/src/services/code-index/__tests__/file-failure-registry.spec.ts b/src/services/code-index/__tests__/file-failure-registry.spec.ts new file mode 100644 index 0000000000..8081684e5d --- /dev/null +++ b/src/services/code-index/__tests__/file-failure-registry.spec.ts @@ -0,0 +1,91 @@ +import { FileFailureRegistry } from "../file-failure-registry" + +describe("FileFailureRegistry", () => { + it("shares retry counters only within one path, revision and operation", () => { + const registry = new FileFailureRegistry() + registry.renew("a.ts", "one") + registry.renew("b.ts", "one") + const budget = registry.getRetryBudget("a.ts", "one", "embed") + budget.retriesUsed = 2 + expect(registry.getRetryBudget("a.ts", "one", "embed")).toBe(budget) + expect(registry.getRetryBudget("a.ts", "one", "upsert").retriesUsed).toBe(0) + registry.renew("b.ts", "two") + expect(registry.getRetryBudget("a.ts", "one", "embed").retriesUsed).toBe(2) + registry.renew("a.ts", "two") + expect(registry.getRetryBudget("a.ts", "two", "embed").retriesUsed).toBe(0) + registry.getRetryBudget("a.ts", "two", "embed").retriesUsed = 1 + registry.resetAllBudgets() + expect(registry.getRetryBudget("a.ts", "two", "embed").retriesUsed).toBe(0) + registry.getRetryBudget("a.ts", "two", "embed").retriesUsed = 1 + registry.clear("a.ts", "two") + expect(registry.getRetryBudget("a.ts", "two", "embed").retriesUsed).toBe(0) + }) + + it("retains exhausted budgets per revision and operation until explicit reset", () => { + const registry = new FileFailureRegistry() + registry.renew("a.ts", "one") + registry.record("a.ts", "one", "upsert", "server failed", 3, "exhausted", "server") + registry.renew("a.ts", "one") + registry.renew("b.ts", "new") + expect(registry.isExhausted("a.ts", "one", "upsert")).toBe(true) + expect(registry.isExhausted("a.ts", "one", "embed")).toBe(false) + expect(() => registry.assertRetryable("a.ts", "one")).toThrow("server failed") + registry.resetAllBudgets() + expect(registry.isExhausted("a.ts", "one", "upsert")).toBe(false) + expect(registry.list()).toHaveLength(1) + registry.clear("a.ts", "one") + expect(registry.list()).toEqual([]) + }) + + it("distinguishes retained display entries from current revision failures", () => { + const registry = new FileFailureRegistry() + expect(registry.hasRevisionFailure("a.ts", "one")).toBe(false) + registry.renew("a.ts", "one") + registry.record("a.ts", "one", "embed", "old failure", 3, "exhausted", "server") + expect(registry.hasRevisionFailure("a.ts", "one")).toBe(true) + registry.renew("a.ts", "two") + expect(registry.has("a.ts")).toBe(true) + expect(registry.hasRevisionFailure("a.ts", "one")).toBe(false) + expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) + registry.record("a.ts", "two", "upsert", "new failure", 1, "exhausted") + expect(registry.hasRevisionFailure("a.ts", "two")).toBe(true) + registry.resetAllBudgets() + expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) + expect(registry.has("a.ts")).toBe(true) + registry.record("a.ts", "two", "verify", "failed again", 1, "exhausted") + registry.clear("a.ts", "two") + expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) + }) + + it("aborts only renewed path work and rejects stale records and clearing", () => { + const registry = new FileFailureRegistry() + const old = registry.renew("a.ts", "old") + const other = registry.renew("b.ts", "one") + registry.record("a.ts", "old", "embed", "failed", 1, "retrying", "connection") + registry.renew("a.ts", "new") + expect(old.aborted).toBe(true) + expect(other.aborted).toBe(false) + registry.record("a.ts", "new", "parse", "new failure", 1, "exhausted") + registry.record("a.ts", "old", "delete", "late failure", 3, "exhausted") + registry.clear("a.ts", "old") + expect(registry.list()[0]).toMatchObject({ revision: "new", activity: "parse", reason: "new failure" }) + }) + + it("emits every failed attempt and returns detached details", () => { + const registry = new FileFailureRegistry() + const records = vi.fn() + const changed = vi.fn() + const subscription = registry.onDidRecord(records) + registry.onDidChange(changed) + registry.renew("a.ts", "hash") + registry.record("a.ts", "hash", "verify", "busy", 1, "retrying", "rate-limit") + registry.record("a.ts", "hash", "verify", "busy", 2, "retrying", "rate-limit") + expect(records).toHaveBeenCalledTimes(2) + expect(changed).toHaveBeenCalledTimes(2) + registry.list()[0].reason = "mutated" + expect(registry.list()[0].reason).toBe("busy") + subscription.dispose() + registry.clear("a.ts") + expect(records).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/services/code-index/__tests__/fixtures/fault-loop.wasm b/src/services/code-index/__tests__/fixtures/fault-loop.wasm new file mode 100644 index 0000000000000000000000000000000000000000..146022e9e9b2bf6dadf01099b0967ec3e4f4257b GIT binary patch literal 39 ucmZQbEY4+QU|?WmVN76PU}j=uVCP_D$;r Promise>().mockResolvedValue(undefined), + reserveGenerations: vi.fn(async (nextWatermark: number) => { + cache.generationWatermark = nextWatermark + }), + } + const store = { + hasIndexedData: vi.fn<() => Promise>().mockResolvedValue(false), + clearCollection: vi.fn<() => Promise>().mockResolvedValue(undefined), + } + // Doubles supply only durable generation reservation/reset and collection inspection/reset. + const allocator = new GenerationAllocator(cache as unknown as CacheManager) + const vectorStore = store as unknown as IVectorStore + const signal = new AbortController().signal + const gate = new IndexWriteGate() + return { cache, store, allocator, vectorStore, signal, gate } +} + +describe("GenerationAllocator", () => { + it("awaits durable reservation before issuing a generation", async () => { + const { cache, allocator, vectorStore, signal, gate } = setup() + let release!: () => void + const reservation = new Promise((resolve) => { + release = resolve + }) + cache.reserveGenerations.mockReturnValueOnce(reservation) + const initialized = vi.fn() + const pending = allocator.initialize(vectorStore, signal, gate).then(initialized) + await Promise.resolve() + expect(cache.reserveGenerations).toHaveBeenCalledWith(1024) + expect(initialized).not.toHaveBeenCalled() + release() + await pending + await expect(allocator.allocate()).resolves.toBe(1) + }) + + it("serializes allocations and reserves only when the 1024-value block runs out", async () => { + const { cache, allocator, vectorStore, signal, gate } = setup() + await allocator.initialize(vectorStore, signal, gate) + const generations = await Promise.all(Array.from({ length: 1024 }, () => allocator.allocate())) + expect(generations).toEqual(Array.from({ length: 1024 }, (_, offset) => offset + 1)) + expect(cache.reserveGenerations).toHaveBeenCalledExactlyOnceWith(1024) + await expect(allocator.allocate()).resolves.toBe(1025) + expect(cache.reserveGenerations).toHaveBeenLastCalledWith(2048) + }) + + it("resumes above the entire durable block without consulting the clock", async () => { + const { cache, allocator, vectorStore, signal, gate } = setup(4096) + await allocator.initialize(vectorStore, signal, gate) + await expect(allocator.allocate()).resolves.toBe(4097) + expect(cache.reserveGenerations).toHaveBeenCalledWith(5120) + expect(cache.clearCacheFile).not.toHaveBeenCalled() + }) + + it.each([false, true])("resets unknown ordering with indexed data %s", async (hasData) => { + const { cache, store, allocator, vectorStore, signal, gate } = setup() + cache.generationWatermark = undefined + store.hasIndexedData.mockResolvedValue(hasData) + await allocator.initialize(vectorStore, signal, gate) + expect(store.clearCollection).toHaveBeenCalledTimes(hasData ? 1 : 0) + expect(cache.clearCacheFile).toHaveBeenCalledOnce() + expect(cache.reserveGenerations).toHaveBeenCalledWith(1024) + }) + + it("refuses initialization after reset failure without reserving", async () => { + const { cache, store, allocator, vectorStore, signal, gate } = setup() + cache.generationWatermark = undefined + store.hasIndexedData.mockResolvedValue(true) + store.clearCollection.mockRejectedValue(new Error("reset failed")) + await expect(allocator.initialize(vectorStore, signal, gate)).rejects.toThrow("reset failed") + expect(cache.reserveGenerations).not.toHaveBeenCalled() + }) + + it("recovers the allocation queue after reservation failure", async () => { + const { cache, allocator } = setup(1024) + cache.reserveGenerations.mockRejectedValueOnce(new Error("disk full")) + await expect(allocator.allocate()).rejects.toThrow("disk full") + await expect(allocator.allocate()).resolves.toBe(1025) + expect(cache.reserveGenerations).toHaveBeenLastCalledWith(2048) + }) + + it("rejects cancellation without consuming a generation", async () => { + const { allocator, vectorStore, signal, gate } = setup() + await allocator.initialize(vectorStore, signal, gate) + await expect(allocator.allocate(AbortSignal.abort())).rejects.toMatchObject({ name: "AbortError" }) + await expect(allocator.allocate(signal)).resolves.toBe(1) + }) + + it("rejects unsafe integer exhaustion before persistence", async () => { + const { cache, allocator, vectorStore, signal, gate } = setup(Number.MAX_SAFE_INTEGER) + await expect(allocator.initialize(vectorStore, signal, gate)).rejects.toThrow("counter exhausted") + expect(cache.reserveGenerations).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/code-index/__tests__/index-write-gate.spec.ts b/src/services/code-index/__tests__/index-write-gate.spec.ts new file mode 100644 index 0000000000..350744a55e --- /dev/null +++ b/src/services/code-index/__tests__/index-write-gate.spec.ts @@ -0,0 +1,320 @@ +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" + +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} + +describe("IndexWriteGate", () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it.each(["already aborted", "aborted before issue"])("refuses %s writes without uncertainty", async (timing) => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + const write = vi.fn() + if (timing === "already aborted") attempt.abort() + const rejected = expect(gate.run(attempt.signal, write, ["file.ts"])).rejects.toMatchObject({ + name: "AbortError", + }) + attempt.abort() + await rejected + expect(write).not.toHaveBeenCalled() + expect(gate.uncertainFiles(attempt.signal)).toEqual([]) + expect(gate.needsReconciliation("file.ts")).toBe(false) + expect(gate.hasPending(attempt.signal)).toBe(false) + }) + + it.each(["resolve", "reject"] as const)( + "never waits across attempts when an old write will %s", + async (outcome) => { + const gate = new IndexWriteGate() + const oldAttempt = new AbortController() + const nextAttempt = new AbortController() + const held = deferred() + const oldWrite = vi.fn(() => held.promise) + const pending = gate.run(oldAttempt.signal, oldWrite, ["file.ts"]).catch((error) => error) + await Promise.resolve() + expect(oldWrite).toHaveBeenCalledOnce() + oldAttempt.abort() + const newWrite = vi.fn(() => "current") + await expect(gate.run(nextAttempt.signal, newWrite, ["retry.ts"])).resolves.toBe("current") + expect(newWrite).toHaveBeenCalledOnce() + expect(gate.hasPending(oldAttempt.signal)).toBe(true) + expect(gate.hasPending(nextAttempt.signal)).toBe(false) + if (outcome === "resolve") held.resolve() + else held.reject(new Error("Request timeout")) + await pending + expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(outcome === "reject" ? ["file.ts"] : []) + expect(gate.needsReconciliation("file.ts")).toBe(outcome === "reject") + expect(gate.needsReconciliation("retry.ts")).toBe(false) + }, + ) + + it("discards a successful write after abort without extending an existing uncertainty deadline", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"]) + const deadline = gate.graceDeadline("file.ts") + await vi.advanceTimersByTimeAsync(1000) + await expect( + gate.run( + attempt.signal, + async () => { + attempt.abort() + return "remote success" + }, + ["file.ts"], + ), + ).rejects.toMatchObject({ name: "AbortError" }) + expect(gate.uncertainFiles(attempt.signal)).toEqual([]) + expect(gate.graceDeadline("file.ts")).toBe(deadline) + }) + + it.each([new Error("Transport failure"), new DOMException("Cancelled remotely", "AbortError")])( + "marks issued rejection uncertain even for $name", + async (failure) => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + await expect( + gate.run(attempt.signal, () => Promise.reject(failure), ["first.ts", "second.ts"]), + ).rejects.toBe(failure) + expect(gate.uncertainFiles(attempt.signal)).toEqual(["first.ts", "second.ts"]) + const done = vi.fn() + const grace = gate.waitForGrace("first.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + expect(done).toHaveBeenCalledOnce() + expect(gate.needsReconciliation("first.ts")).toBe(true) + gate.reconciled("first.ts") + expect(gate.needsReconciliation("first.ts")).toBe(false) + }, + ) + + it("starts rejection grace when rejection is observed, not when the request was issued", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + const held = deferred() + const pending = gate.run(attempt.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await vi.advanceTimersByTimeAsync(50_000) + held.reject(new Error("Timeout")) + await pending + const done = vi.fn() + const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + }) + + it("starts stop-expiry grace on abandon and tracks only pending writes from that attempt", async () => { + const gate = new IndexWriteGate() + const oldAttempt = new AbortController() + const otherAttempt = new AbortController() + const held = deferred() + const otherHeld = deferred() + const pending = gate.run(oldAttempt.signal, () => held.promise, ["old.ts"]).catch(() => {}) + const otherPending = gate.run(otherAttempt.signal, () => otherHeld.promise, ["other.ts"]) + await vi.advanceTimersByTimeAsync(2000) + oldAttempt.abort() + gate.abandon(oldAttempt.signal) + expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(["old.ts"]) + expect(gate.needsReconciliation("other.ts")).toBe(false) + const done = vi.fn() + const grace = gate.waitForGrace("old.ts", otherAttempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + held.resolve() + otherHeld.resolve() + await Promise.all([pending, otherPending]) + }) + + it("uses cache-load time and extends an active wait without shortening its deadline", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"], Date.now() - 10_000) + const done = vi.fn() + const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(5000) + gate.recover(["file.ts"]) + gate.recover(["file.ts"], Date.now() - 50_000) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + expect(done).toHaveBeenCalledOnce() + }) + + it("cancels grace promptly and cleans up its timer while retaining uncertainty", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"]) + const rejected = expect(gate.waitForGrace("file.ts", attempt.signal)).rejects.toMatchObject({ + name: "AbortError", + }) + expect(vi.getTimerCount()).toBe(1) + attempt.abort() + await rejected + expect(vi.getTimerCount()).toBe(0) + expect(gate.needsReconciliation("file.ts")).toBe(true) + await expect(gate.waitForGrace("unaffected.ts", attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + }) + + it("tracks combined watcher cancellation under its orchestrator attempt", async () => { + const gate = new IndexWriteGate() + const owner = new AbortController() + const local = new AbortController() + const combined = AbortSignal.any([owner.signal, local.signal]) + gate.own(combined, owner.signal) + const held = deferred() + const pending = gate.run(combined, () => held.promise, ["watcher.ts"]).catch(() => {}) + await Promise.resolve() + expect(gate.hasPending(owner.signal)).toBe(true) + owner.abort() + gate.abandon(owner.signal) + expect(gate.uncertainFiles(owner.signal)).toEqual(["watcher.ts"]) + held.resolve() + await pending + await gate.settled(owner.signal) + expect(gate.hasPending(owner.signal)).toBe(false) + }) + + it("completion excludes its current owner, live attempts and unrelated aborted files", async () => { + const gate = new IndexWriteGate() + const current = new AbortController() + const local = new AbortController() + const combined = AbortSignal.any([current.signal, local.signal]) + gate.own(combined, current.signal) + const live = new AbortController() + const old = new AbortController() + const held = deferred() + const writes = [ + gate.run(current.signal, () => held.promise, ["file.ts"]), + gate.run(live.signal, () => held.promise, ["file.ts"]), + gate.run(old.signal, () => held.promise, ["unrelated.ts"]).catch(() => {}), + ] + await Promise.resolve() + old.abort() + await expect(gate.waitForPriorWrites("file.ts", combined)).resolves.toBe(false) + expect(gate.graceDeadline("file.ts")).toBe(0) + held.resolve() + await Promise.all(writes) + }) + + it("bounds a prior aborted write without an orchestrator stop and never repeats abandonment", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + const done = vi.fn() + const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) + await vi.advanceTimersByTimeAsync(1999) + expect(done).not.toHaveBeenCalled() + expect(gate.graceDeadline("file.ts")).toBe(0) + await vi.advanceTimersByTimeAsync(1) + await wait + expect(done).toHaveBeenCalledWith(true) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + await vi.advanceTimersByTimeAsync(1000) + gate.abandon(old.signal) + await expect(gate.waitForPriorWrites("file.ts", current.signal)).resolves.toBe(false) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.reject(new Error("Late rejection")) + await pending + expect(gate.graceDeadline("file.ts")).toBe(133_000) + }) + + it("uses the orchestrator expiry without rebuilding it on abort", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + gate.stopAt(old.signal, Date.now() + 2000) + await vi.advanceTimersByTimeAsync(500) + old.abort() + const done = vi.fn() + const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) + await vi.advanceTimersByTimeAsync(1499) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await wait + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.resolve() + await pending + }) + + it("uses the abort stop deadline even when completion starts after its expiry", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + await vi.advanceTimersByTimeAsync(5000) + await gate.waitForPriorWrites("file.ts", current.signal) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.resolve() + await pending + }) + + it("cancels the prior-outcome wait and clears its timer without creating uncertainty", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + const wait = expect(gate.waitForPriorWrites("file.ts", current.signal)).rejects.toMatchObject({ + name: "AbortError", + }) + expect(vi.getTimerCount()).toBe(1) + current.abort() + await wait + expect(vi.getTimerCount()).toBe(0) + expect(gate.graceDeadline("file.ts")).toBe(0) + held.resolve() + await pending + }) + + it("settles only its own attempt and successful writes introduce no uncertainty", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + const other = new AbortController() + const held = deferred() + const pending = gate.run(other.signal, () => held.promise, ["other.ts"]) + await gate.run(attempt.signal, () => 42, ["file.ts"]) + await gate.settled(attempt.signal) + expect(gate.hasPending(other.signal)).toBe(true) + const done = vi.fn() + const settled = gate.settled(other.signal).then(done) + await Promise.resolve() + expect(done).not.toHaveBeenCalled() + held.resolve() + await Promise.all([pending, settled]) + expect(gate.needsReconciliation("file.ts")).toBe(false) + expect(gate.uncertainFiles(other.signal)).toEqual([]) + }) +}) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 1263aaef14..f400fd284f 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -1,205 +1,257 @@ +import * as path from "path" +import * as vscode from "vscode" import { ContextProxy } from "../../../core/config/ContextProxy" -import { makeExtensionContext } from "../../../test-utils/vscode" import { CodeIndexManager } from "../manager" -import { CodeIndexManagerRegistry } from "../code-index-manager-registry" +import { CodeIndexConfigManager } from "../config-manager" +import { CodeIndexStateManager } from "../state-manager" import { CodeIndexServiceFactory } from "../service-factory" import { CodeIndexOrchestrator } from "../orchestrator" import { CodeIndexSearchService } from "../search-service" -import type { MockedClass } from "vitest" -import * as path from "path" +import { CacheManager } from "../cache-manager" +import { GenerationAllocator } from "../generation-allocator" +import { clearAllMocks, restoreGlobals } from "../../../test-utils/reset" import { providerIdentifiers } from "@roo-code/types/provider-identifiers" +import { SembleProvider } from "../semble" +import { codeIndexLog } from "../code-index-log" +import { ParserFatalError } from "../parser-errors" +import { LATE_WRITE_GRACE_MS } from "../index-write-gate" +import fs from "fs/promises" +import type { IDirectoryScanner, IFileWatcher, IVectorStore } from "../interfaces" +import type { BatchProcessingSummary } from "../interfaces/file-processor" +import { ParseWorkerClient } from "../parse-worker-client" +import { retryWithPolicy } from "../shared/retry-policy" + +vi.mock("vscode", () => ({ + EventEmitter: class { + private listeners = new Set<(value: Value) => void>() + event = (listener: (value: Value) => void) => { + this.listeners.add(listener) + return { + dispose: () => { + this.listeners.delete(listener) + }, + } + } + fire(value: Value) { + this.listeners.forEach((listener) => listener(value)) + } + dispose() { + this.listeners.clear() + } + }, + Uri: { + file: (filePath: string) => ({ fsPath: filePath, toString: () => `file://${filePath}` }), + joinPath: (uri: { fsPath: string }, suffix: string) => ({ fsPath: `${uri.fsPath}/${suffix}` }), + }, + workspace: { workspaceFolders: [{ uri: { fsPath: "/workspace" } }] }, +})) +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: vi.fn().mockImplementation(() => ({ initialize: vi.fn().mockResolvedValue(undefined) })), +})) +vi.mock("fs/promises", () => ({ default: { readFile: vi.fn().mockResolvedValue("") } })) +vi.mock("ignore", () => ({ default: () => ({ add: vi.fn() }) })) +vi.mock("../code-index-log-host", () => ({ initializeCodeIndexHostLogging: vi.fn() })) +vi.mock("../service-factory") +vi.mock("../orchestrator") +vi.mock("../search-service") +vi.mock("../parse-worker-client") +vi.mock("../cache-manager") +vi.mock("../generation-allocator") +vi.mock("../semble", () => ({ SembleProvider: vi.fn() })) +vi.mock("../code-index-log", () => ({ + codeIndexLog: { + configure: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + error: vi.fn(), + redact: (value: string) => value.replaceAll("configured-secret", "[REDACTED]"), + }, +})) -// Helper: create a mock vscode.Uri from an fsPath -function mockUri(fsPath: string, scheme = "file") { - return { - fsPath, - scheme, - authority: "", - path: fsPath, - toString: (_skipEncoding?: boolean) => `${scheme}://${fsPath}`, - } +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } } -// Mock vscode module -vi.mock("vscode", () => { - const testPath = require("path") - const testWorkspacePath = testPath.join(testPath.sep, "test", "workspace") +async function drain() { + for (let iteration = 0; iteration < 30; iteration++) await Promise.resolve() +} + +function memento(): vscode.Memento { + const values = new Map() return { - Uri: { - file: (p: string) => ({ - fsPath: p, - scheme: "file", - authority: "", - path: p, - toString: (_skipEncoding?: boolean) => `file://${p}`, - }), - joinPath: vi.fn((...args: any[]) => ({ fsPath: args.join("/") })), + keys: () => [...values.keys()], + get(key: string, defaultValue?: Value): Value { + return (values.has(key) ? values.get(key) : defaultValue) as Value }, - window: { - activeTextEditor: null, + update: async (key: string, value: unknown) => { + values.set(key, value) }, - workspace: { - workspaceFolders: [ - { - uri: { - fsPath: testWorkspacePath, - scheme: "file", - authority: "", - path: testWorkspacePath, - toString: (_skipEncoding?: boolean) => `file://${testWorkspacePath}`, - }, - name: "test", - index: 0, - }, - ], - createFileSystemWatcher: vi.fn().mockReturnValue({ - onDidCreate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidChange: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidDelete: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - }), - getWorkspaceFolder: vi.fn(), - }, - RelativePattern: vi.fn().mockImplementation(function (base: any, pattern: any) { - return { base, pattern } - }), } -}) +} -// Mock only the essential dependencies -vi.mock("../../../utils/path", () => { - const testPath = require("path") - const testWorkspacePath = testPath.join(testPath.sep, "test", "workspace") +function extensionContext(): vscode.ExtensionContext { + // The manager and configuration reader use storage, secrets and extension paths only. return { - getWorkspacePath: vi.fn(() => testWorkspacePath), - } -}) - -// Mock fs/promises for RooIgnoreController -vi.mock("fs/promises", () => ({ - default: { - readFile: vi.fn().mockRejectedValue(new Error("File not found")), // Simulate no .gitignore/.rooignore - }, -})) - -// Mock file utils for RooIgnoreController -vi.mock("../../../utils/fs", () => ({ - fileExistsAtPath: vi.fn().mockResolvedValue(false), // Simulate no .rooignore file -})) - -// Mock ignore module -vi.mock("ignore", () => ({ - default: vi.fn().mockReturnValue({ - add: vi.fn(), - ignores: vi.fn().mockReturnValue(false), - }), -})) + workspaceState: memento(), + globalState: memento(), + subscriptions: [], + globalStorageUri: vscode.Uri.file("/storage"), + extensionPath: "/extension", + secrets: { get: vi.fn(), store: vi.fn(), delete: vi.fn() }, + } as unknown as vscode.ExtensionContext +} -vi.mock("../state-manager", () => ({ - CodeIndexStateManager: vi.fn().mockImplementation(function () { - return { - onProgressUpdate: vi.fn(), - getCurrentStatus: vi.fn(), - dispose: vi.fn(), - setSystemState: vi.fn(), - } - }), -})) +function configuration(context: vscode.ExtensionContext) { + const proxy = new ContextProxy(context) + const config = new CodeIndexConfigManager(proxy) + vi.spyOn(config, "isFeatureEnabled", "get").mockReturnValue(true) + vi.spyOn(config, "isFeatureConfigured", "get").mockReturnValue(true) + vi.spyOn(config, "currentEmbedderProvider", "get").mockReturnValue("openai") + vi.spyOn(config, "getConfig").mockReturnValue({ + isConfigured: true, + embedderProvider: providerIdentifiers.openai, + openAiOptions: { openAiNativeApiKey: "configured-secret" }, + }) + const load = vi.spyOn(config, "loadConfiguration").mockResolvedValue({ + requiresRestart: false, + configSnapshot, + currentConfig: config.getConfig(), + }) + return { config, proxy, load } +} -// Mock TelemetryService -vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), - }, - }, -})) +const configSnapshot = { + enabled: true, + configured: true, + embedderProvider: providerIdentifiers.openai, +} -vi.mock("../service-factory") -const MockedCodeIndexServiceFactory = CodeIndexServiceFactory as MockedClass +// Factory-only fixtures are never invoked; manager tests verify the dependency identities. +const services = { + embedder: { rateLimitScope: "embedding:test" }, + vectorStore: {}, + scanner: {}, + fileWatcher: { dispose: vi.fn() }, +} as unknown as Omit, "parser"> + +function semble() { + const provider = { + initialize: vi.fn().mockResolvedValue(undefined), + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn(), + searchIndex: vi.fn().mockResolvedValue([]), + clearIndexData: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn(), + state: "Indexed" as const, + } + // Semble's public lifecycle surface is sufficient; its CLI fields are not used by the manager. + vi.mocked(SembleProvider).mockReturnValue(provider as unknown as SembleProvider) + return provider +} -describe("CodeIndexManager - handleSettingsChange regression", () => { - let mockContext: any +describe("CodeIndexManager lifecycle", () => { let manager: CodeIndexManager - - // Define test paths for use in tests - const testWorkspacePath = path.join(path.sep, "test", "workspace") - const testExtensionPath = path.join(path.sep, "test", "extension") - const testStoragePath = path.join(path.sep, "test", "storage") - const testGlobalStoragePath = path.join(path.sep, "test", "global-storage") - const testLogPath = path.join(path.sep, "test", "log") + let state: CodeIndexStateManager + let context: vscode.ExtensionContext + let configured: ReturnType beforeEach(() => { - // Clear all instances before each test - CodeIndexManagerRegistry.disposeAll() - - const workspaceStateStore: Record = {} - const globalStateStore: Record = {} - mockContext = { - subscriptions: [], - workspaceState: { - get: vi.fn((key: string, defaultValue?: any) => workspaceStateStore[key] ?? defaultValue), - update: vi.fn(async (key: string, value: any) => { - workspaceStateStore[key] = value - }), - } as any, - globalState: { - get: vi.fn((key: string, defaultValue?: any) => globalStateStore[key] ?? defaultValue), - update: vi.fn(async (key: string, value: any) => { - globalStateStore[key] = value - }), - } as any, - extensionUri: {} as any, - extensionPath: testExtensionPath, - asAbsolutePath: vi.fn(), - storageUri: {} as any, - storagePath: testStoragePath, - globalStorageUri: {} as any, - globalStoragePath: testGlobalStoragePath, - logUri: {} as any, - logPath: testLogPath, - extensionMode: 3, // vscode.ExtensionMode.Test - secrets: {} as any, - environmentVariableCollection: {} as any, - extension: {} as any, - languageModelAccessInformation: {} as any, - } - - manager = CodeIndexManagerRegistry.getOrCreate(mockContext)! + clearAllMocks() + context = extensionContext() + state = new CodeIndexStateManager() + manager = new CodeIndexManager("/workspace", vscode.Uri.file("/workspace"), context, state) + configured = configuration(context) + manager["_configManager"] = configured.config + vi.mocked(fs.readFile).mockResolvedValue("") + vi.mocked(CacheManager.prototype.initialize).mockResolvedValue(undefined) + vi.mocked(CacheManager.prototype.getOpenFiles).mockReturnValue({}) + vi.mocked(CodeIndexServiceFactory.prototype.createServices).mockImplementation(() => ({ + ...services, + parser: manager["parser"]!, + })) + vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockResolvedValue({ valid: true }) + vi.mocked(CodeIndexOrchestrator).mockImplementation(() => orchestratorDouble()) + semble() }) + function orchestratorDouble(): CodeIndexOrchestrator { + // These tests exercise manager coordination, not the orchestrator's bounded-stop implementation. + return { + startupResult: Promise.resolve(), + stopped: false, + isScanning: false, + isActive: true, + isNaturallySettledPartial: false, + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + stopWatcher: vi.fn().mockResolvedValue(undefined), + clearIndexData: vi.fn().mockResolvedValue(undefined), + } as unknown as CodeIndexOrchestrator + } + + function replaceOnStart(replacement = orchestratorDouble()) { + const rebuild = vi.fn().mockImplementation(async () => { + manager["_orchestrator"] = replacement + manager["_searchService"] = new CodeIndexSearchService( + configured.config, + state, + services.embedder, + services.vectorStore, + manager.fileFailureRegistry, + manager.providerPacerRegistry, + ) + manager["servicesStopped"] = false + }) + manager["_recreateServices"] = rebuild + return { rebuild, replacement } + } + + function recordFailure(failureState: "retrying" | "exhausted" = "exhausted") { + manager.fileFailureRegistry.renew("bad.ts", "revision") + manager.fileFailureRegistry.record("bad.ts", "revision", "embed", "configured-secret failed", 1, failureState) + } + afterEach(() => { - CodeIndexManagerRegistry.disposeAll() + manager.dispose() + restoreGlobals() + vi.useRealTimers() }) describe("configuration readiness", () => { - it("does not regain readiness when a detached configuration finishes loading", async () => { - let finishRefresh!: () => void - const contextProxy = new ContextProxy(makeExtensionContext()) - vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) - vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) - vi.spyOn(contextProxy, "refreshSecrets").mockReturnValue( - new Promise((resolve) => { - finishRefresh = resolve - }), - ) + function unloadedConfiguration() { + manager["_configManager"] = undefined + const proxy = new ContextProxy(context) + vi.spyOn(proxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) + vi.spyOn(proxy, "getSecret").mockReturnValue(undefined) + return proxy + } - const initialization = manager.initialize(contextProxy) + it("does not regain readiness when a detached configuration finishes loading", async () => { + const proxy = unloadedConfiguration() + const refresh = deferred() + vi.spyOn(proxy, "refreshSecrets").mockReturnValue(refresh.promise) + const initialization = manager.initialize(proxy) expect(manager.isConfigurationLoaded).toBe(false) await manager.recoverFromError() - finishRefresh() + refresh.resolve() await initialization expect(manager.isConfigurationLoaded).toBe(false) + expect(CodeIndexServiceFactory).not.toHaveBeenCalled() }) it("keeps failed configuration unloaded until a settings refresh succeeds", async () => { - const error = new Error("secrets refresh failed") - const contextProxy = new ContextProxy(makeExtensionContext()) - vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) - vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) - vi.spyOn(contextProxy, "refreshSecrets").mockRejectedValueOnce(error).mockResolvedValue(undefined) - - await expect(manager.initialize(contextProxy)).rejects.toThrow(error) + const proxy = unloadedConfiguration() + vi.spyOn(proxy, "refreshSecrets") + .mockRejectedValueOnce(new Error("secrets refresh failed")) + .mockResolvedValue(undefined) + await expect(manager.initialize(proxy)).rejects.toThrow("secrets refresh failed") expect(manager.isConfigurationLoaded).toBe(false) await manager.handleSettingsChange() expect(manager.isConfigurationLoaded).toBe(true) @@ -207,652 +259,825 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { }) it("marks disabled configuration loaded only after secrets refresh completes", async () => { - let finishRefresh!: () => void - const refresh = new Promise((resolve) => { - finishRefresh = resolve - }) - const contextProxy = new ContextProxy(makeExtensionContext()) - vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) - vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) - vi.spyOn(contextProxy, "refreshSecrets").mockReturnValue(refresh) - + const proxy = unloadedConfiguration() + const refresh = deferred() + vi.spyOn(proxy, "refreshSecrets").mockReturnValue(refresh.promise) + const initialization = manager.initialize(proxy) + expect(proxy.refreshSecrets).toHaveBeenCalledOnce() expect(manager.isConfigurationLoaded).toBe(false) - const initialization = manager.initialize(contextProxy) - expect(contextProxy.refreshSecrets).toHaveBeenCalledOnce() - expect(manager.isConfigurationLoaded).toBe(false) - finishRefresh() + refresh.resolve() await initialization - expect(manager.isConfigurationLoaded).toBe(true) expect(manager.isFeatureEnabled).toBe(false) expect(manager.isInitialized).toBe(false) - await manager.recoverFromError() expect(manager.isConfigurationLoaded).toBe(false) }) }) - describe("handleSettingsChange", () => { - it("should log background indexing failures", async () => { - const indexingError = new Error("indexing startup failed") - const startIndexing = vi.fn().mockRejectedValue(indexingError) - const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined) - Object.defineProperty(manager, "_orchestrator", { - value: { startIndexing, stopIndexing: vi.fn() }, - configurable: true, + it.each(["initialize", "settings"] as const)( + "loads cache before allocator and recovers load-time grace through %s", + async (entry) => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + vi.mocked(CacheManager.prototype.initialize).mockImplementation(async function (this: CacheManager) { + this.loadedAt = Date.now() + vi.setSystemTime(105_000) }) - - manager["startIndexingInBackground"]() - await Promise.resolve() - - expect(consoleErrorSpy).toHaveBeenCalledWith( - "[CodeIndexManager] Background indexing failed:", - indexingError, + vi.mocked(CacheManager.prototype.getOpenFiles).mockReturnValue({ "open.ts": true }) + configured.load.mockResolvedValue({ + requiresRestart: true, + configSnapshot, + currentConfig: configured.config.getConfig(), + }) + if (entry === "initialize") await manager.initialize(configured.proxy) + else await manager.handleSettingsChange() + expect(manager["writeGate"].graceDeadline("open.ts")).toBe(100_000 + LATE_WRITE_GRACE_MS) + expect(vi.mocked(CacheManager.prototype.initialize).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(GenerationAllocator).mock.invocationCallOrder[0], ) - consoleErrorSpy.mockRestore() + }, + ) + + it("shares startup while rebuilding and resolves before a delayed scan", async () => { + const held = deferred() + const scan = deferred() + const { rebuild, replacement } = replaceOnStart() + vi.mocked(replacement.startIndexing).mockReturnValue(scan.promise) + rebuild.mockImplementation(async () => { + await held.promise + manager["_orchestrator"] = replacement }) + manager["_searchService"] = new CodeIndexSearchService( + configured.config, + state, + services.embedder, + services.vectorStore, + manager.fileFailureRegistry, + manager.providerPacerRegistry, + ) + const startup = manager.startIndexing() + expect(manager.startIndexing()).toBe(startup) + await drain() + expect(rebuild).toHaveBeenCalledOnce() + held.resolve() + await startup + expect(replacement.startIndexing).toHaveBeenCalledOnce() + scan.resolve() + }) - it("should not throw when called on uninitialized manager (regression test)", async () => { - // This is the core regression test: handleSettingsChange() should not throw - // when called before the manager is initialized (during first-time configuration) + it("joins only a live active scan's exact startup promise", () => { + const active = orchestratorDouble() + Object.defineProperty(active, "isScanning", { value: true }) + manager["_orchestrator"] = active + expect(manager.startIndexing()).toBe(active.startupResult) + expect(active.startIndexing).not.toHaveBeenCalled() + }) - // Ensure manager is not initialized - expect(manager.isInitialized).toBe(false) + it.each(["stopIndexing", "stopWatcher"] as const)( + "returns the exact shared %s promise without waiting for clear", + (stop) => { + const active = orchestratorDouble() + const shared = deferred() + vi.mocked(active[stop]).mockReturnValue(shared.promise) + manager["_orchestrator"] = active + manager["clearing"] = deferred().promise + expect(manager[stop]()).toBe(shared.promise) + expect(manager[stop]()).toBe(shared.promise) + shared.resolve() + }, + ) - // Mock a minimal config manager that simulates first-time configuration - const mockConfigManager = { - loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), - isFeatureConfigured: true, - isFeatureEnabled: true, - getConfig: vi.fn().mockReturnValue({ - isConfigured: true, - embedderProvider: providerIdentifiers.openai, - modelId: "text-embedding-3-small", - openAiOptions: { openAiNativeApiKey: "test-key" }, - qdrantUrl: "http://localhost:6333", - qdrantApiKey: "test-key", - searchMinScore: 0.4, - }), - } - ;(manager as any)._configManager = mockConfigManager + it("returns a shared idle stop without services", () => { + expect(manager.stopIndexing()).toBe(manager.stopWatcher()) + }) - // Mock cache manager - const mockCacheManager = { - initialize: vi.fn(), - clearCacheFile: vi.fn(), - } - ;(manager as any)._cacheManager = mockCacheManager - - // Mock the feature state to simulate valid configuration that would normally trigger restart - vi.spyOn(manager, "isFeatureEnabled", "get").mockReturnValue(true) - vi.spyOn(manager, "isFeatureConfigured", "get").mockReturnValue(true) - - // Mock service factory to handle _recreateServices call - const mockServiceFactoryInstance = { - configManager: mockConfigManager, - workspacePath: testWorkspacePath, - cacheManager: mockCacheManager, - createEmbedder: vi.fn().mockReturnValue({ embedderInfo: { name: "openai" } }), - createVectorStore: vi.fn().mockReturnValue({}), - createDirectoryScanner: vi.fn().mockReturnValue({}), - createFileWatcher: vi.fn().mockReturnValue({ - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - }), - createServices: vi.fn().mockReturnValue({ - embedder: { embedderInfo: { name: "openai" } }, - vectorStore: {}, - scanner: {}, - fileWatcher: { - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - }, - }), - validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), - } - MockedCodeIndexServiceFactory.mockImplementation(function () { - return mockServiceFactoryInstance as any + it.each(["stopIndexing", "stopWatcher", "disable", "feature-disable"] as const)( + "fences a late rejected rebuild after %s", + async (ending) => { + const held = deferred() + const { rebuild } = replaceOnStart() + rebuild.mockReturnValue(held.promise) + const startup = manager.startIndexing() + await drain() + if (ending === "disable") await manager.setWorkspaceEnabled(false) + else if (ending === "feature-disable") { + vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) + await manager.handleSettingsChange() + } else await manager[ending]() + held.reject(new Error("late rebuild")) + await expect(startup).resolves.toBeUndefined() + expect(manager.state).not.toBe("Error") + expect(codeIndexLog.error).not.toHaveBeenCalled() + }, + ) + + it.each(["rebuild", "startup"] as const)("reports and rejects sanitized %s failures", async (phase) => { + const { rebuild, replacement } = replaceOnStart() + if (phase === "rebuild") rebuild.mockRejectedValue(new Error("configured-secret failed")) + else + vi.mocked(replacement.startIndexing).mockImplementation(() => { + replacement.startupResult = Promise.reject(new Error("configured-secret failed")) + return Promise.resolve() }) + await expect(manager.startIndexing()).rejects.toThrow("configured-secret failed") + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Error", message: "[REDACTED] failed" }) + expect(manager["startInFlight"]).toBeUndefined() + }) - // The key test: this should NOT throw "CodeIndexManager not initialized" error - await expect(manager.handleSettingsChange()).resolves.not.toThrow() - - // Verify that loadConfiguration was called (the method should still work) - expect(mockConfigManager.loadConfiguration).toHaveBeenCalled() + it.each([true, false])("retries startup failure on settings save even if changed=%s", async (requiresRestart) => { + const { rebuild } = replaceOnStart() + rebuild.mockRejectedValueOnce(new Error("bad configuration")) + await expect(manager.startIndexing()).rejects.toThrow("bad configuration") + configured.load.mockResolvedValue({ + requiresRestart, + configSnapshot, + currentConfig: configured.config.getConfig(), }) + await manager.handleSettingsChange() + expect(rebuild).toHaveBeenCalledTimes(2) + expect(manager["startupFailureMessage"]).toBeUndefined() + }) - it("should work normally when manager is initialized", async () => { - // Mock a complete config manager with all required properties - const mockConfigManager = { - loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), - isFeatureConfigured: true, - isFeatureEnabled: true, - getConfig: vi.fn().mockReturnValue({ - isConfigured: true, - embedderProvider: providerIdentifiers.openai, - modelId: "text-embedding-3-small", - openAiOptions: { openAiNativeApiKey: "test-key" }, - qdrantUrl: "http://localhost:6333", - qdrantApiKey: "test-key", - searchMinScore: 0.4, - }), - } - ;(manager as any)._configManager = mockConfigManager - - // Mock cache manager - const mockCacheManager = { - initialize: vi.fn(), - clearCacheFile: vi.fn(), - } - ;(manager as any)._cacheManager = mockCacheManager - - // Simulate an initialized manager by setting the required properties - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } - ;(manager as any)._searchService = {} - - // Verify manager is considered initialized - expect(manager.isInitialized).toBe(true) - - // Mock the feature state - vi.spyOn(manager, "isFeatureEnabled", "get").mockReturnValue(true) - vi.spyOn(manager, "isFeatureConfigured", "get").mockReturnValue(true) - - // Mock service factory to handle _recreateServices call - const mockServiceFactoryInstance = { - configManager: mockConfigManager, - workspacePath: testWorkspacePath, - cacheManager: mockCacheManager, - createEmbedder: vi.fn().mockReturnValue({ embedderInfo: { name: "openai" } }), - createVectorStore: vi.fn().mockReturnValue({}), - createDirectoryScanner: vi.fn().mockReturnValue({}), - createFileWatcher: vi.fn().mockReturnValue({ - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - }), - createServices: vi.fn().mockReturnValue({ - embedder: { embedderInfo: { name: "openai" } }, - vectorStore: {}, - scanner: {}, - fileWatcher: { - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - }, - }), - validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), - } - MockedCodeIndexServiceFactory.mockImplementation(function () { - return mockServiceFactoryInstance as any + it("drains a settings rebuild before scanning and shares Start throughout", async () => { + const held = deferred() + const secondHeld = deferred() + const { rebuild, replacement } = replaceOnStart() + rebuild + .mockImplementationOnce(async () => { + await held.promise + manager["_orchestrator"] = replacement }) - - // Mock the methods that would be called during restart - const recreateServicesSpy = vi.spyOn(manager as any, "_recreateServices") - - await manager.handleSettingsChange() - - // Verify that the restart sequence was called - expect(mockConfigManager.loadConfiguration).toHaveBeenCalled() - // _recreateServices should be called when requiresRestart is true - expect(recreateServicesSpy).toHaveBeenCalled() - // Note: startIndexing is NOT called by handleSettingsChange - it's only called by initialize() - }) - - it("should handle case when config manager is not set", async () => { - // Ensure config manager is not set (edge case) - ;(manager as any)._configManager = undefined - - // This should not throw an error - await expect(manager.handleSettingsChange()).resolves.not.toThrow() + .mockImplementationOnce(async () => { + await secondHeld.promise + manager["_orchestrator"] = replacement + }) + manager["_searchService"] = new CodeIndexSearchService( + configured.config, + state, + services.embedder, + services.vectorStore, + manager.fileFailureRegistry, + manager.providerPacerRegistry, + ) + const startup = manager.startIndexing() + await drain() + configured.load.mockResolvedValue({ + requiresRestart: true, + configSnapshot, + currentConfig: configured.config.getConfig(), }) + const settings = manager.handleSettingsChange() + await drain() + held.resolve() + await drain() + expect(rebuild).toHaveBeenCalledTimes(2) + expect(manager.startIndexing()).toBe(startup) + expect(replacement.startIndexing).not.toHaveBeenCalled() + secondHeld.resolve() + await Promise.all([startup, settings]) + expect(replacement.startIndexing).toHaveBeenCalledOnce() }) - describe("embedder validation integration", () => { - let mockServiceFactoryInstance: any - let mockStateManager: any - let mockEmbedder: any - let mockVectorStore: any - let mockScanner: any - let mockFileWatcher: any - - beforeEach(() => { - // Mock service factory objects - mockEmbedder = { embedderInfo: { name: "openai" } } - mockVectorStore = {} - mockScanner = {} - mockFileWatcher = { - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - } - - // Mock service factory instance - mockServiceFactoryInstance = { - createServices: vi.fn().mockReturnValue({ - embedder: mockEmbedder, - vectorStore: mockVectorStore, - scanner: mockScanner, - fileWatcher: mockFileWatcher, - }), - validateEmbedder: vi.fn(), - } - - // Mock the ServiceFactory constructor - MockedCodeIndexServiceFactory.mockImplementation(function () { - return mockServiceFactoryInstance - }) - - // Mock state manager methods directly on the existing instance - mockStateManager = (manager as any)._stateManager - mockStateManager.setSystemState = vi.fn() - - // Mock config manager - const mockConfigManager = { - loadConfiguration: vitest.fn().mockResolvedValue({ requiresRestart: false }), - isFeatureConfigured: true, - isFeatureEnabled: true, - getConfig: vitest.fn().mockReturnValue({ - isConfigured: true, - embedderProvider: providerIdentifiers.openai, - modelId: "text-embedding-3-small", - openAiOptions: { openAiNativeApiKey: "test-key" }, - qdrantUrl: "http://localhost:6333", - qdrantApiKey: "test-key", - searchMinScore: 0.4, - }), - } - ;(manager as any)._configManager = mockConfigManager + it.each(["stop", "fatal"] as const)("does not drain pending settings after %s", async (ending) => { + const held = deferred() + const { rebuild, replacement } = replaceOnStart() + rebuild.mockImplementation(async () => { + await held.promise + manager["_orchestrator"] = replacement }) - - it("should create indexing services without a startup embedder validation request", async () => { - mockServiceFactoryInstance.validateEmbedder.mockResolvedValue({ - valid: false, - error: "Embedder unavailable", - }) - - await manager["_recreateServices"]() - - expect(mockServiceFactoryInstance.createServices).toHaveBeenCalled() - expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() - expect(mockStateManager.setSystemState).not.toHaveBeenCalledWith("Error", expect.any(String)) - expect(manager["_orchestrator"]).toBeInstanceOf(CodeIndexOrchestrator) - expect(manager["_searchService"]).toBeInstanceOf(CodeIndexSearchService) + manager["_searchService"] = new CodeIndexSearchService( + configured.config, + state, + services.embedder, + services.vectorStore, + manager.fileFailureRegistry, + manager.providerPacerRegistry, + ) + const startup = manager.startIndexing() + await drain() + configured.load.mockResolvedValue({ + requiresRestart: true, + configSnapshot, + currentConfig: configured.config.getConfig(), }) + const settings = manager.handleSettingsChange() + await drain() + if (ending === "stop") await manager.stopIndexing() + else manager["parserFailed"] = true + held.resolve() + await Promise.all([startup, settings]) + expect(rebuild).toHaveBeenCalledOnce() + expect(replacement.startIndexing).not.toHaveBeenCalled() + }) - it("should handle embedder creation failure", async () => { - // Arrange - mockServiceFactoryInstance.createServices.mockImplementation(() => { - throw new Error("Invalid configuration") - }) + it("blocks automatic paths on fatal failure but explicit retry rebuilds without recoverFromError", async () => { + manager["parserFailed"] = true + manager["parserFailureMessage"] = "fatal" + const { rebuild, replacement } = replaceOnStart() + const recovery = vi.spyOn(manager, "recoverFromError") + await manager.initialize(configured.proxy) + await manager.handleSettingsChange() + expect(rebuild).not.toHaveBeenCalled() + await manager.startIndexing() + expect(recovery).not.toHaveBeenCalled() + expect(manager["parserFailed"]).toBe(false) + expect(replacement.startIndexing).toHaveBeenCalledOnce() + }) - // Act & Assert - should throw the error - await expect((manager as any)._recreateServices()).rejects.toThrow("Invalid configuration") + it.each(["resolve", "reject"] as const)( + "waits for clear to %s before recreation, without invalidating the clearing owner", + async (ending) => { + await manager.startIndexing() + const old = manager["_orchestrator"]! + const held = deferred() + vi.mocked(old.clearIndexData).mockReturnValue(held.promise) + const clearing = manager.clearIndexData().catch(() => {}) + const oldAttempt = manager["currentAttempt"] + const created = vi.mocked(CodeIndexServiceFactory).mock.calls.length + const startup = manager.startIndexing() + await drain() + expect(manager["currentAttempt"]).toBe(oldAttempt) + expect(vi.mocked(CodeIndexServiceFactory).mock.calls).toHaveLength(created) + if (ending === "resolve") held.resolve() + else held.reject(new Error("delete failed")) + await Promise.all([clearing, startup]) + expect(vi.mocked(CodeIndexServiceFactory).mock.calls).toHaveLength(created + 1) + }, + ) + + it("retains workspace dependencies, retires the old owner, and fences stale fatal callbacks", async () => { + await manager.startIndexing() + const first = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! + const factory = vi.mocked(CodeIndexServiceFactory).mock.calls.at(-1)! + const old = manager["_orchestrator"]! + const parser = manager["parser"]! + await manager.startIndexing() + const second = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! + expect(second[8]).toBe(first[8]) + expect(second[9]).toBe(first[9]) + expect(second[10]).toBe(manager.fileFailureRegistry) + expect(factory[6]).toBe(first[7]) + expect(second[7]).not.toBe(parser) + expect(parser.stop).toHaveBeenCalled() + expect(old.stopWatcher).toHaveBeenCalled() + expect(first[11]!()).toBe(false) + first[12]!(new ParserFatalError("parser-abort", "old.ts"), "stale failure") + expect(manager["parserFailed"]).toBe(false) + expect(CodeIndexServiceFactory.prototype.validateEmbedder).toHaveBeenCalledTimes(2) + expect(factory[8]).toBe(manager.providerPacerRegistry) + expect(first[13]).toBe(manager.providerPacerRegistry) + expect(second[13]).toBe(first[13]) + }) - // Should not attempt validation if embedder creation fails - expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() - }) + it("aborts real watcher retry backoff when explicit Start replaces its owner", async () => { + vi.useFakeTimers() + await manager.startIndexing() + const { CodeIndexOrchestrator: ActualOrchestrator } = + await vi.importActual("../orchestrator") + const batchStarted = new vscode.EventEmitter() + const batchProgress = new vscode.EventEmitter<{ + processedInBatch: number + totalInBatch: number + currentFile?: string + }>() + const batchFinished = new vscode.EventEmitter() + let watcherSignal!: AbortSignal + const watcher: IFileWatcher = { + initialize: vi.fn(async (signal?: AbortSignal) => { + watcherSignal = signal! + }), + dispose: vi.fn(), + processFile: vi.fn(), + onDidStartBatchProcessing: batchStarted.event, + onBatchProgressUpdate: batchProgress.event, + onDidFinishBatchProcessing: batchFinished.event, + } + const scanner: IDirectoryScanner = { + scanDirectory: vi.fn().mockResolvedValue({ + stats: { processed: 0, skipped: 0 }, + totalBlockCount: 0, + }), + } + // Only startup and completeness writes are used; no vector search or file writes occur. + const store = { + initialize: vi.fn().mockResolvedValue(false), + hasIndexedData: vi.fn().mockResolvedValue(false), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + } as unknown as IVectorStore + // The real orchestrator only retires this worker and waits for physical exit. + const parser = { + stop: vi.fn(), + stopped: Promise.resolve(), + logStopTimeout: vi.fn(), + } as unknown as ParseWorkerClient + const attempt = manager["currentAttempt"] + const active = new ActualOrchestrator( + configured.config, + state, + "/workspace", + manager["_cacheManager"]!, + store, + scanner, + watcher, + parser, + manager["writeGate"], + manager["generationAllocator"]!, + manager.fileFailureRegistry, + () => manager["currentAttempt"] === attempt, + ) + manager["_orchestrator"] = active + await active.startIndexing() + batchStarted.fire(["bad.ts"]) + expect(active.isScanning).toBe(false) + expect(manager.state).toBe("Indexing") + const registry = manager.fileFailureRegistry + registry.renew("bad.ts", "revision") + const operation = vi + .fn<() => Promise>() + .mockRejectedValue(Object.assign(new Error("unavailable"), { status: 503 })) + const retry = retryWithPolicy(operation, { + signal: watcherSignal, + budget: registry.getRetryBudget("bad.ts", "revision", "embed"), + onFailure: (failure) => + registry.record( + "bad.ts", + "revision", + "embed", + "unavailable", + failure.attempt, + failure.willRetry ? "retrying" : "exhausted", + failure.classification, + ), + }).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + expect(registry.get("bad.ts")).toMatchObject({ state: "retrying", attempt: 1 }) + const startup = manager.startIndexing() + expect(manager.startIndexing()).toBe(startup) + await startup + expect(await retry).toBe(watcherSignal.reason) + expect(watcherSignal.aborted).toBe(true) + expect(watcher.dispose).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(2_000) + expect(operation).toHaveBeenCalledOnce() + expect(registry.getFailure("bad.ts", "revision", "embed")).toBeUndefined() + batchStarted.dispose() + batchProgress.dispose() + batchFinished.dispose() }) - describe("recoverFromError", () => { - let mockConfigManager: any - let mockCacheManager: any - let mockStateManager: any - - beforeEach(() => { - // Mock config manager - mockConfigManager = { - loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: false }), - isFeatureConfigured: true, - isFeatureEnabled: true, - getConfig: vi.fn().mockReturnValue({ - isConfigured: true, - embedderProvider: providerIdentifiers.openai, - modelId: "text-embedding-3-small", - openAiOptions: { openAiNativeApiKey: "test-key" }, - qdrantUrl: "http://localhost:6333", - qdrantApiKey: "test-key", - searchMinScore: 0.4, - }), - } - ;(manager as any)._configManager = mockConfigManager + it("retains failure budgets on automatic rebuild and resets them only on explicit start or re-enable", async () => { + await manager.startIndexing() + recordFailure() + const registry = manager.fileFailureRegistry + const reset = vi.spyOn(registry, "resetAllBudgets") + configured.load.mockResolvedValue({ + requiresRestart: true, + configSnapshot, + currentConfig: configured.config.getConfig(), + }) + await manager.handleSettingsChange() + expect(reset).not.toHaveBeenCalled() + expect(registry.hasRevisionFailure("bad.ts", "revision")).toBe(true) + await manager.startIndexing() + expect(reset).toHaveBeenCalledOnce() + await manager.setWorkspaceEnabled(false) + await manager.setWorkspaceEnabled(true) + expect(reset).toHaveBeenCalledTimes(2) + expect(manager.fileFailureRegistry).toBe(registry) + }) - // Mock cache manager - mockCacheManager = { - initialize: vi.fn(), - clearCacheFile: vi.fn(), - } - ;(manager as any)._cacheManager = mockCacheManager - - // Mock state manager - mockStateManager = (manager as any)._stateManager - mockStateManager.setSystemState = vi.fn() - mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ - systemStatus: "Error", - message: "Failed during initial scan: fetch failed", - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", + it.each([ + ["Error", false, false, false, "Error"], + ["Stopping", false, false, false, "Stopping"], + ["Standby", true, false, false, "Indexing"], + ["Standby", false, true, false, "Standby"], + ["Indexed", false, false, false, "Indexed"], + ["Indexed", false, false, true, "Standby"], + ] as const)( + "computes precedence from %s scanning=%s partial=%s stopped=%s", + (raw, scanning, partial, stopped, expected) => { + const active = orchestratorDouble() + Object.defineProperties(active, { + isScanning: { value: scanning }, + isNaturallySettledPartial: { value: partial }, }) + manager["_orchestrator"] = active + manager["servicesStopped"] = stopped + state.setSystemState(raw, "original") + expect(manager.state).toBe(expected) + expect(manager.getCurrentStatus().systemStatus).toBe(expected) + }, + ) - // Mock orchestrator and search service to simulate initialized state - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn(), state: "Error" } - ;(manager as any)._searchService = {} - ;(manager as any)._serviceFactory = {} - }) - - it("should clear error state when recoverFromError is called", async () => { - // Act - await manager.recoverFromError() + it("does not render an idle live watcher as active indexing", () => { + manager["_orchestrator"] = orchestratorDouble() + state.setSystemState("Indexed", "ready") + expect(manager.state).toBe("Indexed") + }) - // Assert - expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "") + it("publishes state and sanitized registry updates and preserves stop messages", async () => { + const update = vi.fn() + manager.onProgressUpdate(update) + state.setSystemState("Indexed", "ready") + expect(update).toHaveBeenLastCalledWith(expect.objectContaining({ systemStatus: "Indexed" })) + recordFailure("retrying") + expect(update).toHaveBeenLastCalledWith( + expect.objectContaining({ + systemStatus: "Indexing", + failedFiles: [expect.objectContaining({ reason: "[REDACTED] failed" })], + }), + ) + await manager.stopIndexing() + state.setSystemState("Standby", "user stopped") + expect(manager.getCurrentStatus()).toMatchObject({ + systemStatus: "Standby", + message: "user stopped", + failedFiles: [expect.objectContaining({ state: "exhausted" })], }) + state.setSystemState("Stopping", "stopping") + manager["parserFailed"] = true + manager["parserFailureMessage"] = "configured-secret fatal" + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Error", message: "[REDACTED] fatal" }) + }) - it("should reset internal service instances", async () => { - // Verify initial state - expect((manager as any)._configManager).toBeDefined() - expect((manager as any)._serviceFactory).toBeDefined() - expect((manager as any)._orchestrator).toBeDefined() - expect((manager as any)._searchService).toBeDefined() - - // Act - await manager.recoverFromError() + it("forwards search cancellation and injects registry plus computed status", async () => { + await manager.startIndexing() + const search = manager["_searchService"]! + const response = { results: [], incomplete: true, failedPaths: ["bad.ts"], waitingPaths: [] } + vi.mocked(search.searchIndex).mockResolvedValue(response) + const controller = new AbortController() + expect(await manager.searchIndex("query", "src", controller.signal)).toBe(response) + expect(search.searchIndex).toHaveBeenCalledWith("query", "src", controller.signal) + const argumentsUsed = vi.mocked(CodeIndexSearchService).mock.calls.at(-1)! + expect(argumentsUsed[4]).toBe(manager.fileFailureRegistry) + manager["parserFailed"] = true + expect(argumentsUsed[5]).toBe(manager.providerPacerRegistry) + expect(argumentsUsed[6]!().systemStatus).toBe("Error") + }) - // Assert - all service instances should be undefined - expect((manager as any)._configManager).toBeUndefined() - expect((manager as any)._serviceFactory).toBeUndefined() - expect((manager as any)._orchestrator).toBeUndefined() - expect((manager as any)._searchService).toBeUndefined() + it("returns a failure envelope while disabled", async () => { + vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) + await expect(manager.searchIndex("query")).resolves.toEqual({ + results: [], + incomplete: false, + failedPaths: [], + waitingPaths: [], }) - - it("should make manager report as not initialized after recovery", async () => { - // Verify initial state - expect(manager.isInitialized).toBe(true) - - // Act - await manager.recoverFromError() - - // Assert - expect(manager.isInitialized).toBe(false) + recordFailure() + await expect(manager.searchIndex("query")).resolves.toEqual({ + results: [], + incomplete: true, + failedPaths: ["bad.ts"], + waitingPaths: [], }) + }) - it("should allow re-initialization after recovery", async () => { - // Setup mock for re-initialization - const mockServiceFactoryInstance = { - createServices: vi.fn().mockReturnValue({ - embedder: { embedderInfo: { name: "openai" } }, - vectorStore: {}, - scanner: {}, - fileWatcher: { - onDidStartBatchProcessing: vi.fn(), - onBatchProgressUpdate: vi.fn(), - watch: vi.fn(), - stopWatcher: vi.fn(), - dispose: vi.fn(), - }, - }), - validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), - } - MockedCodeIndexServiceFactory.mockImplementation(function () { - return mockServiceFactoryInstance as any - }) - - // Act - recover from error - await manager.recoverFromError() - - // Verify manager is not initialized - expect(manager.isInitialized).toBe(false) - - // Mock context proxy for initialization - const mockContextProxy = { - getValue: vi.fn(), - setValue: vi.fn(), - storeSecret: vi.fn(), - getSecret: vi.fn(), - refreshSecrets: vi.fn().mockResolvedValue(undefined), - getGlobalState: vi.fn().mockReturnValue({ - codebaseIndexEnabled: true, - codebaseIndexQdrantUrl: "http://localhost:6333", - codebaseIndexEmbedderProvider: providerIdentifiers.openai, - codebaseIndexEmbedderModelId: "text-embedding-3-small", - codebaseIndexEmbedderModelDimension: 1536, - codebaseIndexSearchMaxResults: 10, - codebaseIndexSearchMinScore: 0.4, - }), - } - - // Enable workspace indexing before re-initialization - await manager.setWorkspaceEnabled(true) - - // Re-initialize - await manager.initialize(mockContextProxy as any) - - // Assert - manager should be initialized again - expect(manager.isInitialized).toBe(true) - expect(mockServiceFactoryInstance.createServices).toHaveBeenCalled() - expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() + it("starts newly recreated Semble through the shared helper and ignores query cancellation", async () => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const provider = semble() + await manager.initialize(configured.proxy) + expect(provider.startIndexing).toHaveBeenCalledOnce() + const controller = new AbortController() + controller.abort() + await expect(manager.searchIndex("query", "src", controller.signal)).resolves.toEqual({ + results: [], + incomplete: false, + failedPaths: [], + waitingPaths: [], }) + expect(provider.searchIndex).toHaveBeenCalledWith("query", "src") + }) - it("should be safe to call when not in error state (idempotent)", async () => { - // Setup manager in non-error state - mockStateManager.getCurrentStatus.mockReturnValue({ - systemStatus: "Standby", - message: "", - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", + it.each(["repeat", "stopIndexing", "stopWatcher", "clear"] as const)( + "reuses existing Semble and restores status forwarding after %s", + async (action) => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const provider = semble() + await manager.initialize(configured.proxy) + const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] + provider.startIndexing.mockImplementation(async () => { + providerState.setSystemState("Indexed", "Semble ready") }) + if (action === "clear") await manager.clearIndexData() + else if (action !== "repeat") await manager[action]() + recordFailure() + const reset = vi.spyOn(manager.fileFailureRegistry, "resetAllBudgets") + const rebuild = vi.fn() + manager["_recreateServices"] = rebuild + const startup = manager.startIndexing(configured.proxy) + expect(manager.startIndexing()).toBe(startup) + await startup + expect(SembleProvider).toHaveBeenCalledOnce() + expect(provider.initialize).toHaveBeenCalledOnce() + expect(provider.dispose).not.toHaveBeenCalled() + expect(provider.startIndexing).toHaveBeenCalledTimes(2) + expect(rebuild).not.toHaveBeenCalled() + expect(reset).not.toHaveBeenCalled() + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Indexed", message: "Semble ready" }) + }, + ) + + it.each(["workspace", "feature"] as const)("does not start existing Semble when %s is disabled", async (scope) => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const provider = semble() + await manager.initialize(configured.proxy) + if (scope === "workspace") await manager.setWorkspaceEnabled(false) + else vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) + await manager.startIndexing() + expect(provider.startIndexing).toHaveBeenCalledOnce() + expect(SembleProvider).toHaveBeenCalledOnce() + }) - // Verify initial state is not error - const initialStatus = manager.getCurrentStatus() - expect(initialStatus.systemStatus).not.toBe("Error") - - // Act - call recoverFromError when not in error state - await expect(manager.recoverFromError()).resolves.not.toThrow() - - // Assert - should still clear state and service instances - expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "") - expect((manager as any)._configManager).toBeUndefined() - expect((manager as any)._serviceFactory).toBeUndefined() - expect((manager as any)._orchestrator).toBeUndefined() - expect((manager as any)._searchService).toBeUndefined() + it("retries an existing Semble startup failure without rebuilding", async () => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const provider = semble() + await manager.initialize(configured.proxy) + provider.startIndexing.mockRejectedValueOnce(new Error("Semble unavailable")) + await expect(manager.startIndexing()).rejects.toThrow("Semble unavailable") + expect(manager.state).toBe("Error") + const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] + provider.startIndexing.mockImplementation(async () => { + providerState.setSystemState("Indexed", "Semble recovered") }) + await manager.startIndexing() + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Indexed", message: "Semble recovered" }) + expect(SembleProvider).toHaveBeenCalledOnce() + expect(provider.dispose).not.toHaveBeenCalled() + }) - it("should continue recovery even if setSystemState throws", async () => { - // Setup state manager to throw on setSystemState - mockStateManager.setSystemState.mockImplementation(() => { - throw new Error("State update failed") - }) - - // Setup manager with service instances - ;(manager as any)._configManager = mockConfigManager - ;(manager as any)._serviceFactory = {} - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } - ;(manager as any)._searchService = {} - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) - - // Act - should not throw despite setSystemState error - await expect(manager.recoverFromError()).resolves.not.toThrow() - - // Assert - error should be logged - expect(consoleErrorSpy).toHaveBeenCalledWith( - "Failed to clear error state during recovery:", - expect.any(Error), - ) - - // Assert - service instances should still be cleared - expect((manager as any)._configManager).toBeUndefined() - expect((manager as any)._serviceFactory).toBeUndefined() - expect((manager as any)._orchestrator).toBeUndefined() - expect((manager as any)._searchService).toBeUndefined() - - // Cleanup - consoleErrorSpy.mockRestore() - }) + it("waits for Semble clear before restarting the existing provider", async () => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const provider = semble() + await manager.initialize(configured.proxy) + const held = deferred() + provider.clearIndexData.mockReturnValue(held.promise) + const clearing = manager.clearIndexData() + const startup = manager.startIndexing() + await drain() + expect(provider.startIndexing).toHaveBeenCalledOnce() + await manager.stopIndexing() + held.resolve() + await Promise.all([clearing, startup]) + expect(provider.startIndexing).toHaveBeenCalledOnce() }) - describe("workspace-enabled gating", () => { - it("should not start indexing when workspace is not enabled", async () => { - await manager.setAutoEnableDefault(false) + it("recreates Semble after initialization was retired by stop", async () => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const held = deferred() + const retired = semble() + retired.initialize.mockReturnValue(held.promise) + const startup = manager.startIndexing() + await drain() + await manager.stopIndexing() + held.resolve() + await startup + const replacement = semble() + await manager.startIndexing() + expect(retired.dispose).toHaveBeenCalledOnce() + expect(retired.startIndexing).not.toHaveBeenCalled() + expect(replacement.initialize).toHaveBeenCalledOnce() + expect(replacement.startIndexing).toHaveBeenCalledOnce() + }) - const mockStateManager = (manager as any)._stateManager - mockStateManager.setSystemState = vi.fn() - mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ - systemStatus: "Standby", - message: "", - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", - }) + it.each(["before-create", "during-initialize"] as const)("fences Semble after stop %s", async (phase) => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const held = deferred() + const provider = semble() + if (phase === "before-create") manager["clearing"] = held.promise + else provider.initialize.mockReturnValue(held.promise) + const startup = manager.startIndexing() + await drain() + await manager.stopIndexing() + held.resolve() + await startup + expect(provider.startIndexing).not.toHaveBeenCalled() + if (phase === "before-create") expect(SembleProvider).not.toHaveBeenCalled() + else expect(provider.dispose).toHaveBeenCalled() + }) - expect(manager.isWorkspaceEnabled).toBe(false) + it("fences late Semble status publications", async () => { + vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") + const held = deferred() + semble().initialize.mockReturnValue(held.promise) + const startup = manager.startIndexing() + await drain() + const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] + await manager.stopIndexing() + providerState.setSystemState("Error", "stale download failure") + held.resolve() + await startup + expect(manager.state).not.toBe("Error") + }) - await manager.startIndexing() + it("recoverFromError retires the old attempt and preserves the fatal latch and cache", async () => { + await manager.startIndexing() + const cache = manager["_cacheManager"] + const old = manager["_orchestrator"]! + const attempt = manager["currentAttempt"] + manager["parserFailed"] = true + await manager.recoverFromError() + expect(old.stopWatcher).toHaveBeenCalled() + expect(manager["currentAttempt"]).toBeGreaterThan(attempt) + expect(manager["parserFailed"]).toBe(true) + expect(manager["_cacheManager"]).toBe(cache) + expect(manager.isInitialized).toBe(false) + expect(manager.isConfigurationLoaded).toBe(false) + }) - expect(mockStateManager.setSystemState).not.toHaveBeenCalledWith("Indexing", expect.any(String)) + it("continues recovery when publishing Standby throws", async () => { + await manager.startIndexing() + vi.spyOn(state, "setSystemState").mockImplementation(() => { + throw new Error("publication failed") }) + await expect(manager.recoverFromError()).resolves.toBeUndefined() + expect(manager["_configManager"]).toBeUndefined() + expect(manager["_orchestrator"]).toBeUndefined() + }) - it("should include workspaceEnabled in getCurrentStatus", async () => { - await manager.setAutoEnableDefault(false) - - const mockStateManager = (manager as any)._stateManager - mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ - systemStatus: "Standby", - message: "", - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", + it("aggregates actual configured secrets across workspace owners", async () => { + const otherContext = extensionContext() + const otherState = new CodeIndexStateManager() + const other = new CodeIndexManager("/other", vscode.Uri.file("/other"), otherContext, otherState) + try { + other["_configManager"] = configuration(otherContext).config + await manager.initialize(configured.proxy, false) + expect(codeIndexLog.configure).toHaveBeenLastCalledWith({ + workspacePaths: ["/workspace", "/other"], + secrets: ["configured-secret", "configured-secret"], }) + } finally { + other.dispose() + } + }) - const status = manager.getCurrentStatus() - expect(status.workspaceEnabled).toBe(false) - }) - - it("should persist workspace enabled state", async () => { - await manager.setAutoEnableDefault(false) - expect(manager.isWorkspaceEnabled).toBe(false) - - await manager.setWorkspaceEnabled(true) - expect(manager.isWorkspaceEnabled).toBe(true) - + it("keeps folder URI enablement distinct and honors auto-enable false", async () => { + const other = new CodeIndexManager( + "/workspace", + { ...vscode.Uri.file("/workspace"), toString: () => "remote:///workspace" }, + context, + new CodeIndexStateManager(), + ) + try { await manager.setWorkspaceEnabled(false) - expect(manager.isWorkspaceEnabled).toBe(false) - }) - - it("should store enablement per folder URI, not per window", async () => { - CodeIndexManagerRegistry.disposeAll() - - const vscode = await import("vscode") - - const folderAPath = path.join(path.sep, "test", "folderA") - const folderBPath = path.join(path.sep, "test", "folderB") - const folderAUri = mockUri(folderAPath) - const folderBUri = mockUri(folderBPath) - - // Both folders share the same workspaceState (same window) - const sharedStore: Record = {} - const sharedContext = { - ...mockContext, - workspaceState: { - get: vi.fn((key: string, defaultValue?: any) => sharedStore[key] ?? defaultValue), - update: vi.fn(async (key: string, value: any) => { - sharedStore[key] = value - }), - } as any, - globalState: { - get: vi.fn((_key: string, _defaultValue?: any) => false), - update: vi.fn(), - } as any, - } - - // Patch workspaceFolders to include both folders - ;(vscode.workspace as any).workspaceFolders = [ - { uri: folderAUri, name: "folderA", index: 0 }, - { uri: folderBUri, name: "folderB", index: 1 }, - ] - - const managerA = CodeIndexManagerRegistry.getOrCreate(sharedContext, folderAPath)! - const managerB = CodeIndexManagerRegistry.getOrCreate(sharedContext, folderBPath)! - - // Both start disabled (autoEnableDefault is false via globalState mock) - expect(managerA.isWorkspaceEnabled).toBe(false) - expect(managerB.isWorkspaceEnabled).toBe(false) - - // Enable A only - await managerA.setWorkspaceEnabled(true) - - expect(managerA.isWorkspaceEnabled).toBe(true) - expect(managerB.isWorkspaceEnabled).toBe(false) - - // Enable B, disable A - await managerB.setWorkspaceEnabled(true) - await managerA.setWorkspaceEnabled(false) - - expect(managerA.isWorkspaceEnabled).toBe(false) - expect(managerB.isWorkspaceEnabled).toBe(true) + expect(other.isWorkspaceEnabled).toBe(true) + await manager.setAutoEnableDefault(false) + expect(other.isWorkspaceEnabled).toBe(false) + } finally { + other.dispose() + } + }) - CodeIndexManagerRegistry.disposeAll() + it("does not construct external services after stop during ignore loading", async () => { + const held = deferred() + vi.mocked(fs.readFile).mockImplementationOnce(async () => { + await held.promise + return "" }) + const startup = manager.startIndexing() + await drain() + await manager.stopWatcher() + held.resolve() + await startup + expect(CodeIndexServiceFactory).not.toHaveBeenCalled() }) - describe("stopIndexing", () => { - it("should delegate to orchestrator.stopIndexing()", () => { - const mockOrchestrator = { - stopIndexing: vi.fn(), - stopWatcher: vi.fn(), - state: "Indexing", - } - ;(manager as any)._orchestrator = mockOrchestrator - - manager.stopIndexing() + it.each(["paused", "429"])( + "defers %s startup validation and retains one registry across restarts", + async (kind) => { + vi.useFakeTimers() + const { EmbeddingError } = await import("../shared/indexing-errors") + const registry = manager.providerPacerRegistry + const pacer = registry.get(services.embedder.rateLimitScope) + const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }) + const validate = vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder) + if (kind === "paused") pacer.reportRateLimit(error, pacer.markIssued()) + else validate.mockRejectedValueOnce(error) + await manager.startIndexing() + const argumentsUsed = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! + expect(argumentsUsed[13]).toBe(registry) + expect(argumentsUsed[14]).toBeTypeOf("function") + expect(validate).toHaveBeenCalledTimes(kind === "paused" ? 0 : 1) + const controller = new AbortController() + const waiting = argumentsUsed[14]!(controller.signal).catch((failure: unknown) => failure) + await drain() + expect(manager.state).toBe("Indexing") + expect(manager.getCurrentStatus().providerWaits).toEqual([ + expect.objectContaining({ resumeAt: Date.now() + 61_000, waitingFiles: [] }), + ]) + controller.abort() + await expect(waiting).resolves.toBe(controller.signal.reason) + await manager.stopIndexing() + await manager.startIndexing() + expect(manager.providerPacerRegistry).toBe(registry) + expect(registry.get(services.embedder.rateLimitScope)).toBe(pacer) + expect(vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)![14]).toBeTypeOf("function") + }, + ) - expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() + it.each(["resolve", "reject"])("disposes stale parser and watcher after validation %s", async (ending) => { + const held = deferred() + vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockImplementationOnce(async () => { + await held.promise + return { valid: true } }) + const startup = manager.startIndexing() + await drain() + const parser = manager["parser"]! + await manager.stopIndexing() + if (ending === "resolve") held.resolve() + else held.reject(new Error("stale configuration error")) + await startup + expect(parser.stop).toHaveBeenCalled() + expect(services.fileWatcher.dispose).toHaveBeenCalled() + expect(CodeIndexOrchestrator).not.toHaveBeenCalled() + expect(CodeIndexSearchService).not.toHaveBeenCalled() + expect(manager["servicesStopped"]).toBe(true) + expect(manager["startupFailureMessage"]).toBeUndefined() + }) - it("should be safe to call when orchestrator is not set", () => { - ;(manager as any)._orchestrator = undefined + it.each(["invalid", "throw"])("cleans up a non-deferred %s validation failure", async (kind) => { + const validate = vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder) + if (kind === "invalid") validate.mockResolvedValueOnce({ valid: false, error: "invalid configuration" }) + else validate.mockRejectedValueOnce(new Error("invalid configuration")) + await expect(manager.startIndexing()).rejects.toThrow("invalid configuration") + expect(services.fileWatcher.dispose).toHaveBeenCalled() + expect(manager["parser"]!.stop).toHaveBeenCalled() + expect(manager["startupFailureMessage"]).toBe("invalid configuration") + expect(manager.state).toBe("Error") + expect(CodeIndexOrchestrator).not.toHaveBeenCalled() + }) - expect(() => manager.stopIndexing()).not.toThrow() + it("a deferred permanent validation failure is Error without startup failure, parser latch or clearing", async () => { + vi.useFakeTimers() + const { EmbeddingError } = await import("../shared/indexing-errors") + const { CodeIndexOrchestrator: ActualOrchestrator } = + await vi.importActual("../orchestrator") + vi.mocked(CodeIndexOrchestrator).mockImplementation( + (...argumentsUsed) => new ActualOrchestrator(...argumentsUsed), + ) + const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 1000 }), + pacer.markIssued(), + ) + vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockResolvedValueOnce({ + valid: false, + error: "bad key", }) + await manager.startIndexing() + Object.defineProperty(manager["parser"]!, "stopped", { value: Promise.resolve() }) + expect(manager.state).toBe("Indexing") + expect(manager["startupFailureMessage"]).toBeUndefined() + await vi.advanceTimersByTimeAsync(1000) + await drain() + expect(manager.state).toBe("Error") + expect(manager["startupFailureMessage"]).toBeUndefined() + expect(manager["parserFailed"]).toBe(false) + expect(CacheManager.prototype.clearCacheFile).not.toHaveBeenCalled() + expect(manager["_orchestrator"]!["vectorStore"]).toBe(services.vectorStore) }) - describe("handleSettingsChange - disable toggle bug fix", () => { - it("should abort active indexing when feature is disabled", async () => { - const mockOrchestrator = { - stopIndexing: vi.fn(), - stopWatcher: vi.fn(), - state: "Indexing", - } - ;(manager as any)._orchestrator = mockOrchestrator - - const mockConfigManager = { - loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: false }), - isFeatureConfigured: true, - isFeatureEnabled: false, - } - ;(manager as any)._configManager = mockConfigManager - - const mockStateManager = (manager as any)._stateManager - mockStateManager.setSystemState = vi.fn() - - await manager.handleSettingsChange() + it.each(["Error", "Stopping", "Indexed"] as const)("keeps %s during a search-only provider pause", async (raw) => { + vi.useFakeTimers() + const { EmbeddingError } = await import("../shared/indexing-errors") + state.setSystemState(raw, "original") + const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), + pacer.markIssued(), + ) + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: raw, message: "original" }) + expect(manager.getCurrentStatus().providerWaits[0].waitingFiles).toEqual([]) + }) - expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() - expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "Code indexing is disabled") - }) + it("publishes redacted relative waiting paths separately from genuine failures", async () => { + vi.useFakeTimers() + const { EmbeddingError } = await import("../shared/indexing-errors") + const update = vi.fn() + manager.onProgressUpdate(update) + recordFailure() + const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), + pacer.markIssued(), + ) + const controller = new AbortController() + const pending = pacer + .run("/workspace/src/configured-secret.ts", controller.signal, async () => {}) + .catch(() => {}) + await drain() + expect(update).toHaveBeenLastCalledWith( + expect.objectContaining({ + systemStatus: "Indexing", + providerWaits: [expect.objectContaining({ waitingFiles: [path.join("src", "[REDACTED].ts")] })], + failedFiles: [expect.objectContaining({ path: "bad.ts" })], + }), + ) + state.setSystemState("Stopping", "stopping") + controller.abort() + await pending + expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Stopping", message: "stopping" }) }) }) diff --git a/src/services/code-index/__tests__/mock-code-index-log.ts b/src/services/code-index/__tests__/mock-code-index-log.ts new file mode 100644 index 0000000000..9ee6c3e07d --- /dev/null +++ b/src/services/code-index/__tests__/mock-code-index-log.ts @@ -0,0 +1,3 @@ +vi.mock("../code-index-log", () => ({ + codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})) diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 1a81c1abfa..70e2575d80 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -1,626 +1,820 @@ -import { describe, it, expect, beforeEach, vi } from "vitest" import { CodeIndexOrchestrator } from "../orchestrator" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" - +import { ParserFatalError } from "../parser-errors" +import { INDEX_STOP_LIMIT_MS } from "../constants" +import { FileFailureRegistry } from "../file-failure-registry" +import { ProviderPacerRegistry } from "../provider-pacer" +import { EmbeddingError } from "../shared/indexing-errors" +import { CodeIndexStateManager } from "../state-manager" +import { codeIndexLog } from "../code-index-log" +import { IndexWriteGate } from "../index-write-gate" +import type { CacheManager } from "../cache-manager" +import type { CodeIndexConfigManager } from "../config-manager" +import type { GenerationAllocator } from "../generation-allocator" +import type { ParseWorkerClient } from "../parse-worker-client" +import type { IDirectoryScanner, IFileWatcher, IVectorStore } from "../interfaces" import { clearAllMocks } from "../../../test-utils/reset" -// Mock vscode workspace so startIndexing passes workspace check -vi.mock("vscode", () => { - const path = require("path") - const testWorkspacePath = path.join(path.sep, "test", "workspace") - return { - window: { - activeTextEditor: null, - }, - workspace: { - workspaceFolders: [ - { - uri: { fsPath: testWorkspacePath }, - name: "test", - index: 0, - }, - ], - createFileSystemWatcher: vi.fn().mockReturnValue({ - onDidCreate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidChange: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidDelete: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - }), - }, - RelativePattern: vi.fn().mockImplementation(function (base: string, pattern: string) { - return { base, pattern } - }), - } -}) - -// Mock TelemetryService -vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), - }, - }, -})) - -// Mock i18n translator used in orchestrator messages -vi.mock("../../../i18n", () => ({ - t: (key: string, params?: { errorMessage?: string }) => { - if (key === "embeddings:orchestrator.failedDuringInitialScan" && params?.errorMessage) { - return `Failed during initial scan: ${params.errorMessage}` - } - return key +vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), error: vi.fn() } })) +vi.mock("vscode", () => ({ + workspace: { workspaceFolders: [{ uri: { fsPath: "/workspace" } }] }, + EventEmitter: class { + event = vi.fn() + fire = vi.fn() + dispose = vi.fn() }, })) - -describe("CodeIndexOrchestrator - error path cleanup gating", () => { - const workspacePath = "/test/workspace" - - let configManager: any - let stateManager: any - let cacheManager: any - let vectorStore: any - let scanner: any - let fileWatcher: any +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) + +const scanResult = { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} +async function drain() { + for (let iter = 0; iter < 50; iter++) await Promise.resolve() +} +function fixtures() { + const cache = { + getOpenFiles: vi.fn(() => ({})), + clearCacheFile: vi.fn().mockResolvedValue(undefined), + flush: vi.fn().mockResolvedValue(undefined), + invalidateFile: vi.fn().mockResolvedValue(undefined), + } + const store = { + initialize: vi.fn().mockResolvedValue(false), + hasIndexedData: vi.fn().mockResolvedValue(false), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), + } + const scanner = { + hasIncompleteFiles: false, + scanDirectory: vi.fn().mockResolvedValue(scanResult), + } + const watcher = { + isProcessing: false, + initialize: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn(), + onDidStartBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + onBatchProgressUpdate: vi.fn(() => ({ dispose: vi.fn() })), + onDidFinishBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + onParserFatal: vi.fn>(() => ({ dispose: vi.fn() })), + processFile: vi.fn(), + } + const worker = { stop: vi.fn(), stopped: Promise.resolve(), logStopTimeout: vi.fn() } + const gate = new IndexWriteGate() + vi.spyOn(gate, "run") + vi.spyOn(gate, "settled") + vi.spyOn(gate, "hasPending") + vi.spyOn(gate, "abandon") + const allocator = { initialize: vi.fn().mockResolvedValue(undefined) } + return { cache, store, scanner, watcher, worker, gate, allocator } +} + +describe("CodeIndexOrchestrator lifecycle", () => { + let state: CodeIndexStateManager + let cache: ReturnType["cache"] + let store: ReturnType["store"] + let scanner: ReturnType["scanner"] + let watcher: ReturnType["watcher"] + let worker: ReturnType["worker"] + let gate: IndexWriteGate + let allocator: ReturnType["allocator"] + let current: boolean + let fatal: ReturnType + let orchestrator: CodeIndexOrchestrator + let failures: FileFailureRegistry + let pacers: ProviderPacerRegistry beforeEach(() => { + vi.useFakeTimers() clearAllMocks() + state = new CodeIndexStateManager() + state.setDiagnosticContext({ source: "manager", attempt: 57 }) + vi.spyOn(state, "setSystemState") + vi.spyOn(state, "reportBlockIndexingProgress") + vi.spyOn(state, "reportFileQueueProgress") + ;({ cache, store, scanner, watcher, worker, gate, allocator } = fixtures()) + current = true + fatal = vi.fn() + failures = new FileFailureRegistry() + pacers = new ProviderPacerRegistry() + orchestrator = createOrchestrator() + }) - configManager = { - isFeatureConfigured: true, - } - - // Minimal state manager that tracks state transitions - let currentState = "Standby" - stateManager = { - get state() { - return currentState - }, - setSystemState: vi.fn().mockImplementation((state: string, _msg: string) => { - currentState = state - }), - reportFileQueueProgress: vi.fn(), - reportBlockIndexingProgress: vi.fn(), - } + function createOrchestrator(beforeScan?: (signal: AbortSignal) => Promise) { + // These structural doubles provide only the public lifecycle dependencies used by the owner. + return new CodeIndexOrchestrator( + { isFeatureConfigured: true } as CodeIndexConfigManager, + state, + "/workspace", + cache as unknown as CacheManager, + store as unknown as IVectorStore, + scanner, + watcher, + worker as unknown as ParseWorkerClient, + gate, + allocator as unknown as GenerationAllocator, + failures, + () => current, + fatal, + pacers, + beforeScan, + ) + } - cacheManager = { - clearCacheFile: vi.fn().mockResolvedValue(undefined), - flush: vi.fn().mockResolvedValue(undefined), - } + afterEach(() => vi.useRealTimers()) - vectorStore = { - initialize: vi.fn(), - hasIndexedData: vi.fn(), - markIndexingIncomplete: vi.fn(), - markIndexingComplete: vi.fn(), - clearCollection: vi.fn().mockResolvedValue(undefined), - } + it("initializes the allocator after the store and before dispatch", async () => { + const order: string[] = [] + store.initialize.mockImplementation(async () => { + order.push("store") + return false + }) + allocator.initialize.mockImplementation(async () => { + order.push("allocator") + }) + scanner.scanDirectory.mockImplementation(async () => { + order.push("scan") + return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } + }) + await orchestrator.startIndexing() + expect(order).toEqual(["store", "allocator", "scan"]) + expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + expect(store.markIndexingIncomplete).toHaveBeenCalledBefore(scanner.scanDirectory) + expect(store.markIndexingComplete).toHaveBeenCalledAfter(watcher.initialize) + expect(allocator.initialize).toHaveBeenCalledWith(store, expect.any(AbortSignal), gate) + expect(state.state).toBe("Indexed") + }) - scanner = { - scanDirectory: vi.fn(), - } + it("exposes startup completion before the scan settles", async () => { + const held = deferred() + scanner.scanDirectory.mockReturnValue(held.promise.then(() => scanResult)) + const scan = orchestrator.startIndexing() + await orchestrator.startupResult + expect(scanner.scanDirectory).toHaveBeenCalledOnce() + expect(state.state).toBe("Indexing") + held.resolve() + await scan + }) - fileWatcher = { - initialize: vi.fn().mockResolvedValue(undefined), - onDidStartBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onBatchProgressUpdate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidFinishBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - } + it.each(["resolve", "reject"])("settles startup promptly on stop before initialization can %s", async (ending) => { + const held = deferred() + store.initialize.mockReturnValue(held.promise.then(() => false)) + const scan = orchestrator.startIndexing() + const startupSettled = vi.fn() + void orchestrator.startupResult.then(startupSettled, startupSettled) + await drain() + expect(startupSettled).not.toHaveBeenCalled() + const stop = orchestrator.stopIndexing() + await drain() + expect(startupSettled).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stop + expect(state.state).toBe("Standby") + if (ending === "resolve") held.resolve() + else held.reject(new Error("late initialization failure")) + await scan + expect(allocator.initialize).not.toHaveBeenCalled() + expect(scanner.scanDirectory).not.toHaveBeenCalled() + expect(startupSettled).toHaveBeenCalledOnce() + expect(state.state).toBe("Standby") }) - it.each([false, true])( - "keeps watcher startup and completion after scanning (incremental: %s)", - async (incremental) => { - const events: string[] = [] - vectorStore.initialize.mockResolvedValue(false) - vectorStore.hasIndexedData.mockResolvedValue(incremental) - vectorStore.markIndexingIncomplete.mockImplementation(async () => { - events.push("incomplete") - }) - scanner.scanDirectory.mockImplementation(async () => { - events.push("scan") - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - fileWatcher.initialize.mockImplementation(async () => { - events.push("watcher") - }) - vectorStore.markIndexingComplete.mockImplementation(async () => { - events.push("complete") - }) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + it("logs scan, changed-file batch and stop transitions with the original attempt", async () => { + scanner.scanDirectory.mockImplementation(async (_directory, _onError, onIndexed) => { + state.setSystemState("Standby", "Paused", "test") + onIndexed?.(1) + return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 1 } + }) + await orchestrator.startIndexing() + expect(codeIndexLog.info).toHaveBeenCalledWith( + "State transition", + expect.objectContaining({ newState: "Indexed", source: "scan-end", attempt: 57 }), + ) + watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) + state.setSystemState("Standby", "Paused", "test") + watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) + watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 2 }) + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) + for (const [newState, source] of [ + ["Indexing", "scan"], + ["Indexing", "changed-file-batch"], + ["Indexed", "changed-file-batch"], + ]) { + expect(codeIndexLog.info).toHaveBeenCalledWith( + "State transition", + expect.objectContaining({ newState, source, attempt: 57 }), ) - await orchestrator.startIndexing() - expect(events).toEqual(["incomplete", "scan", "watcher", "complete"]) - expect(orchestrator.state).toBe("Indexed") - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - }, - ) - - it.each([false, true])( - "keeps returned cancellation cleanup in the owner (incremental: %s)", - async (incremental) => { - vectorStore.initialize.mockResolvedValue(false) - vectorStore.hasIndexedData.mockResolvedValue(incremental) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + } + watcher.isProcessing = true + await orchestrator.stopIndexing() + for (const newState of ["Stopping", "Standby"]) { + expect(codeIndexLog.info).toHaveBeenCalledWith( + "State transition", + expect.objectContaining({ newState, source: "stop", attempt: 57 }), ) - scanner.scanDirectory.mockImplementation(async () => { - orchestrator.stopIndexing() - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - // Preserve the original normal-return path: a failed flush is retried by the catch handler. - cacheManager.flush.mockRejectedValueOnce(new Error("flush failed")) - await orchestrator.startIndexing() - expect(cacheManager.flush).toHaveBeenCalledTimes(2) - expect(orchestrator.state).toBe("Standby") - expect(fileWatcher.initialize).not.toHaveBeenCalled() - expect(vectorStore.markIndexingComplete).not.toHaveBeenCalled() - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - }, - ) - - it("should not call clearCollection() or clear cache when initialize() fails (indexing not started)", async () => { - // Arrange: fail at initialize() - vectorStore.initialize.mockRejectedValue(new Error("Qdrant unreachable")) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) + } + }) - // Act + it("returns to Indexed when all failed files recover without a new scan", async () => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") await orchestrator.startIndexing() + expect(state.state).toBe("Standby") + expect(store.markIndexingComplete).not.toHaveBeenCalled() + const completion = deferred() + store.markIndexingComplete.mockReturnValue(completion.promise) + failures.clear("bad.ts", "revision") + await drain() + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + expect(state.state).toBe("Standby") + completion.resolve() + await drain() + expect(state.state).toBe("Indexed") + expect(scanner.scanDirectory).toHaveBeenCalledOnce() + }) - // Assert - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - - // Error state should be set - expect(stateManager.setSystemState).toHaveBeenCalled() - const lastCall = stateManager.setSystemState.mock.calls[stateManager.setSystemState.mock.calls.length - 1] - expect(lastCall[0]).toBe("Error") - }) - - it.each([false, true])( - "only cleans up a failed full scan when the collection was created by this run: %s", - async (created) => { - vectorStore.initialize.mockResolvedValue(created) - vectorStore.hasIndexedData.mockResolvedValue(false) // force full scan path - vectorStore.markIndexingIncomplete.mockRejectedValue(new Error("mark incomplete failure")) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - // Act + it.each(["stop", "parser", "superseded", "batch"])( + "does not issue recovery completion after queued work becomes %s", + async (ending) => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") await orchestrator.startIndexing() - - expect(vectorStore.clearCollection).toHaveBeenCalledTimes(created ? 1 : 0) - // A new collection clears stale cache at initialization and again on failure. - expect(cacheManager.clearCacheFile).toHaveBeenCalledTimes(created ? 2 : 0) - - // Error state should be set - expect(stateManager.setSystemState).toHaveBeenCalled() - const lastCall = stateManager.setSystemState.mock.calls[stateManager.setSystemState.mock.calls.length - 1] - expect(lastCall[0]).toBe("Error") - }, - ) - - it.each([false, true])( - "only cleans up after partial full-scan progress when the collection was created by this run: %s", - async (created) => { - const failure = new Error("batch failure after partial progress") - vectorStore.initialize.mockResolvedValue(created) - vectorStore.hasIndexedData.mockResolvedValue(false) - vectorStore.markIndexingIncomplete.mockResolvedValue(undefined) - scanner.scanDirectory.mockImplementation( - async ( - _dir: string, - onError: (error: Error) => void, - onIndexed: (count: number) => void, - onParsed: (count: number) => void, - ) => { - onParsed(3) - onIndexed(1) - onError(failure) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 3 } + const queued = deferred() + vi.mocked(gate.run).mockImplementation( + async (_signal: AbortSignal, write: () => Result | Promise) => { + await queued.promise + return write() }, ) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - await orchestrator.startIndexing() - - expect(scanner.scanDirectory).toHaveBeenCalledOnce() - expect(stateManager.reportBlockIndexingProgress).toHaveBeenLastCalledWith(1, 3) - expect(stateManager.setSystemState).toHaveBeenLastCalledWith( - "Error", - expect.stringContaining(failure.message), - ) - expect(stateManager.setSystemState).not.toHaveBeenCalledWith("Indexed", expect.any(String)) - expect(vectorStore.clearCollection).toHaveBeenCalledTimes(created ? 1 : 0) - // New collections clear stale cache before scanning and again during error cleanup. - expect(cacheManager.clearCacheFile).toHaveBeenCalledTimes(created ? 2 : 0) - expect(vectorStore.markIndexingIncomplete).toHaveBeenCalledOnce() - expect(vectorStore.markIndexingComplete).not.toHaveBeenCalled() - expect(fileWatcher.initialize).not.toHaveBeenCalled() - expect(fileWatcher.dispose).toHaveBeenCalledOnce() - }, - ) - - it.each([false, true])( - "keeps private error details out of telemetry (cleanup failure: %s)", - async (cleanupFails) => { - const messages = [ - "Embedding failed (Workspace: /Users/private-user/Secret Project, File: /Users/private-user/Secret Project/src/private.ts)", - "Embedding failed (Workspace: C:\\Users\\private-user\\Secret Project, File: C:\\Users\\private-user\\Secret Project\\private.ts)", - ] - const failures = messages.map((message) => { - const error = new Error(message) - error.stack = `${message}\n at privateFunction (/Users/private-user/Secret Project/private.ts:12:3)` - return error - }) - vectorStore.initialize.mockResolvedValue(cleanupFails) - vectorStore.hasIndexedData.mockResolvedValue(!cleanupFails) - vectorStore.markIndexingIncomplete.mockResolvedValue(undefined) - scanner.scanDirectory.mockImplementation(async (_dir: string, onError: (error: Error) => void) => { - for (const failure of failures) onError(failure) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - if (cleanupFails) vectorStore.clearCollection.mockRejectedValue(failures[1]) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - await orchestrator.startIndexing() - - const expectedEvents = [ - [TelemetryEventName.CODE_INDEX_ERROR, { error: "Indexing failed", location: "startIndexing" }], - ] - if (cleanupFails) { - expectedEvents.push([ - TelemetryEventName.CODE_INDEX_ERROR, - { error: "Index cleanup failed", location: "startIndexing.cleanup" }, - ]) + failures.clear("bad.ts", "revision") + if (ending === "stop") { + const stopping = orchestrator.stopWatcher() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopping } - // Exact payload assertions also exclude stacks, nested causes and aggregate error objects. - expect(vi.mocked(TelemetryService.instance.captureEvent).mock.calls).toEqual(expectedEvents) - expect(stateManager.setSystemState).toHaveBeenLastCalledWith("Error", expect.stringContaining(messages[0])) - if (!cleanupFails) { - expect(stateManager.setSystemState).toHaveBeenLastCalledWith( - "Error", - expect.stringContaining(messages[1]), - ) + if (ending === "parser") + watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parser-abort", "bad.ts")) + if (ending === "superseded") current = false + if (ending === "batch") watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) + queued.resolve() + await drain() + expect(store.markIndexingComplete).not.toHaveBeenCalled() + if (ending === "batch") { + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) + await drain() + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + expect(state.state).toBe("Indexed") } }, ) - it("preserves an existing index after an incremental failure and a failed full-scan retry", async () => { - let complete = true - vectorStore.initialize.mockResolvedValue(false) - vectorStore.hasIndexedData.mockImplementation(async () => complete) - vectorStore.markIndexingIncomplete.mockImplementation(async () => { - complete = false - }) - scanner.scanDirectory.mockImplementation(async (_dir: string, onError: (error: Error) => void) => { - onError(new Error("embedding failed")) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - for (let attempt = 1; attempt <= 2; attempt++) { - await orchestrator.startIndexing() - expect(complete).toBe(false) - expect(orchestrator.state).toBe("Error") - expect(scanner.scanDirectory).toHaveBeenCalledTimes(attempt) - expect(vectorStore.markIndexingIncomplete).toHaveBeenCalledTimes(attempt) - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - expect(vectorStore.markIndexingComplete).not.toHaveBeenCalled() - expect(fileWatcher.initialize).not.toHaveBeenCalled() + it.each(["stop", "superseded", "failure"])("handles pending recovery completion on %s", async (ending) => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") + await orchestrator.startIndexing() + const completion = deferred() + store.markIndexingComplete.mockReturnValue(completion.promise) + failures.clear("bad.ts", "revision") + await drain() + if (ending === "stop") { + const stopping = orchestrator.stopWatcher() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopping } - expect(stateManager.setSystemState).toHaveBeenCalledWith("Indexing", "Checking for new or modified files...") - expect(stateManager.setSystemState).toHaveBeenCalledWith( - "Indexing", - "Services ready. Starting workspace scan...", - ) - expect(stateManager.setSystemState).not.toHaveBeenCalledWith("Indexed", expect.any(String)) + if (ending === "superseded") current = false + vi.mocked(state.setSystemState).mockClear() + if (ending === "failure") completion.reject(new Error("metadata failed")) + else completion.resolve() + await drain() + if (ending === "failure") { + expect(state.state).toBe("Error") + expect(codeIndexLog.error).toHaveBeenCalledWith("Recovery completion failed", { error: expect.any(Error) }) + } else expect(state.setSystemState).not.toHaveBeenCalled() }) - it("collects batch errors from full scan and transitions to Error when all blocks fail", async () => { - const batchError = new Error("batch failure") - vectorStore.initialize.mockResolvedValue(false) // existing collection - vectorStore.hasIndexedData.mockResolvedValue(false) // force full scan path - vectorStore.markIndexingIncomplete.mockResolvedValue(undefined) - vectorStore.markIndexingComplete.mockResolvedValue(undefined) + it.each(["stop", "parser", "superseded"])("recovery never overrides %s", async (ending) => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") + await orchestrator.startIndexing() + if (ending === "stop") { + const stopping = orchestrator.stopWatcher() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopping + } + if (ending === "parser") watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parser-abort", "bad.ts")) + if (ending === "superseded") current = false + vi.mocked(state.setSystemState).mockClear() + failures.clear("bad.ts", "revision") + expect(state.setSystemState).not.toHaveBeenCalled() + }) - // Report a batch error — no blocks indexed, so orchestrator treats it as complete failure - scanner.scanDirectory.mockImplementation(async (_dir: string, onBatchError: (e: Error) => void) => { - onBatchError(batchError) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } + it("distinguishes active retries, exhausted failures and a user stop", async () => { + await orchestrator.startIndexing() + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "embed", "failed", 1, "retrying") + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Indexing", + message: "embeddings:orchestrator.indexingFilesRetrying", }) + failures.record("bad.ts", "revision", "embed", "failed", 3, "exhausted") + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Standby", + message: "embeddings:orchestrator.indexingFilesFailed", + }) + watcher.isProcessing = true + watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) + await orchestrator.stopIndexing() + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Standby", + message: "embeddings:orchestrator.indexingStoppedPartial", + }) + }) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - + it("keeps recovery Indexing while a watcher batch remains active", async () => { await orchestrator.startIndexing() + watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "embed", "retrying", 1, "retrying") + failures.clear("bad.ts", "revision") + expect(state.state).toBe("Indexing") + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) + await drain() + expect(state.state).toBe("Indexed") + }) - // With a batch error and zero indexed blocks the orchestrator sets Error state - const calls = stateManager.setSystemState.mock.calls.map((c: any[]) => c[0]) - expect(calls[calls.length - 1]).toBe("Error") - }) - - it.each([0, 2])( - "preserves the existing index on incremental batch failure after %s indexed blocks", - async (indexed) => { - const batchError = new Error("incremental batch failure") - vectorStore.initialize.mockResolvedValue(false) // existing collection - vectorStore.hasIndexedData.mockResolvedValue(true) // force incremental scan path - vectorStore.markIndexingIncomplete.mockResolvedValue(undefined) - vectorStore.markIndexingComplete.mockResolvedValue(undefined) - - scanner.scanDirectory.mockImplementation( - async (_dir: string, onBatchError: (e: Error) => void, onIndexed: (count: number) => void) => { - onIndexed(indexed) - onBatchError(batchError) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 3 } - }, - ) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) + it("refuses scanning after a durable reservation failure without destroying data", async () => { + allocator.initialize.mockRejectedValue(new Error("Storage reservation failed")) + const scan = orchestrator.startIndexing() + await expect(orchestrator.startupResult).rejects.toThrow("Storage reservation failed") + await scan + expect(scanner.scanDirectory).not.toHaveBeenCalled() + expect(state.state).toBe("Error") + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + }) + it.each(["batch error", "incomplete file", "missing blocks"])( + "does not complete a scan with %s", + async (reason) => { + scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "verify", "save failed", 3, "exhausted") + if (reason === "batch error") onError?.(new Error("save failed")) + if (reason === "missing blocks") { + onParsed?.(10) + onIndexed?.(9) + } + return { + stats: { processed: 1, skipped: 0 }, + totalBlockCount: 10, + incompleteFiles: reason === "incomplete file" ? ["bad.ts"] : [], + } + }) await orchestrator.startIndexing() - - expect(orchestrator.state).toBe("Error") - expect(stateManager.setSystemState).toHaveBeenLastCalledWith( - "Error", - expect.stringContaining(batchError.message), + expect(store.markIndexingComplete).not.toHaveBeenCalled() + expect(state.state).toBe("Standby") + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingFilesFailed", + "scan-end", ) - expect(stateManager.setSystemState).not.toHaveBeenCalledWith("Indexed", expect.any(String)) - expect(vectorStore.markIndexingIncomplete).toHaveBeenCalledOnce() - expect(vectorStore.markIndexingComplete).not.toHaveBeenCalled() - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - expect(fileWatcher.initialize).not.toHaveBeenCalled() - expect(fileWatcher.dispose).toHaveBeenCalledOnce() }, ) -}) -describe("CodeIndexOrchestrator - stopIndexing", () => { - const workspacePath = "/test/workspace" - - let configManager: any - let stateManager: any - let cacheManager: any - let vectorStore: any - let scanner: any - let fileWatcher: any - - beforeEach(() => { - clearAllMocks() + it("queue progress never declares a failed watcher batch Indexed", async () => { + await orchestrator.startIndexing() + vi.mocked(state.setSystemState).mockClear() + watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) + watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) + expect(state.state).toBe("Indexing") + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [{ path: "bad.ts", status: "error" }] }) + expect(state.state).toBe("Standby") + expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Indexed")).toBe(false) + }) - configManager = { - isFeatureConfigured: true, - } + it("reports partial when cancellation interrupts a watcher before durable invalidation", async () => { + await orchestrator.startIndexing() + watcher.isProcessing = true + watcher.onDidStartBatchProcessing.mock.calls[0][0](["reading.ts"]) + await orchestrator.stopIndexing() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + "stop", + ) + }) - let currentState = "Standby" - stateManager = { - get state() { - return currentState - }, - setSystemState: vi.fn().mockImplementation((state: string, _msg: string) => { - currentState = state - }), - reportFileQueueProgress: vi.fn(), - reportBlockIndexingProgress: vi.fn(), - } + it("stops the watcher session after a naturally ended scan", async () => { + await orchestrator.startIndexing() + expect(orchestrator.isScanning).toBe(false) + expect(orchestrator.isActive).toBe(true) + const stop = orchestrator.stopIndexing() + expect(orchestrator.stopWatcher()).toBe(stop) + expect(worker.stop).toHaveBeenCalledOnce() + expect(watcher.dispose).toHaveBeenCalledOnce() + await stop + expect(orchestrator.isActive).toBe(false) + }) - cacheManager = { - clearCacheFile: vi.fn().mockResolvedValue(undefined), - flush: vi.fn().mockResolvedValue(undefined), + it.each(["scan", "gate", "worker"])("bounds stopping when %s never settles", async (pending) => { + const held = deferred() + if (pending === "scan") scanner.scanDirectory.mockReturnValue(held.promise.then(() => scanResult)) + if (pending === "gate") { + vi.mocked(gate.settled).mockReturnValue(held.promise) + vi.mocked(gate.hasPending).mockReturnValue(true) } + if (pending === "worker") worker.stopped = held.promise + const scan = orchestrator.startIndexing() + await drain() + watcher.isProcessing = true + const stop = orchestrator.stopIndexing() + expect(orchestrator.stopIndexing()).toBe(stop) + expect(orchestrator.stopWatcher()).toBe(stop) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(1) + await stop + expect(gate.abandon).toHaveBeenCalledOnce() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + "stop", + ) + expect(worker.logStopTimeout).toHaveBeenCalledTimes(pending === "worker" ? 1 : 0) + held.resolve() + await scan + expect(store.markIndexingComplete).toHaveBeenCalledTimes(pending === "scan" ? 0 : 1) + }) - vectorStore = { - initialize: vi.fn().mockResolvedValue(false), - hasIndexedData: vi.fn().mockResolvedValue(false), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - clearCollection: vi.fn().mockResolvedValue(undefined), - } + it("waits for all scan, writes and physical worker exit before a fully settled stop", async () => { + const scanFinished = deferred() + const writesFinished = deferred() + const workerFinished = deferred() + scanner.scanDirectory.mockReturnValue(scanFinished.promise.then(() => scanResult)) + vi.mocked(gate.settled).mockReturnValue(writesFinished.promise) + worker.stopped = workerFinished.promise + const scan = orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + scanFinished.resolve() + writesFinished.resolve() + await drain() + expect(state.state).toBe("Stopping") + workerFinished.resolve() + await stop + await scan + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStopped", + "stop", + ) + expect(gate.abandon).not.toHaveBeenCalled() + }) - scanner = { - scanDirectory: vi.fn(), - } + it("preserves incomplete-file status even when every stop wait settles", async () => { + const scanFinished = deferred() + scanner.scanDirectory.mockReturnValue(scanFinished.promise.then(() => scanResult)) + cache.getOpenFiles.mockReturnValue({ "bad.ts": true }) + void orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + scanFinished.resolve() + await stop + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + "stop", + ) + }) - fileWatcher = { - initialize: vi.fn().mockResolvedValue(undefined), - onDidStartBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onBatchProgressUpdate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidFinishBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - } + it("does not overwrite a newer attempt when the previous stop expires", async () => { + scanner.scanDirectory.mockReturnValue(deferred().promise.then(() => scanResult)) + void orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + current = false + state.setSystemState("Indexing", "new attempt") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stop + expect(state.setSystemState).toHaveBeenLastCalledWith("Indexing", "new attempt") }) - it("should abort indexing when stopIndexing() is called", async () => { - // Make scanner hang until aborted - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - // Wait for abort signal - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) + it.each(["scan", "watcher"])( + "latches fatal %s failure, preserves Error and rejects late callbacks", + async (source) => { + let lateProgress: (count: number) => void = () => {} + if (source === "scan") { + scanner.scanDirectory.mockImplementation(async (_directory, _onError, onProgress) => { + lateProgress = (count) => onProgress?.(count) + throw new ParserFatalError("parser-abort", "bad.ts") }) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }, - ) + } + await orchestrator.startIndexing() + if (source === "watcher") { + watcher.isProcessing = true + lateProgress = () => + watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) + watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parse-timeout", "bad.ts")) + } + await orchestrator.stopIndexing() + lateProgress(100) + expect(state.state).toBe("Error") + expect(codeIndexLog.info).toHaveBeenCalledWith( + "State transition", + expect.objectContaining({ newState: "Error", source: "error", attempt: 57 }), + ) + expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Stopping")).toBe(false) + expect(fatal).toHaveBeenCalledOnce() + expect(worker.stop).toHaveBeenCalledOnce() + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + expect(state.reportBlockIndexingProgress).not.toHaveBeenCalled() + expect(state.reportFileQueueProgress).not.toHaveBeenCalled() + await orchestrator.startIndexing() + expect(scanner.scanDirectory).toHaveBeenCalledOnce() + }, + ) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + it("keeps ordinary scan errors and completed data after cancel", async () => { + scanner.scanDirectory.mockRejectedValue(new Error("network failed")) + await orchestrator.startIndexing() + await orchestrator.stopIndexing() + expect(state.state).toBe("Error") + expect(codeIndexLog.info).toHaveBeenCalledWith( + "State transition", + expect.objectContaining({ newState: "Error", source: "error", attempt: 57 }), ) + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + }) + it.each([false, true])("chooses the scan before exactly one incomplete mark: %s", async (incremental) => { + const order: string[] = [] + store.hasIndexedData.mockImplementation(async () => { + order.push("choose") + return incremental + }) + store.markIndexingIncomplete.mockImplementation(async () => { + order.push("mark") + }) + const held = deferred() + scanner.scanDirectory.mockImplementation(async () => { + order.push("scan") + await held.promise + return scanResult + }) + const scan = orchestrator.startIndexing() + await orchestrator.startupResult + expect(order).toEqual(["choose", "mark", "scan"]) + expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() + held.resolve() + await scan + expect(watcher.initialize).toHaveBeenCalledWith(expect.any(AbortSignal)) + }) - // Start indexing (async, don't await) - const indexingPromise = orchestrator.startIndexing() - - // Give it a tick to begin - await new Promise((resolve) => setTimeout(resolve, 10)) - - // Stop indexing - orchestrator.stopIndexing() + it.each([false, true])("only cleans up a failed newly created collection: %s", async (created) => { + store.initialize.mockResolvedValue(created) + store.markIndexingIncomplete.mockRejectedValue(new Error("metadata failed")) + await orchestrator.startIndexing() + expect(store.clearCollection).toHaveBeenCalledTimes(created ? 1 : 0) + expect(cache.clearCacheFile).toHaveBeenCalledTimes(created ? 2 : 0) + expect(state.state).toBe("Error") + }) - // Wait for indexing to complete - await indexingPromise + it("keeps partial summary until recovery and stopped text until a new start", async () => { + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "failed", 1, "exhausted") + await orchestrator.startIndexing() + expect(orchestrator.isNaturallySettledPartial).toBe(true) + await orchestrator.stopIndexing() + const stoppedMessage = state.getCurrentStatus().message + failures.clear("bad.ts") + await drain() + expect(orchestrator.isNaturallySettledPartial).toBe(false) + expect(state.getCurrentStatus().message).toBe(stoppedMessage) + }) - // State should be Standby (not Error) - const setStateCalls = stateManager.setSystemState.mock.calls - const lastCall = setStateCalls[setStateCalls.length - 1] - expect(lastCall[0]).toBe("Standby") + it("abandons and persists uncertain paths at the stop deadline", async () => { + await orchestrator.startIndexing() + const held = deferred() + const signal = watcher.initialize.mock.calls[0][0]! + const pending = gate.run(signal, () => held.promise, ["uncertain.ts"]).catch(() => {}) + await drain() + const stop = orchestrator.stopIndexing() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stop + expect(cache.invalidateFile).toHaveBeenCalledWith("uncertain.ts") + expect(cache.flush).toHaveBeenCalled() + held.resolve() + await pending }) - it("should set state to Standby after abort, not Error", async () => { - // Make scanner throw AbortError when signal is aborted - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) + it.each(["success", "delete", "cache"] as const)( + "settles clear with the caller-visible %s result", + async (result) => { + await orchestrator.startIndexing() + const failure = new Error(`${result} failed`) + if (result === "delete") store.deleteCollection.mockRejectedValue(failure) + if (result === "cache") cache.clearCacheFile.mockRejectedValue(failure) + const clearing = orchestrator.clearIndexData() + if (result === "success") { + await expect(clearing).resolves.toBeUndefined() + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Standby", + message: "Index data cleared successfully.", }) - throw new DOMException("Indexing aborted", "AbortError") - }, - ) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) + } else { + await expect(clearing).rejects.toBe(failure) + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Error", + message: `Failed to clear index data: ${String(failure)}`, + }) + } + expect(store.deleteCollection).toHaveBeenCalledOnce() + expect(cache.clearCacheFile).toHaveBeenCalledTimes(result === "delete" ? 0 : 1) + }, + ) - const indexingPromise = orchestrator.startIndexing() - await new Promise((resolve) => setTimeout(resolve, 10)) + it.each(["resolve", "reject"] as const)("quietly unwinds a stale clear that later %s", async (ending) => { + await orchestrator.startIndexing() + const held = deferred() + store.deleteCollection.mockReturnValue(held.promise) + const clearing = orchestrator.clearIndexData() + await drain() + current = false + vi.mocked(state.setSystemState).mockClear() + if (ending === "resolve") held.resolve() + else held.reject(new Error("old delete failed")) + await expect(clearing).resolves.toBeUndefined() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + expect(state.setSystemState).not.toHaveBeenCalled() + }) - orchestrator.stopIndexing() - await indexingPromise + it("keeps stop independent of a hung clear and fences its stale continuation", async () => { + await orchestrator.startIndexing() + const held = deferred() + store.deleteCollection.mockReturnValue(held.promise) + const clearing = orchestrator.clearIndexData() + await drain() + const stop = orchestrator.stopWatcher() + expect(orchestrator.stopIndexing()).toBe(stop) + await stop + current = false + vi.mocked(state.setSystemState).mockClear() + held.resolve() + await clearing + expect(cache.clearCacheFile).not.toHaveBeenCalled() + expect(state.setSystemState).not.toHaveBeenCalled() + }) + it.each(["stop", "supersede"])("fences new-collection cleanup after %s", async (ending) => { + store.initialize.mockResolvedValue(true) + const held = deferred() + scanner.scanDirectory.mockImplementation(async () => { + await held.promise + throw new Error("late scan failure") + }) + const scan = orchestrator.startIndexing() + await orchestrator.startupResult + if (ending === "stop") { + const stop = orchestrator.stopWatcher() + held.resolve() + await stop + } else { + current = false + held.resolve() + } + await scan + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).toHaveBeenCalledOnce() + }) - // Should NOT have set Error state — abort is handled gracefully - const errorCalls = stateManager.setSystemState.mock.calls.filter((call: any[]) => call[0] === "Error") - expect(errorCalls).toHaveLength(0) + it("contains a fatal parser event during watcher initialization", async () => { + watcher.initialize.mockImplementation(async () => { + watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parse-timeout", "bad.ts")) + }) + await orchestrator.startIndexing() + await orchestrator.stopWatcher() + expect(fatal).toHaveBeenCalledOnce() + expect(state.state).toBe("Error") + expect(store.markIndexingComplete).not.toHaveBeenCalled() + }) - // Should NOT have cleared collection on abort - expect(vectorStore.clearCollection).not.toHaveBeenCalled() + it("bounds stop when cache persistence hangs", async () => { + await orchestrator.startIndexing() + const held = deferred() + cache.flush.mockReturnValue(held.promise) + const stop = orchestrator.stopWatcher() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stop + expect(state.state).toBe("Standby") + held.resolve() + await drain() }) - it("should preserve partial index data after stop", async () => { - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) - }) - return { stats: { processed: 5, skipped: 0 }, totalBlockCount: 5 } - }, - ) + it.each(["success", "failure", "stop", "supersede"])( + "acknowledges deferred validation before initialization: %s", + async (ending) => { + const held = deferred() + orchestrator = createOrchestrator(async () => held.promise) + const scan = orchestrator.startIndexing() + await orchestrator.startupResult + expect(store.initialize).not.toHaveBeenCalled() + expect(store.markIndexingIncomplete).not.toHaveBeenCalled() + if (ending === "stop") { + const stopping = orchestrator.stopIndexing() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopping + } + if (ending === "supersede") current = false + if (ending === "failure") held.reject(new Error("bad key")) + else held.resolve() + await scan + if (ending === "success") { + expect(store.initialize).toHaveBeenCalledOnce() + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + expect(state.state).toBe("Indexed") + } else { + expect(store.initialize).not.toHaveBeenCalled() + expect(scanner.scanDirectory).not.toHaveBeenCalled() + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + expect(fatal).not.toHaveBeenCalled() + if (ending === "failure") expect(state.state).toBe("Error") + } + }, + ) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + it("keeps recovery and provider callbacks from replacing Stopping", async () => { + await orchestrator.startIndexing() + state.setSystemState("Stopping", "original") + vi.mocked(state.setSystemState).mockClear() + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "failed", 1, "exhausted") + const pacer = pacers.get("embedding:test") + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), + pacer.markIssued(), ) + const controller = new AbortController() + const waiting = pacer.run("waiting.ts", controller.signal, async () => {}).catch(() => {}) + failures.clear("bad.ts", "revision") + controller.abort() + await waiting + await drain() + expect(state.setSystemState).not.toHaveBeenCalled() + expect(state.getCurrentStatus()).toMatchObject({ systemStatus: "Stopping", message: "original" }) + }) - const indexingPromise = orchestrator.startIndexing() - await new Promise((resolve) => setTimeout(resolve, 10)) + it("publishes provider waiting and returns to Indexed when queued work resumes", async () => { + await orchestrator.startIndexing() + const pacer = pacers.get("embedding:test") + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 1000 }), + pacer.markIssued(), + ) + expect(state.state).toBe("Indexed") + const request = vi.fn(async () => {}) + const waiting = pacer.run("waiting.ts", undefined, request) + await drain() + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Indexing", + message: "embeddings:orchestrator.providerWaitingUntil", + }) + expect(failures.list()).toEqual([]) + expect(request).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1000) + await waiting + expect(state.state).toBe("Indexed") + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + }) - orchestrator.stopIndexing() - await indexingPromise + it("disposes the provider subscription on stop", async () => { + const subscribe = vi.spyOn(pacers, "onDidChange") + await orchestrator.startIndexing() + const subscription = subscribe.mock.results[0].value + const dispose = vi.spyOn(subscription, "dispose") + await orchestrator.stopIndexing() + expect(dispose).toHaveBeenCalledOnce() + }) - // Cache should NOT be cleared on user-initiated stop - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - // Collection should NOT be cleared on user-initiated stop - expect(vectorStore.clearCollection).not.toHaveBeenCalled() + it("leaves a naturally settled partial scan idle and incomplete", async () => { + scanner.scanDirectory.mockResolvedValue({ ...scanResult, incompleteFiles: ["bad.ts"] }) + await orchestrator.startIndexing() + expect(orchestrator.isScanning).toBe(false) + expect(orchestrator.isNaturallySettledPartial).toBe(true) + expect(state.state).toBe("Standby") + expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() + expect(store.markIndexingComplete).not.toHaveBeenCalled() }) }) diff --git a/src/services/code-index/__tests__/parse-worker-client.real.spec.ts b/src/services/code-index/__tests__/parse-worker-client.real.spec.ts new file mode 100644 index 0000000000..672886e773 --- /dev/null +++ b/src/services/code-index/__tests__/parse-worker-client.real.spec.ts @@ -0,0 +1,159 @@ +vi.mock("../code-index-log", () => ({ + codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})) +import { copyFile, mkdtemp, readFile, readdir, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import * as path from "node:path" +import { Worker } from "node:worker_threads" +import { ParseWorkerClient, type ParseReply } from "../parse-worker-client" +import { INDEX_STOP_LIMIT_MS, PARSE_TIMEOUT_MS } from "../constants" + +const options = { + content: "function example() { return 'a sufficiently long function body for indexing'; }", + fileHash: "hash", +} +const distribution = path.resolve("dist") +const builtWorker = path.join(distribution, "workers", "codeIndexParse.js") + +function within(promise: Promise, milliseconds: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Did not settle within ${milliseconds} ms`)), milliseconds) + promise.then( + (value) => { + clearTimeout(timer) + resolve(value) + }, + (error: unknown) => { + clearTimeout(timer) + reject(error) + }, + ) + }) +} + +async function probe(): Promise { + const started = performance.now() + await new Promise((resolve) => setTimeout(resolve, 20)) + const elapsed = performance.now() - started + expect(elapsed).toBeLessThan(500) + return elapsed +} + +describe("ParseWorkerClient built worker", () => { + let directory: string + const workers: Worker[] = [] + + beforeAll(async () => { + directory = await mkdtemp(path.join(tmpdir(), "zoo-parse-worker-")) + await copyFile(builtWorker, path.join(directory, "codeIndexParse.js")) + await copyFile( + path.resolve("services/code-index/__tests__/fixtures/fault-loop.wasm"), + path.join(directory, "fault-loop.wasm"), + ) + }) + + afterEach(async () => { + await Promise.all(workers.splice(0).map((worker) => worker.terminate())) + }) + + afterAll(async () => { + if (directory) await rm(directory, { recursive: true, force: true }) + }) + + function createWorker(fault?: string): Worker { + const environment = { ...process.env } + delete environment.ROO_PARSE_WORKER_FAULT + if (fault) environment.ROO_PARSE_WORKER_FAULT = fault + const worker = new Worker(fault ? path.join(directory, "codeIndexParse.js") : builtWorker, { + env: environment, + workerData: { wasmDirectory: distribution }, + }) + workers.push(worker) + return worker + } + + it("preserves both worker outputs without packaging the test-only fault fixture or VS Code imports", async () => { + const filenames = await readdir(path.join(distribution, "workers")) + expect(filenames).toContain("countTokens.js") + expect(filenames).toContain("codeIndexParse.js") + expect(filenames).not.toContain("fault-loop.wasm") + expect(await readdir(distribution)).not.toContain("fault-loop.wasm") + const bundle = await readFile(builtWorker, "utf8") + expect(bundle).not.toMatch(/require\(["']vscode["']\)/) + }) + + it("loads the runtime and TypeScript grammar from extension dist rather than the worker directory", async () => { + const client = new ParseWorkerClient(() => createWorker()) + try { + const blocks = await within(client.parseFile("normal.ts", options), 5000) + expect(blocks).toEqual([ + expect.objectContaining({ + content: options.content, + type: "function_declaration", + fileHash: options.fileHash, + }), + ]) + } finally { + client.stop() + await within(client.stopped, INDEX_STOP_LIMIT_MS) + } + }) + + it.each(["timeout", "stop"])( + "physically exits a real WASM hang on %s without blocking the main loop", + async (action) => { + const worker = createWorker("wasm-hang") + const entered = new Promise((resolve, reject) => { + worker.once("error", reject) + worker.on("message", (reply: ParseReply) => { + if (reply.diagnostic?.message === "parse worker entering wasm fault loop") resolve() + }) + }) + const client = new ParseWorkerClient(() => worker) + let exitObserved = false + worker.once("exit", () => { + exitObserved = true + }) + const pending = client.parseFile("hang.ts", options).catch((error: unknown) => error) + await within(entered, 5000) + const hangProbe = await probe() + expect(exitObserved).toBe(false) + let stoppingAt: number + if (action === "stop") { + stoppingAt = performance.now() + expect(client.stop()).toBeUndefined() + } else { + expect(await within(pending, PARSE_TIMEOUT_MS)).toMatchObject({ category: "parse-timeout" }) + stoppingAt = performance.now() + } + const stoppingProbe = probe() + await within(client.stopped, INDEX_STOP_LIMIT_MS) + const exitElapsed = performance.now() - stoppingAt + expect(exitElapsed).toBeLessThan(INDEX_STOP_LIMIT_MS) + expect(exitObserved).toBe(true) + expect(await pending).toMatchObject( + action === "stop" ? { name: "AbortError" } : { category: "parse-timeout" }, + ) + expect(await stoppingProbe).toBeLessThan(500) + expect(hangProbe).toBeLessThan(500) + }, + ) + + it("contains a fatal abort and permits explicit recovery using a new worker", async () => { + const failed = new ParseWorkerClient(() => createWorker("abort")) + await expect(failed.parseFile("abort.ts", options)).rejects.toMatchObject({ + name: "ParserFatalError", + category: "parser-abort", + message: expect.stringContaining("Aborted()"), + }) + await within(failed.stopped, INDEX_STOP_LIMIT_MS) + const replacement = new ParseWorkerClient(() => createWorker()) + try { + const blocks = await within(replacement.parseFile("normal.ts", options), 5000) + expect(blocks[0].content).toBe(options.content) + } finally { + replacement.stop() + await within(replacement.stopped, INDEX_STOP_LIMIT_MS) + } + }) +}) diff --git a/src/services/code-index/__tests__/parse-worker-client.spec.ts b/src/services/code-index/__tests__/parse-worker-client.spec.ts new file mode 100644 index 0000000000..67484bd4d2 --- /dev/null +++ b/src/services/code-index/__tests__/parse-worker-client.spec.ts @@ -0,0 +1,255 @@ +vi.mock("../code-index-log", () => ({ + codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})) +import { codeIndexLog } from "../code-index-log" +import { EventEmitter } from "node:events" +import { ParseWorkerClient, type ParseRequest, type ParseWorker } from "../parse-worker-client" + +class MockWorker extends EventEmitter { + requests: ParseRequest[] = [] + terminate = vi.fn().mockResolvedValue(0) + postMessage(request: ParseRequest): void { + this.requests.push(request) + } +} + +const options = { + content: "function example() { return 'a sufficiently long function body for indexing'; }", + fileHash: "hash", +} + +async function drain(): Promise { + for (let turn = 0; turn < 10; turn++) await Promise.resolve() +} + +describe("ParseWorkerClient", () => { + afterEach(() => vi.useRealTimers()) + + it("settles stopped immediately when stopped before creating a worker", async () => { + const factory = vi.fn<() => ParseWorker>() + const client = new ParseWorkerClient(factory) + expect(client.stop()).toBeUndefined() + await client.stopped + expect(factory).not.toHaveBeenCalled() + }) + + it.each(["resolved", "rejected", "pending"])( + "waits for physical exit with a %s termination promise and logs the parent's timeout once", + async (termination) => { + vi.useFakeTimers() + const log = vi.spyOn(codeIndexLog, "info").mockImplementation(() => {}) + try { + const worker = new MockWorker() + if (termination === "rejected") worker.terminate.mockRejectedValue(new Error("termination failed")) + if (termination === "pending") worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + let exited = false + void client.stopped.then(() => { + exited = true + }) + const pending = client.parseFile("active.ts", options).catch((error: unknown) => error) + await drain() + expect(client.stop()).toBeUndefined() + expect(await pending).toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(2000) + expect(exited).toBe(false) + client.logStopTimeout() + client.logStopTimeout() + expect(log).toHaveBeenCalledExactlyOnceWith("parse worker exit not observed within stop limit") + worker.emit("exit", 1) + await client.stopped + expect(exited).toBe(true) + expect(log).toHaveBeenLastCalledWith(expect.stringMatching(/^parse worker exited after \d+ ms$/)) + client.logStopTimeout() + expect(log).toHaveBeenCalledTimes(2) + } finally { + log.mockRestore() + } + }, + ) + + it("observes an unexpected physical exit even when terminate never settles", async () => { + const worker = new MockWorker() + worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + const pending = client.parseFile("active.ts", options).catch((error: unknown) => error) + await drain() + worker.emit("exit", 1) + expect(await pending).toMatchObject({ name: "ParserFatalError" }) + await client.stopped + expect(worker.terminate).not.toHaveBeenCalled() + }) + + it("forwards diagnostics without completing or interrupting the active parse", async () => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const pending = client.parseFile("active.ts", options) + await drain() + worker.emit("message", { + id: 0, + diagnostic: { + level: "warn", + message: "parser warning", + fields: { path: "active.ts" }, + }, + }) + expect(codeIndexLog.warn).toHaveBeenLastCalledWith("parser warning", { path: "active.ts" }) + expect(worker.terminate).not.toHaveBeenCalled() + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await expect(pending).resolves.toEqual([]) + client.stop() + }) + + it("dispatches FIFO and starts each deadline at dispatch, not enqueue", async () => { + vi.useFakeTimers() + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker, 100) + const first = client.parseFile("first.ts", options) + const second = client.parseFile("second.ts", options) + await drain() + expect(worker.requests.map((request) => request.filePath)).toEqual(["first.ts"]) + await vi.advanceTimersByTimeAsync(90) + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + await vi.advanceTimersByTimeAsync(90) + expect(worker.terminate).not.toHaveBeenCalled() + worker.emit("message", { id: worker.requests[1].id, blocks: [] }) + await second + client.stop() + }) + + it("uses the ten-second watchdog without granting a queued request its own failed-worker attempt", async () => { + vi.useFakeTimers() + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const pending = client.parseFile("hung.ts", options).catch((error: unknown) => error) + await drain() + await vi.advanceTimersByTimeAsync(9999) + expect(worker.terminate).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(await pending).toMatchObject({ category: "parse-timeout", filePath: "hung.ts" }) + expect(worker.terminate).toHaveBeenCalledOnce() + }) + + it("rejects an ordinary parse error without preventing the next file from completing", async () => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const failed = client.parseFile("bad.ts", options).catch((error: unknown) => error) + const next = client.parseFile("good.ts", options) + await drain() + worker.emit("message", { id: worker.requests[0].id, error: { message: "invalid source", fatal: false } }) + expect(await failed).toMatchObject({ message: "invalid source", name: "Error" }) + expect(worker.terminate).not.toHaveBeenCalled() + worker.emit("message", { id: worker.requests[1].id, blocks: [] }) + await expect(next).resolves.toEqual([]) + client.stop() + }) + + it("ignores mismatched replies and performs fallback chunking only when requested", async () => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const pending = client.parseFile("source.ts", options) + await drain() + worker.emit("message", { id: -1, error: { message: "Aborted()", fatal: true } }) + expect(worker.terminate).not.toHaveBeenCalled() + worker.emit("message", { id: worker.requests[0].id, fallback: true }) + await expect(pending).resolves.toEqual([ + expect.objectContaining({ file_path: "source.ts", content: options.content, fileHash: options.fileHash }), + ]) + client.stop() + }) + + it.each(["Aborted()", "memory access out of bounds", "error", "exit", "hang"])( + "contains %s and terminally rejects queued work", + async (failure) => { + vi.useFakeTimers() + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker, 100) + const active = client.parseFile("bad.ts", options).catch((error: unknown) => error) + const queued = client.parseFile("queued.ts", options).catch((error: unknown) => error) + await drain() + if (failure === "hang") await vi.advanceTimersByTimeAsync(100) + else if (failure === "error") worker.emit("error", new Error("crash")) + else if (failure === "exit") worker.emit("exit", 1) + else worker.emit("message", { id: worker.requests[0].id, error: { message: failure, fatal: false } }) + expect(await active).toMatchObject({ + name: "ParserFatalError", + filePath: "bad.ts", + category: failure === "hang" ? "parse-timeout" : "parser-abort", + }) + expect(await queued).toMatchObject({ name: "AbortError" }) + await expect(client.parseFile("later.md", options)).rejects.toMatchObject({ name: "AbortError" }) + expect(worker.requests).toHaveLength(1) + expect(worker.terminate).toHaveBeenCalledTimes(failure === "exit" ? 0 : 1) + }, + ) + + it("removes a cancelled queue entry without stopping another caller", async () => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const controller = new AbortController() + const first = client.parseFile("first.ts", options) + const cancelled = client + .parseFile("cancelled.ts", { ...options, signal: controller.signal }) + .catch((error: unknown) => error) + const last = client.parseFile("last.ts", options) + await drain() + controller.abort() + expect(await cancelled).toMatchObject({ name: "AbortError" }) + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + expect(worker.requests[1].filePath).toBe("last.ts") + worker.emit("message", { id: worker.requests[1].id, blocks: [] }) + await last + client.stop() + }) + + it.each(["stop", "abort"])( + "%s synchronously settles active and queued requests without waiting for termination", + async (action) => { + const worker = new MockWorker() + worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + const controller = new AbortController() + const active = client + .parseFile("active.ts", { ...options, signal: controller.signal }) + .catch((error: unknown) => error) + const queued = client.parseFile("queued.ts", options).catch((error: unknown) => error) + await drain() + if (action === "stop") client.stop() + else controller.abort() + expect(await active).toMatchObject({ name: "AbortError" }) + expect(await queued).toMatchObject({ name: "AbortError" }) + client.stop() + expect(worker.terminate).toHaveBeenCalledTimes(1) + }, + ) + + it.each(["error", "exit"])("reports an idle worker %s as a parser failure on the next parse", async (failure) => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const first = client.parseFile("first.ts", options) + await drain() + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + if (failure === "error") worker.emit("error", new Error("crash")) + else worker.emit("exit", 1) + await expect(client.parseFile("next.ts", options)).rejects.toMatchObject({ name: "ParserFatalError" }) + }) + + it("keeps independent workspaces isolated and a replacement usable", async () => { + const failedWorker = new MockWorker() + const otherWorker = new MockWorker() + const failed = new ParseWorkerClient(() => failedWorker) + const other = new ParseWorkerClient(() => otherWorker) + const failure = failed.parseFile("bad.ts", options).catch((error: unknown) => error) + const success = other.parseFile("good.ts", options) + await drain() + failedWorker.emit("exit", 1) + await failure + otherWorker.emit("message", { id: otherWorker.requests[0].id, blocks: [] }) + await expect(success).resolves.toEqual([]) + expect(otherWorker.terminate).not.toHaveBeenCalled() + other.stop() + }) +}) diff --git a/src/services/code-index/__tests__/parse-worker-imports.spec.ts b/src/services/code-index/__tests__/parse-worker-imports.spec.ts new file mode 100644 index 0000000000..7f8ed47057 --- /dev/null +++ b/src/services/code-index/__tests__/parse-worker-imports.spec.ts @@ -0,0 +1,56 @@ +import { existsSync, readFileSync } from "node:fs" +import * as path from "node:path" +import * as ts from "typescript" + +function collectWorkerImports(entryPath: string) { + const visited = new Set() + const imports: string[] = [] + const pending = [entryPath] + while (pending.length > 0) { + const filePath = pending.pop()! + if (visited.has(filePath)) continue + visited.add(filePath) + const source = ts.createSourceFile(filePath, readFileSync(filePath, "utf8"), ts.ScriptTarget.Latest, true) + function visit(node: ts.Node): void { + let modulePath: string | undefined + if ((ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) && node.moduleSpecifier) { + if (ts.isStringLiteral(node.moduleSpecifier)) modulePath = node.moduleSpecifier.text + } else if (ts.isImportTypeNode(node) && ts.isLiteralTypeNode(node.argument)) { + if (ts.isStringLiteral(node.argument.literal)) modulePath = node.argument.literal.text + } else if ( + ts.isCallExpression(node) && + (node.expression.kind === ts.SyntaxKind.ImportKeyword || + (ts.isIdentifier(node.expression) && node.expression.text === "require")) + ) { + const argument = node.arguments[0] + if (argument && ts.isStringLiteral(argument)) modulePath = argument.text + } + if (ts.isIdentifier(node)) expect(node.text).not.toBe("createRequire") + if (modulePath !== undefined) { + imports.push(modulePath) + if (modulePath.startsWith(".")) { + const resolved = path.resolve(path.dirname(filePath), modulePath) + const dependency = [resolved, `${resolved}.ts`, path.join(resolved, "index.ts")].find( + (candidate) => candidate.endsWith(".ts") && existsSync(candidate), + ) + expect(dependency, `${filePath}: ${modulePath}`).toBeDefined() + if (dependency) pending.push(dependency) + } + } + ts.forEachChild(node, visit) + } + visit(source) + } + return { visited, imports } +} + +it("keeps host imports out of the entire worker source graph, including type-only and dynamic imports", () => { + const { visited, imports } = collectWorkerImports(path.resolve("workers/codeIndexParse.ts")) + expect(imports.filter((modulePath) => /^(?:vscode(?:\/|$)|@roo-code\/)/.test(modulePath))).toEqual([]) + const files = [...visited].map((filePath) => filePath.replaceAll("\\", "/")) + expect(files.some((filePath) => filePath.endsWith("/code-index-log.ts"))).toBe(true) + expect(files.some((filePath) => filePath.endsWith("/parse-worker-protocol.ts"))).toBe(true) + expect( + files.filter((filePath) => /\/(?:code-index-log-host|parse-worker-client|manager)\.ts$/.test(filePath)), + ).toEqual([]) +}) diff --git a/src/services/code-index/__tests__/parser-errors.spec.ts b/src/services/code-index/__tests__/parser-errors.spec.ts new file mode 100644 index 0000000000..11d3a6b6c9 --- /dev/null +++ b/src/services/code-index/__tests__/parser-errors.spec.ts @@ -0,0 +1,28 @@ +import { abortError, isParserAbort, ParserFatalError, throwIfAborted } from "../parser-errors" + +describe("parser errors", () => { + it.each(["parser-abort", "parse-timeout"] as const)("preserves the %s category and file", (category) => { + const error = new ParserFatalError(category, "source.ts") + expect(error).toBeInstanceOf(Error) + expect(error).toMatchObject({ name: "ParserFatalError", category, filePath: "source.ts", message: category }) + expect(new ParserFatalError(category, "source.ts", "detail").message).toBe("detail") + }) + + it.each([ + ["Aborted()", true], + [new Error("MEMORY ACCESS OUT OF BOUNDS"), true], + [new Error("ordinary parse failure"), false], + [undefined, false], + ])("classifies %s without treating ordinary failures as fatal", (error, expected) => { + expect(isParserAbort(error)).toBe(expected) + }) + + it("only throws a cancellation for an aborted signal", () => { + expect(abortError()).toMatchObject({ name: "AbortError", message: "Indexing aborted" }) + expect(() => throwIfAborted()).not.toThrow() + const controller = new AbortController() + expect(() => throwIfAborted(controller.signal)).not.toThrow() + controller.abort() + expect(() => throwIfAborted(controller.signal)).toThrow("Indexing aborted") + }) +}) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts new file mode 100644 index 0000000000..573519d414 --- /dev/null +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -0,0 +1,392 @@ +import { createHash } from "node:crypto" +import { ProviderPacer, ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" +import { codeIndexLog } from "../code-index-log" + +vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) + +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +const limited = (retryAfterMs?: number) => ({ status: 429, retryAfterMs }) +const flush = () => vi.advanceTimersByTimeAsync(0) + +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + vi.spyOn(Math, "random").mockReturnValue(0) +}) + +afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() +}) + +describe("ProviderPacer", () => { + it("T35 shares a 30s pause, keeps in-flight results, and isolates allowance keys and stores", async () => { + const registry = new ProviderPacerRegistry() + const scope = makeRateLimitScope("openai", "https://example.test", "first") + const pacer = registry.get(scope) + expect(registry.get(scope)).toBe(pacer) + const inFlight = deferred() + const kept = pacer.run("kept.ts", undefined, () => inFlight.promise) + const first = vi.fn().mockRejectedValueOnce(limited(30_000)).mockResolvedValue("first") + const firstResult = pacer.run("a.ts", undefined, first) + await flush() + const second = vi.fn().mockResolvedValue("second") + const secondResult = pacer.run("b.ts", undefined, second) + await flush() + inFlight.resolve("kept") + expect(await kept).toBe("kept") + expect( + await registry + .get(makeRateLimitScope("qdrant", "https://store.test")) + .run(undefined, undefined, async () => "store"), + ).toBe("store") + expect( + await registry + .get(makeRateLimitScope("openai", "https://example.test", "second")) + .run(undefined, undefined, async () => "other key"), + ).toBe("other key") + await vi.advanceTimersByTimeAsync(29_999) + expect(first).toHaveBeenCalledTimes(1) + expect(second).not.toHaveBeenCalled() + expect(pacer.state()).toMatchObject({ limit: 5, pausedUntil: 30_000, waitingPaths: ["a.ts", "b.ts"] }) + await vi.advanceTimersByTimeAsync(1) + expect(await firstResult).toBe("first") + expect(second).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(125) + expect(await secondResult).toBe("second") + }) + + it("T36 counts ten same-clock 429s once and ignores stale missing-delay reports after expiry", async () => { + const pacer = new ProviderPacer("openai:test") + const controllers = Array.from({ length: 10 }, () => new AbortController()) + const requests = controllers.map(() => deferred()) + const outcomes = requests.map((request, index) => + pacer.run(`${index}.ts`, controllers[index].signal, () => request.promise).catch((error) => error), + ) + await flush() + for (const request of requests) request.reject(limited()) + await flush() + expect(pacer.state()).toMatchObject({ + limit: 5, + consecutiveRateLimits: 1, + episodeSeq: 10, + issueSeq: 10, + pausedUntil: 500, + }) + expect(codeIndexLog.info).toHaveBeenCalledTimes(1) + for (const controller of controllers) controller.abort() + await Promise.all(outcomes) + await vi.advanceTimersByTimeAsync(500) + pacer.reportRateLimit(limited(), 9) + expect(pacer.state()).toMatchObject({ limit: 5, consecutiveRateLimits: 1, pausedUntil: undefined }) + pacer.reportRateLimit(limited(2000), 9) + expect(pacer.state()).toMatchObject({ limit: 5, consecutiveRateLimits: 1, pausedUntil: 2500 }) + }) + + it("T36 stale same-clock success neither resets backoff nor contributes to adaptive growth", async () => { + const pacer = new ProviderPacer("openai:test") + const stale = deferred() + const staleResult = pacer.run("stale", undefined, () => stale.promise) + await flush() + pacer.reportRateLimit(limited(), pacer.markIssued()) + stale.resolve() + await staleResult + expect(pacer.state().consecutiveRateLimits).toBe(1) + await vi.advanceTimersByTimeAsync(500) + const newIssue = pacer.markIssued() + pacer.reportRateLimit(limited(), newIssue) + expect(pacer.state()).toMatchObject({ limit: 2, consecutiveRateLimits: 2, pausedUntil: 1500 }) + await vi.advanceTimersByTimeAsync(1000) + for (let successCount = 0; successCount < 19; successCount++) { + const result = pacer.run("growth", undefined, async () => "ok") + await vi.advanceTimersByTimeAsync(125) + await result + } + expect(pacer.state().limit).toBe(2) + await pacer.run("growth", undefined, async () => "ok") + expect(pacer.state()).toMatchObject({ limit: 3, consecutiveRateLimits: 0 }) + for (let successCount = 0; successCount < 200; successCount++) + await pacer.run(undefined, undefined, async () => undefined) + expect(pacer.state().limit).toBe(10) + }) + + it.each([0, 1])( + "T36 exponential equal jitter respects both bounds and the 300s base ceiling (random=%s)", + async (random) => { + vi.mocked(Math.random).mockReturnValue(random) + const pacer = new ProviderPacer("openai:test") + for (let episode = 1; episode <= 12; episode++) { + const startedAt = Date.now() + pacer.reportRateLimit(limited(), pacer.markIssued()) + const expected = Math.min(300_000, 1000 * 2 ** (episode - 1)) * (0.5 + random * 0.5) + expect(pacer.state().pausedUntil).toBe(startedAt + expected) + await vi.advanceTimersByTimeAsync(expected) + } + }, + ) + + it("requires consecutive successes for growth, resetting progress on non-429 failures", async () => { + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit(limited(1), pacer.markIssued()) + await vi.advanceTimersByTimeAsync(1) + for (let successCount = 0; successCount < 19; successCount++) { + const result = pacer.run(undefined, undefined, async () => undefined) + await vi.advanceTimersByTimeAsync(125) + await result + } + const error = { status: 500 } + await expect( + pacer.run(undefined, undefined, async () => { + throw error + }), + ).rejects.toBe(error) + await pacer.run(undefined, undefined, async () => undefined) + expect(pacer.state().limit).toBe(5) + for (let successCount = 0; successCount < 19; successCount++) + await pacer.run(undefined, undefined, async () => undefined) + expect(pacer.state().limit).toBe(6) + }) + + it("a probe 429 creates the next episode and leaves followers blocked", async () => { + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit(limited(100), pacer.markIssued()) + const controller = new AbortController() + const request = vi.fn().mockRejectedValue(limited()) + const pending = pacer.run("probe", controller.signal, request).catch((error) => error) + const follower = vi.fn().mockResolvedValue(undefined) + const waiting = pacer.run("follower", controller.signal, follower).catch((error) => error) + await vi.advanceTimersByTimeAsync(100) + expect(pacer.state()).toMatchObject({ limit: 2, consecutiveRateLimits: 2, pausedUntil: 1100 }) + expect(request).toHaveBeenCalledOnce() + expect(follower).not.toHaveBeenCalled() + controller.abort() + await Promise.all([pending, waiting]) + }) + + it("releases and requeues a granted permit when another 429 pauses before consumption", async () => { + const pacer = new ProviderPacer("openai:test") + const request = vi.fn().mockResolvedValue("done") + const result = pacer.run("granted.ts", undefined, request) + pacer.reportRateLimit(limited(1000), pacer.markIssued()) + await flush() + expect(request).not.toHaveBeenCalled() + expect(pacer.state()).toMatchObject({ waitingPaths: ["granted.ts"], issueSeq: 1 }) + expect(pacer["active"]).toBe(0) + await vi.advanceTimersByTimeAsync(999) + expect(request).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(await result).toBe("done") + expect(request).toHaveBeenCalledOnce() + expect(pacer.state()).toMatchObject({ waitingPaths: [], issueSeq: 2 }) + expect(pacer["active"]).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it("rechecks cancellation after granting a permit but before invoking the caller", async () => { + const pacer = new ProviderPacer("openai:test") + const controller = new AbortController() + const request = vi.fn().mockResolvedValue(undefined) + const result = pacer.run(undefined, controller.signal, request) + controller.abort() + await expect(result).rejects.toBe(controller.signal.reason) + expect(request).not.toHaveBeenCalled() + }) + + it.each([0, -1, NaN, Infinity, "invalid"])( + "uses fallback for invalid or elapsed provider delay %s", + (retryAfterMs) => { + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit({ status: 429, retryAfterMs }, pacer.markIssued()) + expect(pacer.state().pausedUntil).toBe(500) + }, + ) + + it("paused search reports extend without counting episodes or shortening waits", async () => { + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit(limited(1000), pacer.markIssued()) + await vi.advanceTimersByTimeAsync(100) + pacer.reportRateLimit(limited(100), pacer.markIssued()) + pacer.reportRateLimit(limited(2000), pacer.markIssued()) + expect(pacer.state()).toMatchObject({ limit: 5, episodeSeq: 1, consecutiveRateLimits: 1, pausedUntil: 2100 }) + }) + + it.each(["server", "connection", "permanent", "abort"])( + "T36 reopens one at a time after a probe %s outcome", + async (outcome) => { + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit(limited(1000), pacer.markIssued()) + const probe = deferred() + const controller = new AbortController() + const probeRequest = vi.fn(() => probe.promise) + const probeResult = pacer.run("probe", controller.signal, probeRequest).catch((error) => error) + const startedAt: number[] = [] + const followers = Array.from({ length: 6 }, (_, index) => + pacer.run(`follower-${index}`, undefined, async () => { + startedAt.push(Date.now()) + }), + ) + await vi.advanceTimersByTimeAsync(1000) + expect(probeRequest).toHaveBeenCalledOnce() + expect(startedAt).toEqual([]) + await vi.advanceTimersByTimeAsync(1000) + expect(startedAt).toEqual([]) + const error = + outcome === "abort" + ? new DOMException("aborted", "AbortError") + : outcome === "connection" + ? { code: "ECONNRESET" } + : { status: outcome === "server" ? 500 : 400 } + if (outcome === "abort") controller.abort(error) + probe.reject(error) + expect(await probeResult).toBe(error) + await vi.advanceTimersByTimeAsync(124) + expect(startedAt).toEqual([]) + await vi.advanceTimersByTimeAsync(1) + expect(startedAt).toEqual([2125]) + await vi.advanceTimersByTimeAsync(125) + expect(startedAt).toEqual([2125, 2250]) + await vi.runAllTimersAsync() + await Promise.all(followers) + expect(startedAt.slice(0, 4)).toEqual([2125, 2250, 2375, 2500]) + }, + ) + + it("uses the upper 375ms staged gap and lets probe cancellation reopen before transport settles", async () => { + vi.mocked(Math.random).mockReturnValue(1) + const pacer = new ProviderPacer("openai:test") + pacer.reportRateLimit(limited(100), pacer.markIssued()) + const probe = deferred() + const controller = new AbortController() + const probeResult = pacer.run(undefined, controller.signal, () => probe.promise) + const follower = vi.fn().mockResolvedValue(undefined) + const followerResult = pacer.run(undefined, undefined, follower) + await vi.advanceTimersByTimeAsync(100) + controller.abort() + await vi.advanceTimersByTimeAsync(374) + expect(follower).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await followerResult + probe.resolve() + await probeResult + }) + + it("absorbs unlimited 429s but preserves a subsequent non-429 error unchanged", async () => { + const pacer = new ProviderPacer("openai:test") + const error = { status: 503 } + const request = vi.fn().mockRejectedValue(limited(1)) + const result = pacer.run("retry", undefined, request).catch((caught) => caught) + await vi.advanceTimersByTimeAsync(40) + expect(request.mock.calls.length).toBeGreaterThan(30) + request.mockRejectedValueOnce(error) + await vi.advanceTimersByTimeAsync(1) + expect(await result).toBe(error) + expect(vi.getTimerCount()).toBe(0) + }) + + it("cancels permit waits and pause waits before invoking the caller's gate", async () => { + const pacer = new ProviderPacer("openai:test") + const active = deferred() + const activeResults = Array.from({ length: 10 }, () => pacer.run(undefined, undefined, () => active.promise)) + await flush() + const controller = new AbortController() + const gate = vi.fn().mockResolvedValue(undefined) + const blocked = pacer.run("blocked", controller.signal, gate).catch((error) => error) + controller.abort() + expect(await blocked).toBe(controller.signal.reason) + expect(gate).not.toHaveBeenCalled() + active.resolve() + await Promise.all(activeResults) + pacer.reportRateLimit(limited(30_000), pacer.markIssued()) + const pausedController = new AbortController() + const paused = pacer.run("paused", pausedController.signal, gate).catch((error) => error) + pausedController.abort() + expect(await paused).toBe(pausedController.signal.reason) + expect(pacer.state().waitingPaths).toEqual([]) + expect(pacer.state().pausedUntil).toBe(30_000) + expect(gate).not.toHaveBeenCalled() + await expect(pacer.run("already aborted", pausedController.signal, gate)).rejects.toBe( + pausedController.signal.reason, + ) + }) + + it("retains uncapped delays beyond 32-bit timers without sending early", async () => { + const pacer = new ProviderPacer("openai:test") + const delay = 2 ** 31 + 60_000 + pacer.reportRateLimit({ status: 429, headers: { "retry-after-ms": String(delay) } }, pacer.markIssued()) + const request = vi.fn().mockResolvedValue("ok") + const result = pacer.run("long", undefined, request) + expect(pacer.state().pausedUntil).toBe(delay) + await vi.advanceTimersByTimeAsync(2 ** 31 - 1) + expect(request).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(1) + await vi.advanceTimersByTimeAsync(60_000) + expect(request).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(await result).toBe("ok") + }) +}) + +describe("ProviderPacerRegistry and scope", () => { + it("normalizes URL identity, strips credentials and hashes keys without disclosing configuration", () => { + const scope = makeRateLimitScope("openai", "https://user:password@EXAMPLE.test:443/v1/", "secret") + expect(scope).toBe(makeRateLimitScope("openai", "https://example.test/v1", "secret")) + expect(scope).not.toContain("secret") + expect(scope).not.toContain("password") + expect(scope).not.toContain("example") + expect(scope).not.toBe(makeRateLimitScope("openai", "https://example.test/v1", "other")) + expect(makeRateLimitScope("bedrock", undefined, undefined, "region/profile")).not.toBe( + makeRateLimitScope("bedrock", undefined, undefined, "other/profile"), + ) + expect(makeRateLimitScope("ollama", "http://localhost:11434")).toMatch(/^ollama:[a-f0-9]{64}$/) + const keyHash = createHash("sha256").update("secret").digest("hex") + expect(scope).toBe( + `openai:${createHash("sha256") + .update(JSON.stringify(["openai", "https://example.test/v1", keyHash, ""])) + .digest("hex")}`, + ) + }) + + it("notifies status, returns detached paused snapshots and deduplicates all waiting paths", async () => { + const registry = new ProviderPacerRegistry() + const changed = vi.fn() + const disposables: { dispose(): void }[] = [] + registry.onDidChange(changed, undefined, disposables) + const embedder = registry.get("openai:https://private.test?key=secret") + const store = registry.get("qdrant:https://private-store.test") + embedder.reportRateLimit(limited(1000), embedder.markIssued()) + store.reportRateLimit(limited(2000), store.markIssued()) + const controller = new AbortController() + const outcomes = [embedder, embedder, store].map((pacer) => + pacer.run("same.ts", controller.signal, async () => undefined).catch((error) => error), + ) + await flush() + expect(registry.waitingPaths()).toEqual(["same.ts"]) + expect(registry.providerWaits()).toEqual([ + { service: "openai", resumeAt: 1000, waitingFiles: ["same.ts"] }, + { service: "qdrant", resumeAt: 2000, waitingFiles: ["same.ts"] }, + ]) + registry.providerWaits()[0].waitingFiles.push("mutated") + expect(registry.waitingPaths()).toEqual(["same.ts"]) + controller.abort() + await Promise.all(outcomes) + expect(registry.waitingPaths()).toEqual([]) + expect(changed).toHaveBeenCalled() + disposables[0].dispose() + changed.mockClear() + await vi.advanceTimersByTimeAsync(2000) + expect(changed).not.toHaveBeenCalled() + expect(registry.providerWaits()).toEqual([]) + expect(registry.get("https://user:secret@host").service).toBe("https") + expect(registry.get("unsafe/key:secret").service).toBe("provider") + }) +}) diff --git a/src/services/code-index/__tests__/search-service.spec.ts b/src/services/code-index/__tests__/search-service.spec.ts new file mode 100644 index 0000000000..017cd2d8fe --- /dev/null +++ b/src/services/code-index/__tests__/search-service.spec.ts @@ -0,0 +1,367 @@ +import * as path from "path" +import { CodeIndexSearchService } from "../search-service" +import { FileFailureRegistry } from "../file-failure-registry" +import { ProviderPacerRegistry } from "../provider-pacer" +import { SearchProviderRateLimitError } from "../shared/indexing-errors" +import { CodeIndexStateManager, type IndexingState } from "../state-manager" +import type { IEmbedder } from "../interfaces/embedder" +import type { IVectorStore } from "../interfaces/vector-store" +import { EmbeddingError } from "../shared/indexing-errors" + +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) + +vi.mock("../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), info: vi.fn() } })) + +function prepare(state: IndexingState = "Indexed", failed = true, exists = true) { + const failures = new FileFailureRegistry() + if (failed) { + for (const file of ["src/a.ts", "other/b.ts"]) { + failures.renew(file, "revision") + failures.record(file, "revision", "embed", "unavailable", 3, "exhausted", "server") + } + } + const config = { + isFeatureEnabled: true, + isFeatureConfigured: true, + currentSearchMinScore: 0.4, + currentSearchMaxResults: 12, + } + const stateManager = new CodeIndexStateManager() + stateManager.setSystemState(state, "Original status") + const setSystemState = vi.spyOn(stateManager, "setSystemState") + const embedder = { rateLimitScope: "embedding:test", createEmbeddings: vi.fn() } + embedder.createEmbeddings.mockResolvedValue({ embeddings: [[1, 0]] }) + const store = { + search: vi.fn().mockResolvedValue([]), + collectionExists: vi.fn().mockResolvedValue(exists), + } + const getCurrentStatus = vi.fn(() => stateManager.getCurrentStatus()) + const pacers = new ProviderPacerRegistry() + const service = new CodeIndexSearchService( + config, + stateManager, + embedder, + store, + failures, + pacers, + getCurrentStatus, + ) + const statusBefore = stateManager.getCurrentStatus() + const failuresBefore = failures.list() + const registryMethods = ["record", "clear", "renew", "resetAllBudgets", "getRetryBudget"] as const + const registryWrites = registryMethods.map((method) => vi.spyOn(failures, method)) + function expectUnchanged() { + expect(setSystemState).not.toHaveBeenCalled() + expect(stateManager.getCurrentStatus()).toEqual(statusBefore) + expect(failures.list()).toEqual(failuresBefore) + for (const write of registryWrites) expect(write).not.toHaveBeenCalled() + } + return { service, failures, pacers, store, embedder, config, stateManager, getCurrentStatus, expectUnchanged } +} + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +describe("CodeIndexSearchService", () => { + it.each(["Indexed", "Indexing", "Standby"] as const)( + "returns every failed path with matches or no matches in %s", + async (state) => { + const { service, store } = prepare(state) + for (const results of [[], [{ id: "point", score: 0.9 }]]) { + store.search.mockResolvedValueOnce(results) + await expect(service.searchIndex("query", "src/folder")).resolves.toEqual({ + results, + incomplete: true, + failedPaths: ["src/a.ts", "other/b.ts"], + waitingPaths: [], + }) + } + expect(store.search).toHaveBeenCalledWith([1, 0], path.normalize("src/folder"), 0.4, 12) + expect(store.collectionExists).toHaveBeenCalledTimes(state === "Standby" ? 2 : 0) + }, + ) + + it.each(["Indexed", "Indexing"] as const)("allows %s without known failures", async (state) => { + const { service, expectUnchanged } = prepare(state, false) + await expect(service.searchIndex("query")).resolves.toEqual({ + results: [], + incomplete: false, + failedPaths: [], + waitingPaths: [], + }) + expectUnchanged() + }) + + it.each([ + ["Standby", false, true], + ["Standby", true, false], + ["Error", true, true], + ["Stopping", true, true], + ] as const)("rejects %s / failures %s / collection %s", async (state, failed, exists) => { + const { service, embedder, store, expectUnchanged } = prepare(state, failed, exists) + await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.collectionExists).toHaveBeenCalledTimes(state === "Standby" && failed ? 1 : 0) + expectUnchanged() + }) + + it.each(["isFeatureEnabled", "isFeatureConfigured"] as const)("rejects when %s is false", async (setting) => { + const { service, config, embedder, expectUnchanged } = prepare() + config[setting] = false + await expect(service.searchIndex("query")).rejects.toThrow("disabled or not configured") + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expectUnchanged() + }) + + it("uses the manager's computed status instead of raw progress state", async () => { + const { service, getCurrentStatus, embedder, stateManager } = prepare("Indexing") + getCurrentStatus.mockReturnValue({ ...stateManager.getCurrentStatus(), systemStatus: "Error" }) + await expect(service.searchIndex("query")).rejects.toThrow("Current state: Error") + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it.each(["Error", "Stopping"] as const)("does not bypass %s reached during collection lookup", async (state) => { + const { service, store, getCurrentStatus, stateManager, embedder } = prepare("Standby") + store.collectionExists.mockImplementation(async () => { + getCurrentStatus.mockReturnValue({ ...stateManager.getCurrentStatus(), systemStatus: state }) + return true + }) + await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it.each([true, false])("preserves cancellation during collection lookup, rejects=%s", async (rejects) => { + const { service, store, embedder, expectUnchanged } = prepare("Standby") + const controller = new AbortController() + const reason = Object.freeze({ cancelled: true }) + store.collectionExists.mockImplementation(async () => { + controller.abort(reason) + if (rejects) throw new Error("collection lookup failed") + return true + }) + await expect(service.searchIndex("query", undefined, controller.signal)).rejects.toBe(reason) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expectUnchanged() + }) + + it.each([true, false])("snapshots failures at response time, recovery=%s", async (recovered) => { + const { service, failures, store } = prepare() + store.search.mockImplementation(async () => { + failures.clear("src/a.ts") + failures.clear("other/b.ts") + if (!recovered) { + failures.renew("pending.ts", "new") + failures.record("pending.ts", "new", "embed", "waiting", 1, "retrying", "server") + } + return [] + }) + await expect(service.searchIndex("query", "src")).resolves.toEqual({ + results: [], + incomplete: !recovered, + failedPaths: recovered ? [] : ["pending.ts"], + waitingPaths: [], + }) + }) + + it("supports the state-manager fallback when no status supplier is provided", async () => { + const { config, stateManager, embedder, store, failures, pacers } = prepare() + const service = new CodeIndexSearchService(config, stateManager, embedder, store, failures, pacers) + await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: true }) + }) + + it.each(["Indexed", "Indexing", "Standby"] as const)( + "keeps %s health unchanged on store failure", + async (state) => { + const { service, store, expectUnchanged } = prepare(state) + const error = new Error("store unavailable") + store.search.mockRejectedValue(error) + await expect(service.searchIndex("query")).rejects.toBe(error) + expect(store.search).toHaveBeenCalledOnce() + expectUnchanged() + }, + ) + + it("rejects an empty embedding without changing indexing health", async () => { + const { service, embedder, store, expectUnchanged } = prepare() + embedder.createEmbeddings.mockResolvedValue({ embeddings: [] }) + await expect(service.searchIndex("query")).rejects.toThrow("Failed to generate embedding") + expect(store.search).not.toHaveBeenCalled() + expectUnchanged() + }) + + it.each([401, 408, 409])("does not retry permanent query errors (%s)", async (status) => { + const { service, embedder, expectUnchanged } = prepare() + const error = Object.freeze(Object.assign(new Error("unavailable"), { status })) + embedder.createEmbeddings.mockRejectedValue(error) + await expect(service.searchIndex("query")).rejects.toMatchObject({ + cause: error, + classification: "permanent", + attempts: 1, + }) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(error).not.toHaveProperty("attempts") + expectUnchanged() + }) + + it.each([503])("uses one two-retry budget per query (%s)", async (status) => { + vi.useFakeTimers() + const { service, embedder, expectUnchanged } = prepare() + const error = Object.assign(new Error("unavailable"), { status }) + embedder.createEmbeddings.mockRejectedValue(error) + for (const expectedCalls of [3, 6]) { + const outcome = service.searchIndex("query").catch((failure: unknown) => failure) + await vi.advanceTimersByTimeAsync(499) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls - 2) + await vi.advanceTimersByTimeAsync(1) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls - 1) + await vi.advanceTimersByTimeAsync(1000) + await expect(outcome).resolves.toMatchObject({ cause: error, attempts: 3 }) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls) + } + expectUnchanged() + }) + + it.each([60_000, 61_000, 120_000])( + "reports a query 429 immediately with full %s ms pause", + async (retryAfterMs) => { + vi.useFakeTimers() + const { service, embedder, pacers, expectUnchanged } = prepare() + const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs }) + embedder.createEmbeddings.mockRejectedValue(error) + await expect(service.searchIndex("query")).rejects.toMatchObject({ + name: SearchProviderRateLimitError.name, + message: "embeddings:searchProviderRateLimited", + cause: error, + }) + expect(pacers.get(embedder.rateLimitScope).state().pausedUntil).toBe(Date.now() + retryAfterMs) + expect(pacers.waitingPaths()).toEqual([]) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expectUnchanged() + }, + ) + + it("does not wait for provider pauses or indexing permits", async () => { + vi.useFakeTimers() + const { service, embedder, pacers } = prepare() + const pacer = pacers.get(embedder.rateLimitScope) + pacer.reportRateLimit( + new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 120_000 }), + pacer.markIssued(), + ) + await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: true }) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + }) + + it("bypasses saturated indexing permits and marks each query request issued", async () => { + const { service, embedder, pacers, expectUnchanged } = prepare("Indexed", false) + const pacer = pacers.get(embedder.rateLimitScope) + let release!: () => void + const held = new Promise((resolve) => { + release = resolve + }) + const indexing = Array.from({ length: pacer.state().limit }, (_, position) => + pacer.run(`file-${position}.ts`, undefined, () => held), + ) + await Promise.resolve() + const markIssued = vi.spyOn(pacer, "markIssued") + try { + await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: false }) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(markIssued).toHaveBeenCalledOnce() + expectUnchanged() + } finally { + release() + await Promise.all(indexing) + } + }) + + it.each(["Standby", "Error", "Stopping"] as const)("waiting-only search gate preserves %s", async (state) => { + const { service, pacers, embedder } = prepare(state, false) + vi.spyOn(pacers, "waitingPaths").mockReturnValue(["waiting.ts"]) + if (state === "Standby") { + await expect(service.searchIndex("query")).resolves.toEqual({ + results: [], + incomplete: true, + failedPaths: [], + waitingPaths: ["waiting.ts"], + }) + } else { + await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + } + }) + + it("rechecks waiting-only availability after collection lookup", async () => { + const { service, pacers, store, embedder } = prepare("Standby", false) + const waiting = vi.spyOn(pacers, "waitingPaths").mockReturnValue(["waiting.ts"]) + store.collectionExists.mockImplementation(async () => { + waiting.mockReturnValue([]) + return true + }) + await expect(service.searchIndex("query")).rejects.toThrow("Current state: Standby") + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("snapshots waiting paths only after the store responds", async () => { + const { service, pacers, store } = prepare("Indexed", false) + const waiting = vi.spyOn(pacers, "waitingPaths").mockReturnValue([]) + store.search.mockImplementation(async () => { + waiting.mockReturnValue(["late.ts"]) + return [] + }) + await expect(service.searchIndex("query")).resolves.toEqual({ + results: [], + incomplete: true, + failedPaths: [], + waitingPaths: ["late.ts"], + }) + }) + + it("cancels a retry wait with unchanged reason and no later request", async () => { + vi.useFakeTimers() + const { service, embedder, store, expectUnchanged } = prepare() + embedder.createEmbeddings.mockRejectedValue(new EmbeddingError("unavailable", { status: 503 })) + const controller = new AbortController() + const reason = Object.freeze({ cancelled: true }) + const outcome = service.searchIndex("query", undefined, controller.signal).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + controller.abort(reason) + await expect(outcome).resolves.toBe(reason) + await vi.advanceTimersByTimeAsync(120_000) + expect(embedder.createEmbeddings).toHaveBeenCalledExactlyOnceWith(["query"], undefined, controller.signal) + expect(store.search).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + expectUnchanged() + }) + + it("does not issue requests for an already cancelled caller", async () => { + const { service, embedder, store, expectUnchanged } = prepare() + const signal = AbortSignal.abort(new Error("caller ended")) + await expect(service.searchIndex("query", undefined, signal)).rejects.toBe(signal.reason) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.search).not.toHaveBeenCalled() + expectUnchanged() + }) + + it.each(["embed", "search"] as const)("preserves cancellation after in-flight %s finishes", async (activity) => { + const { service, embedder, store, expectUnchanged } = prepare() + const controller = new AbortController() + const reason = new Error("caller ended") + if (activity === "embed") { + embedder.createEmbeddings.mockImplementation(async () => { + controller.abort(reason) + return { embeddings: [[1, 0]] } + }) + } else { + store.search.mockImplementation(async () => { + controller.abort(reason) + return [] + }) + } + await expect(service.searchIndex("query", undefined, controller.signal)).rejects.toBe(reason) + expect(store.search).toHaveBeenCalledTimes(activity === "embed" ? 0 : 1) + expectUnchanged() + }) +}) diff --git a/src/services/code-index/__tests__/service-factory.spec.ts b/src/services/code-index/__tests__/service-factory.spec.ts index bc1d619af2..3b22f267a5 100644 --- a/src/services/code-index/__tests__/service-factory.spec.ts +++ b/src/services/code-index/__tests__/service-factory.spec.ts @@ -13,8 +13,36 @@ import { BedrockEmbedder } from "../embedders/bedrock" import { OpenRouterEmbedder } from "../embedders/openrouter" import type { CodeIndexConfig } from "../interfaces/config" +import { CodeIndexConfigManager } from "../config-manager" +import { CacheManager } from "../cache-manager" +import { ParseWorkerClient } from "../parse-worker-client" +import { IndexWriteGate } from "../index-write-gate" +import { GenerationAllocator } from "../generation-allocator" +import { FileFailureRegistry } from "../file-failure-registry" +import { ProviderPacerRegistry } from "../provider-pacer" +import { DirectoryScanner } from "../processors/scanner" +import { FileWatcher } from "../processors/file-watcher" +import { makeExtensionContext } from "../../../test-utils/vscode" +import ignore from "ignore" + import { clearAllMocks } from "../../../test-utils/reset" +vitest.mock("../config-manager") +vitest.mock("../cache-manager") +vitest.mock("../parse-worker-client", () => ({ + ParseWorkerClient: vitest.fn( + class { + parseFile = vitest.fn().mockResolvedValue([]) + stop = vitest.fn() + stopped = Promise.resolve() + logStopTimeout = vitest.fn() + dispose = vitest.fn() + }, + ), +})) +vitest.mock("../processors/scanner") +vitest.mock("../processors/file-watcher") + // Mock the embedders and vector store vitest.mock("../embedders/openai") vitest.mock("../embedders/ollama") @@ -57,6 +85,166 @@ import { providerIdentifiers } from "@roo-code/types/provider-identifiers" const mockGetDefaultModelId = getDefaultModelId as MockedFunction const mockGetModelDimension = getModelDimension as MockedFunction +describe("workspace service integration", () => { + beforeEach(() => clearAllMocks()) + afterEach(() => vitest.restoreAllMocks()) + + function setup(isConfigured = true) { + const context = makeExtensionContext() + const cache = new CacheManager(context, "/workspace") + const watcherCache = new CacheManager(context, "/workspace") + const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager + const config: CodeIndexConfig = { isConfigured, embedderProvider: "openai-compatible" } + Object.defineProperty(configManager, "isFeatureConfigured", { value: isConfigured }) + vitest.mocked(configManager.getConfig).mockReturnValue(config) + const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) + const vectorStore = new QdrantVectorStore("/workspace", "http://localhost:6333", 512) + const createEmbedder = vitest.spyOn(EmbedderFactory.prototype, "create").mockReturnValue(embedder) + const createVectorStore = vitest.spyOn(VectorStoreFactory.prototype, "create").mockReturnValue(vectorStore) + const gate = new IndexWriteGate() + const generations = new GenerationAllocator(cache) + const failures = new FileFailureRegistry() + const providerPacers = new ProviderPacerRegistry() + const parser = new ParseWorkerClient() + const isCurrentAttempt = vitest.fn(() => true) + const ignored = ignore() + const createFactory = () => + new CodeIndexServiceFactory( + configManager, + "/workspace", + cache, + gate, + generations, + failures, + parser, + isCurrentAttempt, + providerPacers, + ) + return { + context, + cache, + watcherCache, + configManager, + config, + embedder, + vectorStore, + createEmbedder, + createVectorStore, + gate, + generations, + failures, + parser, + isCurrentAttempt, + providerPacers, + ignored, + createFactory, + } + } + + it("injects the same workspace dependencies into both processors across service and factory recreation", () => { + const options = setup() + options.failures.renew("file.ts", "revision") + const budget = options.failures.getRetryBudget("file.ts", "revision", "embed") + budget.retriesUsed = 2 + options.failures.record("file.ts", "revision", "embed", "exhausted", 3, "exhausted", "server") + const reset = vitest.spyOn(options.failures, "resetAllBudgets") + const renew = vitest.spyOn(options.failures, "renew") + const clear = vitest.spyOn(options.failures, "clear") + const initialize = vitest.spyOn(options.generations, "initialize") + const factory = options.createFactory() + + for (const currentFactory of [factory, factory, options.createFactory()]) { + const services = currentFactory.createServices(options.context, options.watcherCache, options.ignored) + const scannerArguments = vitest.mocked(DirectoryScanner).mock.lastCall! + const watcherArguments = vitest.mocked(FileWatcher).mock.lastCall! + expect(services).toEqual({ + embedder: options.embedder, + vectorStore: options.vectorStore, + parser: options.parser, + scanner: vitest.mocked(DirectoryScanner).mock.instances.at(-1), + fileWatcher: vitest.mocked(FileWatcher).mock.instances.at(-1), + }) + expect(scannerArguments[2]).toBe(options.parser) + expect(watcherArguments[8]).toBe(options.parser) + expect(scannerArguments[3]).toBe(options.cache) + expect(watcherArguments[2]).toBe(options.watcherCache) + expect(scannerArguments[6]).toBe(options.gate) + expect(watcherArguments[9]).toBe(options.gate) + expect(scannerArguments[7]).toBe(options.generations) + expect(watcherArguments[10]).toBe(options.generations) + expect(scannerArguments[8]).toBe(options.failures) + expect(watcherArguments[11]).toBe(options.failures) + expect(scannerArguments[9]).toBe("/workspace") + expect(scannerArguments[10]).toBe(options.isCurrentAttempt) + expect(watcherArguments[12]).toBe(options.isCurrentAttempt) + expect(scannerArguments[11]).toBe(options.providerPacers) + expect(watcherArguments[13]).toBe(options.providerPacers) + for (const current of [true, false]) { + options.isCurrentAttempt.mockReturnValue(current) + expect(scannerArguments[10]?.()).toBe(current) + expect(watcherArguments[12]?.()).toBe(current) + } + } + + expect(options.configManager.getConfig).toHaveBeenCalledTimes(3) + expect(options.createEmbedder).toHaveBeenCalledWith(options.config) + expect(options.createVectorStore).toHaveBeenCalledWith(options.config, "/workspace") + expect(options.failures.getRetryBudget("file.ts", "revision", "embed")).toBe(budget) + expect(budget.retriesUsed).toBe(2) + expect(options.failures.isExhausted("file.ts", "revision", "embed")).toBe(true) + expect(reset).not.toHaveBeenCalled() + expect(renew).not.toHaveBeenCalled() + expect(clear).not.toHaveBeenCalled() + expect(initialize).not.toHaveBeenCalled() + expect(ParseWorkerClient).toHaveBeenCalledTimes(1) + expect(options.parser.stop).not.toHaveBeenCalled() + }) + + it("preserves the first three arguments and shares defaults within one factory, not across workspaces", async () => { + const options = setup() + const firstFactory = new CodeIndexServiceFactory(options.configManager, "/workspace", options.cache) + const otherFactory = new CodeIndexServiceFactory(options.configManager, "/other", options.cache) + const first = firstFactory.createServices(options.context, options.cache, options.ignored) + const second = firstFactory.createServices(options.context, options.cache, options.ignored) + const other = otherFactory.createServices(options.context, options.cache, options.ignored) + expect(first.parser).toBeInstanceOf(ParseWorkerClient) + expect(first.parser).toBe(second.parser) + expect(first.parser).not.toBe(other.parser) + const scannerCalls = vitest.mocked(DirectoryScanner).mock.calls + const watcherCalls = vitest.mocked(FileWatcher).mock.calls + for (const [scannerPosition, watcherPosition] of [ + [6, 9], + [7, 10], + [8, 11], + [11, 13], + ] as const) { + expect(scannerCalls[0][scannerPosition]).toBe(scannerCalls[1][scannerPosition]) + expect(scannerCalls[0][scannerPosition]).toBe(watcherCalls[0][watcherPosition]) + expect(scannerCalls[0][scannerPosition]).not.toBe(scannerCalls[2][scannerPosition]) + } + first.parser.stop() + expect(first.parser.stop).toHaveBeenCalledOnce() + await first.parser.stopped + first.parser.logStopTimeout() + first.parser.dispose() + expect(first.parser.logStopTimeout).toHaveBeenCalledOnce() + expect(first.parser.dispose).toHaveBeenCalledOnce() + }) + + it("rejects unconfigured indexing before constructing processors or reading configuration", () => { + const options = setup(false) + const factory = options.createFactory() + expect(() => factory.createServices(options.context, options.cache, options.ignored)).toThrow( + "serviceFactory.codeIndexingNotConfigured", + ) + expect(options.configManager.getConfig).not.toHaveBeenCalled() + expect(options.createEmbedder).not.toHaveBeenCalled() + expect(options.createVectorStore).not.toHaveBeenCalled() + expect(DirectoryScanner).not.toHaveBeenCalled() + expect(FileWatcher).not.toHaveBeenCalled() + }) +}) + describe("CodeIndexServiceFactory", () => { let factory: CodeIndexServiceFactory let embedderFactory: EmbedderFactory diff --git a/src/services/code-index/__tests__/state-manager.spec.ts b/src/services/code-index/__tests__/state-manager.spec.ts new file mode 100644 index 0000000000..eaa519480b --- /dev/null +++ b/src/services/code-index/__tests__/state-manager.spec.ts @@ -0,0 +1,113 @@ +import { CodeIndexStateManager } from "../state-manager" +import { codeIndexLog } from "../code-index-log" +import { clearAllMocks } from "../../../test-utils/reset" + +vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) +vi.mock("vscode", () => ({ + EventEmitter: class { + event = vi.fn() + fire = vi.fn() + dispose = vi.fn() + }, +})) + +beforeEach(() => clearAllMocks()) + +describe("CodeIndexStateManager", () => { + it("logs changed state and messages with diagnostic context only", () => { + const state = new CodeIndexStateManager() + state.setDiagnosticContext({ source: "manager", attempt: 12 }) + state.setSystemState("Indexing", "Starting", "start-request") + expect(codeIndexLog.info).toHaveBeenLastCalledWith("State transition", { + oldState: "Standby", + newState: "Indexing", + message: "Starting", + source: "start-request", + attempt: 12, + }) + state.setSystemState("Indexing", "Starting") + expect(codeIndexLog.info).toHaveBeenCalledOnce() + state.setSystemState("Standby") + expect(codeIndexLog.info).toHaveBeenLastCalledWith("State transition", { + oldState: "Indexing", + newState: "Standby", + message: "Ready.", + source: "manager", + attempt: 12, + }) + expect(state.getCurrentStatus()).not.toHaveProperty("attempt") + }) + + it.each(["blocks", "files"] as const)("logs entry but not each %s progress update", (unit) => { + const state = new CodeIndexStateManager() + state.setDiagnosticContext({ source: "manager", attempt: "attempt-2" }) + const progress = (processed: number) => { + if (unit === "blocks") state.reportBlockIndexingProgress(processed, 3, "scan") + else state.reportFileQueueProgress(processed, 3, "file.ts", "scan") + } + progress(1) + progress(2) + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Indexing", + processedItems: 2, + totalItems: 3, + currentItemUnit: unit, + }) + expect(codeIndexLog.info).toHaveBeenCalledExactlyOnceWith( + "State transition", + expect.objectContaining({ + oldState: "Standby", + newState: "Indexing", + source: "scan", + attempt: "attempt-2", + }), + ) + state.setSystemState("Standby") + expect(codeIndexLog.info).toHaveBeenLastCalledWith( + "State transition", + expect.objectContaining({ source: "manager", attempt: "attempt-2" }), + ) + }) + + it.each([ + ["Error", "blocks"], + ["Error", "files"], + ["Stopping", "blocks"], + ["Stopping", "files"], + ] as const)("does not let %s be overwritten by %s progress", (systemStatus, unit) => { + const state = new CodeIndexStateManager() + state.reportBlockIndexingProgress(1, 3) + state.setSystemState(systemStatus, "Preserved message") + const before = state.getCurrentStatus() + const fire = vi.spyOn(state["_progressEmitter"], "fire") + vi.mocked(codeIndexLog.info).mockClear() + if (unit === "blocks") state.reportBlockIndexingProgress(3, 10) + else state.reportFileQueueProgress(3, 10, "late.ts") + expect(state.getCurrentStatus()).toEqual(before) + expect(fire).not.toHaveBeenCalled() + expect(codeIndexLog.info).not.toHaveBeenCalled() + }) + + it.each(["Error", "Stopping"] as const)("allows an explicit restart from %s", (systemStatus) => { + const state = new CodeIndexStateManager() + state.setSystemState(systemStatus, "Previous attempt") + state.setSystemState("Indexing", "New attempt") + state.reportBlockIndexingProgress(1, 2) + expect(state.getCurrentStatus()).toMatchObject({ systemStatus: "Indexing", processedItems: 1, totalItems: 2 }) + }) + + it("does not emit duplicate progress and resets counters on completion", () => { + const state = new CodeIndexStateManager() + state.reportBlockIndexingProgress(1, 2) + const fire = vi.spyOn(state["_progressEmitter"], "fire") + state.reportBlockIndexingProgress(1, 2) + expect(fire).not.toHaveBeenCalled() + state.setSystemState("Indexed") + expect(state.getCurrentStatus()).toMatchObject({ + systemStatus: "Indexed", + message: "Index up-to-date.", + processedItems: 0, + totalItems: 0, + }) + }) +}) diff --git a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts index 838fffdc6e..6008ba3d81 100644 --- a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts +++ b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts @@ -10,7 +10,7 @@ describe("WorkspaceIndexingEnablementManager", () => { isFeatureEnabled: true, isFeatureConfigured: true, initialize: vi.fn().mockResolvedValue({ requiresRestart: false }), - startIndexing: vi.fn().mockResolvedValue(undefined), + startIndexing: vi.fn<(contextProxy?: ContextProxy) => Promise>().mockResolvedValue(undefined), stopIndexing: vi.fn(), getCurrentStatus: vi.fn().mockReturnValue({ systemStatus: "Standby", @@ -21,6 +21,7 @@ describe("WorkspaceIndexingEnablementManager", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: false, + failedFiles: [], }), } return { @@ -34,15 +35,21 @@ describe("WorkspaceIndexingEnablementManager", () => { } } - it("persists enablement, initializes and starts in order without awaiting indexing", async () => { + it("persists enablement and awaits one start before publishing", async () => { const { manager, indexing, provider } = setup() - manager.startIndexing.mockReturnValue(new Promise(() => {})) - await indexing.setEnabled(true, provider) + let resolveStart: () => void = () => {} + manager.startIndexing.mockReturnValue(new Promise((resolve) => (resolveStart = resolve))) + const pending = indexing.setEnabled(true, provider) + await Promise.resolve() expect(manager.setWorkspaceEnabled).toHaveBeenCalledWith(true) - expect(manager.initialize).toHaveBeenCalledWith(provider.contextProxy) - expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.initialize) - expect(manager.initialize).toHaveBeenCalledBefore(manager.startIndexing) + expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) + expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.startIndexing) + expect(manager.initialize).not.toHaveBeenCalled() expect(manager.stopIndexing).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + resolveStart() + await pending + expect(provider.postMessageToWebview).toHaveBeenCalledOnce() }) it("persists disablement before stopping", async () => { @@ -65,11 +72,14 @@ describe("WorkspaceIndexingEnablementManager", () => { expect(manager.stopIndexing).not.toHaveBeenCalled() }) - it("logs background indexing failures", async () => { + it("propagates start failures without publishing success", async () => { const { manager, indexing, provider } = setup() - manager.startIndexing.mockRejectedValue(new Error("failed")) - await expect(indexing.setEnabled(true, provider)).resolves.toBeUndefined() - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: failed") + const error = new Error("failed") + manager.startIndexing.mockRejectedValue(error) + await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) + expect(manager.startIndexing).toHaveBeenCalledOnce() + expect(manager.initialize).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() }) it.each([true, false])("publishes the current status after setting enabled=%s", async (enabled) => { @@ -89,15 +99,12 @@ describe("WorkspaceIndexingEnablementManager", () => { await expect(indexing.setEnabled(false, provider)).rejects.toBe(error) }) - it.each(["setWorkspaceEnabled", "initialize"] as const)( - "propagates %s failures without starting", - async (method) => { - const { manager, indexing, provider } = setup() - const error = new Error("failed") - manager[method].mockRejectedValue(error) - await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) - expect(manager.startIndexing).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - }, - ) + it("propagates persistence failures without starting", async () => { + const { manager, indexing, provider } = setup() + const error = new Error("failed") + manager.setWorkspaceEnabled.mockRejectedValue(error) + await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) + expect(manager.startIndexing).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + }) }) diff --git a/src/services/code-index/cache-manager.ts b/src/services/code-index/cache-manager.ts index 7807aac69d..0a3d7d10c2 100644 --- a/src/services/code-index/cache-manager.ts +++ b/src/services/code-index/cache-manager.ts @@ -1,123 +1,153 @@ import * as vscode from "vscode" import { createHash } from "crypto" -import { ICacheManager } from "./interfaces/cache" +import type { ICacheManager } from "./interfaces/cache" import debounce from "lodash.debounce" import { safeWriteJson } from "../../utils/safeWriteJson" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" +import { codeIndexLog } from "./code-index-log" + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value) +} -/** - * Manages the cache for code indexing - */ export class CacheManager implements ICacheManager { - private cachePath: vscode.Uri + private readonly cachePath: vscode.Uri private fileHashes: Record = {} - private _debouncedSaveCache: () => void + private openFiles: Record = {} + private watermark?: number + private saveTail: Promise = Promise.resolve() + private readonly debouncedSave + public loadSucceeded = false + public loadedAt = 0 - /** - * Creates a new cache manager - * @param context VS Code extension context - * @param workspacePath Path to the workspace - */ constructor(context: vscode.ExtensionContext, workspacePath: string) { this.cachePath = vscode.Uri.joinPath( context.globalStorageUri, `roo-index-cache-${createHash("sha256").update(workspacePath).digest("hex")}.json`, ) - this._debouncedSaveCache = debounce(async () => { - await this._performSave() + this.debouncedSave = debounce(() => { + void this.saveNow().catch((error: unknown) => codeIndexLog.error("Failed to save cache:", error)) }, 1500) } - /** - * Initializes the cache manager by loading the cache file - */ async initialize(): Promise { + this.fileHashes = {} + this.openFiles = {} + this.watermark = undefined + this.loadSucceeded = false try { - const cacheData = await vscode.workspace.fs.readFile(this.cachePath) - this.fileHashes = JSON.parse(cacheData.toString()) - } catch (error) { + const buffer = await vscode.workspace.fs.readFile(this.cachePath) + const data: unknown = JSON.parse(Buffer.from(buffer).toString()) + if (!isRecord(data)) throw new Error("Invalid indexing cache") + if (data.version === 2) { + if (!isRecord(data.fileHashes) || !isRecord(data.openFiles)) { + throw new Error("Invalid indexing cache records") + } + for (const [filePath, hash] of Object.entries(data.fileHashes)) { + if (typeof hash !== "string") throw new Error("Invalid indexing hash") + this.fileHashes[filePath] = hash + } + for (const [filePath, open] of Object.entries(data.openFiles)) { + if (open !== true) throw new Error("Invalid open-file record") + this.openFiles[filePath] = true + delete this.fileHashes[filePath] + } + if ( + typeof data.generationWatermark === "number" && + Number.isSafeInteger(data.generationWatermark) && + data.generationWatermark >= 0 + ) { + this.watermark = data.generationWatermark + } + } else { + for (const [filePath, hash] of Object.entries(data)) { + if (typeof hash !== "string") throw new Error("Invalid legacy indexing cache") + this.fileHashes[filePath] = hash + } + } + this.loadSucceeded = true + } catch { this.fileHashes = {} - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "initialize", - }) + this.openFiles = {} + this.watermark = undefined } + this.loadedAt = Date.now() } - /** - * Saves the cache to disk - */ - private async _performSave(): Promise { - try { - await safeWriteJson(this.cachePath.fsPath, this.fileHashes) - } catch (error) { - console.error("Failed to save cache:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "_performSave", - }) + get generationWatermark(): number | undefined { + return this.watermark + } + + async reserveGenerations(watermark: number): Promise { + this.watermark = watermark + await this.saveNow() + } + + async saveNow(): Promise { + this.debouncedSave.cancel() + const data = { + version: 2, + fileHashes: { ...this.fileHashes }, + openFiles: { ...this.openFiles }, + generationWatermark: this.watermark, } + // An older debounced snapshot must never overwrite durable invalidation. + const pending = this.saveTail.catch(() => {}).then(() => safeWriteJson(this.cachePath.fsPath, data)) + this.saveTail = pending + await pending } - /** - * Clears the cache file by writing an empty object to it - */ async clearCacheFile(): Promise { - try { - await safeWriteJson(this.cachePath.fsPath, {}) - this.fileHashes = {} - } catch (error) { - console.error("Failed to clear cache file:", error, this.cachePath) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "clearCacheFile", - }) - } + this.fileHashes = {} + this.openFiles = {} + await this.saveNow() } - /** - * Gets the hash for a file path - * @param filePath Path to the file - * @returns The hash for the file or undefined if not found - */ - getHash(filePath: string): string | undefined { - return this.fileHashes[filePath] + async invalidateFile(filePath: string): Promise { + delete this.fileHashes[filePath] + this.openFiles[filePath] = true + await this.saveNow() } - /** - * Updates the hash for a file path - * @param filePath Path to the file - * @param hash New hash value - */ - updateHash(filePath: string, hash: string): void { + completeFile(filePath: string, hash: string): void { this.fileHashes[filePath] = hash - this._debouncedSaveCache() + delete this.openFiles[filePath] + this.debouncedSave() } - /** - * Deletes the hash for a file path - * @param filePath Path to the file - */ - deleteHash(filePath: string): void { + completeRemoval(filePath: string): void { delete this.fileHashes[filePath] - this._debouncedSaveCache() + delete this.openFiles[filePath] + this.debouncedSave() + } + + updateHash(filePath: string, hash: string): void { + this.completeFile(filePath, hash) + } + + deleteHash(filePath: string): void { + this.completeRemoval(filePath) + } + + getOpenFiles(): Record { + return { ...this.openFiles } + } + + getTrackedPaths(): string[] { + return [...new Set([...Object.keys(this.fileHashes), ...Object.keys(this.openFiles)])] + } + + isTracked(filePath: string): boolean { + return Object.hasOwn(this.fileHashes, filePath) || Object.hasOwn(this.openFiles, filePath) + } + + getHash(filePath: string): string | undefined { + return Object.hasOwn(this.fileHashes, filePath) ? this.fileHashes[filePath] : undefined } - /** - * Flushes any pending debounced cache writes to disk immediately. - */ async flush(): Promise { - await this._performSave() + await this.saveNow() } - /** - * Gets a copy of all file hashes - * @returns A copy of the file hashes record - */ getAllHashes(): Record { return { ...this.fileHashes } } diff --git a/src/services/code-index/code-index-log-host.ts b/src/services/code-index/code-index-log-host.ts new file mode 100644 index 0000000000..3bab4a12f3 --- /dev/null +++ b/src/services/code-index/code-index-log-host.ts @@ -0,0 +1,8 @@ +import { window } from "vscode" +import { codeIndexLog, type CodeIndexLogSink } from "./code-index-log" + +const createHostSink = (): CodeIndexLogSink => window.createOutputChannel("Zoo Code Indexing", { log: true }) + +export function initializeCodeIndexHostLogging(): void { + codeIndexLog.setHostSinkFactory(createHostSink) +} diff --git a/src/services/code-index/code-index-log.ts b/src/services/code-index/code-index-log.ts new file mode 100644 index 0000000000..de88f3a3c3 --- /dev/null +++ b/src/services/code-index/code-index-log.ts @@ -0,0 +1,138 @@ +import { isMainThread, parentPort } from "node:worker_threads" + +export type CodeIndexLogLevel = "info" | "warn" | "error" | "debug" +export interface CodeIndexLogConfiguration { + workspacePaths?: readonly string[] + secrets?: readonly string[] +} +export interface CodeIndexFailureDiagnostic { + path: string + activity: string + reason: string + attempt: number + willRetry: boolean + [key: string]: unknown +} + +export type CodeIndexLogSink = Record void> & { dispose(): void } + +const redacted = "[REDACTED]" +const sensitiveField = /^(?:authorization|.*api[-_]?key|.*secret|.*password|.*token)$/i +const contentField = + /^(?:content|contents|text|texts|input|inputs|body|request|response|data|embedding|embeddings|vectors)$/i +const escapePattern = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +let workspacePattern: RegExp | undefined +let secretPattern: RegExp | undefined +let channel: CodeIndexLogSink | undefined +let hostSinkFactory: (() => CodeIndexLogSink) | undefined + +function configure(configuration: CodeIndexLogConfiguration): void { + if (configuration.workspacePaths !== undefined) { + const roots = configuration.workspacePaths + .filter(Boolean) + .map((root) => root.replace(/[\\/]+$/, "")) + .filter(Boolean) + .sort((left, right) => right.length - left.length) + workspacePattern = roots.length + ? new RegExp( + `(?:${roots.map((root) => root.split(/[\\/]/).map(escapePattern).join("[\\\\/]")).join("|")})(?:[\\\\/]|(?=$|[\\s\"',:)]))`, + "gi", + ) + : undefined + } + if (configuration.secrets !== undefined) { + const secrets = [...new Set(configuration.secrets.filter(Boolean))].sort( + (left, right) => right.length - left.length, + ) + secretPattern = secrets.length ? new RegExp(secrets.map(escapePattern).join("|"), "g") : undefined + } +} + +function redact(value: string): string { + let result = value.replace(/\b([a-z][a-z\d+.-]*:\/\/)[^\s/?#@]*@/gi, `$1${redacted}@`) + result = result.replace(/\b(authorization|api[-_]?key)(["']?\s*[:=]\s*["']?)[^\r\n"',;}]+/gi, `$1$2${redacted}`) + if (workspacePattern) result = result.replace(workspacePattern, "") + if (secretPattern) result = result.replace(secretPattern, redacted) + return result +} + +function sanitize(value: unknown, seen = new WeakSet()): unknown { + if (typeof value === "string") return redact(value) + if (value === null || typeof value !== "object") return value + if (seen.has(value)) return "[Circular]" + seen.add(value) + if (value instanceof Error) { + // SDK errors can hold request bodies and credentials outside the message. + const error = value as Error & { + code?: unknown + status?: unknown + statusCode?: unknown + classification?: unknown + } + return sanitize( + { + name: error.name, + message: error.message, + code: error.code, + status: error.status, + statusCode: error.statusCode, + classification: error.classification, + }, + seen, + ) + } + if (Array.isArray(value)) return value.map((entry) => sanitize(entry, seen)) + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + redact(key), + sensitiveField.test(key) || contentField.test(key) ? redacted : sanitize(entry, seen), + ]), + ) +} + +function write(level: CodeIndexLogLevel, message: string, fields?: unknown): void { + const safeMessage = redact(message) + const safeFields = sanitize(fields) + if (!isMainThread) { + parentPort?.postMessage({ id: 0, diagnostic: { level, message: safeMessage, fields: safeFields } }) + return + } + channel ??= hostSinkFactory?.() + if (!channel) return + const suffix = safeFields === undefined ? "" : ` ${JSON.stringify(safeFields)}` + channel[level](`[CodeIndex] ${safeMessage}${suffix}`) +} + +function createFailureBatch(fields?: Record) { + let failures: CodeIndexFailureDiagnostic[] = [] + return { + record(failure: CodeIndexFailureDiagnostic): void { + failures.push({ ...failure }) + }, + flush(): void { + if (failures.length === 0) return + const completed = failures + failures = [] + write("warn", "File operation failures", { ...fields, failureCount: completed.length, failures: completed }) + }, + } +} + +export const codeIndexLog = { + configure, + setHostSinkFactory(factory: () => CodeIndexLogSink): void { + if (hostSinkFactory === factory) return + codeIndexLog.dispose() + hostSinkFactory = factory + }, + redact, + createFailureBatch, + info: (message: string, fields?: unknown) => write("info", message, fields), + warn: (message: string, fields?: unknown) => write("warn", message, fields), + error: (message: string, fields?: unknown) => write("error", message, fields), + debug: (message: string, fields?: unknown) => write("debug", message, fields), + dispose(): void { + channel?.dispose() + channel = undefined + }, +} diff --git a/src/services/code-index/code-index-scan-executor.ts b/src/services/code-index/code-index-scan-executor.ts index 54760c50f2..616be1e1a7 100644 --- a/src/services/code-index/code-index-scan-executor.ts +++ b/src/services/code-index/code-index-scan-executor.ts @@ -1,83 +1,75 @@ -import type { IDirectoryScanner, IVectorStore } from "./interfaces" +import type { IDirectoryScanner } from "./interfaces" import type { CodeIndexStateManager } from "./state-manager" -import { t } from "../../i18n" +import { ParserFatalError } from "./parser-errors" +import { EmbeddingError, IndexingRetryError, getIndexingErrorStatusCode } from "./shared/indexing-errors" +import { codeIndexLog } from "./code-index-log" + +function isEmbeddingConfigurationError(error: Error): boolean { + let current: unknown = error + let embedding = false + const visited = new Set() + while (current instanceof Error && !visited.has(current)) { + visited.add(current) + if ( + current instanceof EmbeddingError || + (current instanceof IndexingRetryError && current.activity === "embed") + ) { + embedding = true + } + current = current.cause + } + const status = getIndexingErrorStatusCode(error) + return embedding && status !== undefined && [400, 401, 403, 404, 422].includes(status) +} -/** Executes scans; cancellation cleanup, watcher startup and completion remain with the orchestrator. */ export class CodeIndexScanExecutor { + public partial = false + constructor( private readonly workspacePath: string, private readonly scanner: IDirectoryScanner, - private readonly vectorStore: Pick, private readonly stateManager: Pick, + private readonly isCurrentAttempt: () => boolean = () => true, ) {} - /** Returns false when the scanner returns after cancellation, preserving the owner's normal-return path. */ - async runIncrementalScan(signal: AbortSignal): Promise { - console.log( - "[CodeIndexOrchestrator] Collection already has indexed data. Running incremental scan for new/changed files...", - ) - this.stateManager.setSystemState("Indexing", "Checking for new or modified files...") - const summary = await this.scanWorkspace(signal, "incremental") - if (!summary) return false - - // Do not mark an existing index complete when some updates failed. - if (summary.batchErrors.length > 0) { - const messages = [...new Set(summary.batchErrors.map((error) => error.message))] - throw new AggregateError( - summary.batchErrors, - `Incremental scan failed with ${summary.batchErrors.length} errors:\n${messages.join("\n")}`, - ) - } - - if (summary.found > 0) { - console.log( - `[CodeIndexOrchestrator] Incremental scan completed: ${summary.indexed} blocks indexed from new/changed files`, - ) - } else { - console.log("[CodeIndexOrchestrator] No new or changed files found") - } - return true + runIncrementalScan(signal: AbortSignal): Promise { + return this.scanWorkspace(signal, "incremental") } - /** Returns false on returned cancellation; scanner rejections propagate unchanged. */ - async runFullScan(signal: AbortSignal): Promise { - this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") - const summary = await this.scanWorkspace(signal, "full") - if (!summary) return false - this.validateFullScan(summary.indexed, summary.found, summary.batchErrors) - return true + runFullScan(signal: AbortSignal): Promise { + return this.scanWorkspace(signal, "full") } - private async scanWorkspace( - signal: AbortSignal, - kind: "full" | "incremental", - ): Promise<{ indexed: number; found: number; batchErrors: Error[] } | undefined> { - await this.vectorStore.markIndexingIncomplete() + private async scanWorkspace(signal: AbortSignal, kind: "full" | "incremental"): Promise { + const isLive = () => !signal.aborted && this.isCurrentAttempt() + if (!isLive()) return false + this.partial = false + this.stateManager.setSystemState( + "Indexing", + kind === "full" ? "Services ready. Starting workspace scan..." : "Checking for new or modified files...", + "scan", + ) let indexed = 0 let found = 0 const batchErrors: Error[] = [] const result = await this.scanner.scanDirectory( this.workspacePath, - (batchError: Error) => { - console.error( - `[CodeIndexOrchestrator] Error during ${kind === "full" ? "initial" : "incremental"} scan batch: ${batchError.message}`, - batchError, - ) - batchErrors.push(batchError) + (error) => { + if (isLive()) batchErrors.push(error) }, - (count: number) => { + (count) => { + if (!isLive()) return indexed += count - this.stateManager.reportBlockIndexingProgress(indexed, found) + this.stateManager.reportBlockIndexingProgress(indexed, found, "scan") }, - (count: number) => { + (count) => { + if (!isLive()) return found += count - this.stateManager.reportBlockIndexingProgress(indexed, found) + this.stateManager.reportBlockIndexingProgress(indexed, found, "scan") }, signal, ) - - // Cancellation precedes missing-result checks and validation, just as in the orchestrator. - if (signal.aborted) return undefined + if (!isLive()) return false if (!result) { throw new Error( kind === "full" @@ -85,25 +77,24 @@ export class CodeIndexScanExecutor { : "Incremental scan failed, is scanner initialized?", ) } - return { indexed, found, batchErrors } - } - - private validateFullScan(indexed: number, found: number, batchErrors: Error[]): void { - const firstError = batchErrors[0] - if (indexed === 0 && found > 0) { - throw new Error( - firstError - ? `Indexing failed: ${firstError.message}` - : t("embeddings:orchestrator.indexingFailedNoBlocks"), - ) - } - if (firstError && (found - indexed) / found > 0.1) { - throw new Error( - `Indexing partially failed: Only ${indexed} of ${found} blocks were indexed. ${firstError.message}`, - ) - } - if (firstError && indexed === 0) { - throw new Error(`Indexing failed completely: ${firstError.message}`) + const fatal = batchErrors.find((error) => error instanceof ParserFatalError) + if (fatal) throw fatal + if ( + kind === "full" && + indexed === 0 && + found > 0 && + batchErrors.length > 0 && + batchErrors.every(isEmbeddingConfigurationError) + ) { + throw batchErrors[0] } + this.partial = batchErrors.length > 0 || (result.incompleteFiles?.length ?? 0) > 0 || indexed < found + codeIndexLog.info("Scan summary", { + blocksIndexed: indexed, + blocksFound: found, + failureCount: batchErrors.length, + incompleteFiles: result.incompleteFiles, + }) + return true } } diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 6f0e0fe7e6..0619fd6bfa 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -5,6 +5,8 @@ export const MAX_BLOCK_CHARS = 1000 export const MIN_BLOCK_CHARS = 50 export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars +export const PARSE_TIMEOUT_MS = 10_000 +export const INDEX_STOP_LIMIT_MS = 2_000 /**Search */ export const DEFAULT_SEARCH_MIN_SCORE = CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE @@ -17,8 +19,11 @@ export const MAX_FILE_SIZE_BYTES = 1 * 1024 * 1024 // 1MB /**Directory Scanner */ export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000 export const BATCH_SEGMENT_THRESHOLD = 60 // Number of code segments to batch for embeddings/upserts -export const MAX_BATCH_RETRIES = 3 -export const INITIAL_RETRY_DELAY_MS = 500 +export const FILE_RETRY_ATTEMPTS = 3 +export const FILE_RETRY_BASE_DELAY_MS = 500 +export const PACER_INCREASE_AFTER = 20 +export const PACER_MAX_BACKOFF_MS = 300_000 +export const PACER_RESUME_SPACING_MS = 250 export const PARSING_CONCURRENCY = 10 export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting diff --git a/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts b/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts new file mode 100644 index 0000000000..1aaf0b5d45 --- /dev/null +++ b/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts @@ -0,0 +1,70 @@ +import "../../__tests__/mock-code-index-log" +import { createServer } from "node:http2" +import { BedrockEmbedder } from "../bedrock" +import { pacedTransportWriter, waitForTransport } from "./paced-transport-writer" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../../../../utils/networkProxy", () => ({ createProxyRoutingRequestHandler: () => undefined })) + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }) + vi.spyOn(Math, "random").mockReturnValue(0) +}) +afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() + vi.unstubAllEnvs() +}) + +it("paces four HTTP/2 throttles without exhaustion and then completes", async () => { + let requests = 0 + const server = createServer() + server.on("stream", (stream) => { + requests++ + if (requests <= 4) { + stream.respond({ + ":status": 429, + "content-type": "application/json", + "x-amzn-errortype": "ThrottlingException", + }) + stream.end(JSON.stringify({ message: "limited", __type: "ThrottlingException" })) + } else { + stream.respond({ ":status": 200, "content-type": "application/json" }) + stream.end(JSON.stringify({ embedding: [1, 0], inputTextTokenCount: 1 })) + } + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const address = server.address() + if (!address || typeof address === "string") throw new Error("Expected a TCP test listener") + vi.stubEnv("AWS_ACCESS_KEY_ID", "test-access-key") + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + vi.stubEnv("AWS_SESSION_TOKEN", "") + vi.stubEnv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", `http://127.0.0.1:${address.port}`) + const embedder = new BedrockEmbedder("us-east-1") + const controller = new AbortController() + const { replace, failures, cache, pacers } = pacedTransportWriter() + const pending = replace(embedder, controller.signal).catch((error: unknown) => error) + try { + for (const [index, delay] of [500, 1000, 2000, 4000].entries()) { + await waitForTransport(() => { + expect(requests).toBe(index + 1) + expect(pacers.get(embedder.rateLimitScope).state().pausedUntil).toBe(Date.now() + delay) + expect(pacers.waitingPaths()).toEqual(["test.ts"]) + }) + expect(failures.list()).toEqual([]) + vi.advanceTimersByTime(delay - 1) + expect(requests).toBe(index + 1) + vi.advanceTimersByTime(1) + } + await pending + expect(requests).toBe(5) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() + } finally { + controller.abort() + await pending + embedder["bedrockClient"].destroy() + await new Promise((resolve) => server.close(() => resolve())) + } +}) diff --git a/src/services/code-index/embedders/__tests__/bedrock.spec.ts b/src/services/code-index/embedders/__tests__/bedrock.spec.ts index f089fece39..8aa29e1d36 100644 --- a/src/services/code-index/embedders/__tests__/bedrock.spec.ts +++ b/src/services/code-index/embedders/__tests__/bedrock.spec.ts @@ -1,8 +1,9 @@ -import type { MockedFunction } from "vitest" +import { codeIndexLog } from "../../code-index-log" +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime" import { BedrockEmbedder } from "../bedrock" -import { MAX_ITEM_TOKENS, INITIAL_RETRY_DELAY_MS } from "../../constants" +import { MAX_ITEM_TOKENS } from "../../constants" import { createProxyRoutingRequestHandler } from "../../../../utils/networkProxy" import { clearAllMocks } from "../../../../test-utils/reset" @@ -23,7 +24,7 @@ vitest.mock("@aws-sdk/client-bedrock-runtime", () => { } }) vitest.mock("@aws-sdk/credential-providers", () => ({ - fromEnv: vitest.fn().mockReturnValue(Promise.resolve({})), + fromNodeProviderChain: vitest.fn().mockReturnValue(Promise.resolve({})), fromIni: vitest.fn().mockReturnValue(Promise.resolve({})), })) @@ -42,13 +43,13 @@ vitest.mock("@roo-code/telemetry", () => ({ // Mock i18n vitest.mock("../../../../i18n", () => ({ - t: (key: string, params?: Record) => { + t: (key: string, params?: Record) => { const translations: Record = { "embeddings:authenticationFailed": "Failed to create embeddings: Authentication failed. Please check your AWS credentials.", - "embeddings:failedWithStatus": `Failed to create embeddings after ${params?.attempts} attempts: HTTP ${params?.statusCode} - ${params?.errorMessage}`, - "embeddings:failedWithError": `Failed to create embeddings after ${params?.attempts} attempts: ${params?.errorMessage}`, - "embeddings:failedMaxAttempts": `Failed to create embeddings after ${params?.attempts} attempts`, + "embeddings:failedWithStatus": `Failed to create embeddings: HTTP ${params?.statusCode} - ${params?.errorMessage}`, + "embeddings:failedWithError": `Failed to create embeddings: ${params?.errorMessage}`, + "embeddings:failedMaxAttempts": `Failed to create embeddings`, "embeddings:textExceedsTokenLimit": `Text at index ${params?.index} exceeds maximum token limit (${params?.itemTokens} > ${params?.maxTokens}). Skipping.`, "embeddings:rateLimitRetry": `Rate limit hit, retrying in ${params?.delayMs}ms (attempt ${params?.attempt}/${params?.maxRetries})`, "embeddings:bedrock.invalidResponseFormat": "Invalid response format from Bedrock", @@ -66,13 +67,13 @@ vitest.mock("../../../../i18n", () => ({ // Mock console methods const consoleMocks = { - error: vitest.spyOn(console, "error").mockImplementation(function () {}), - warn: vitest.spyOn(console, "warn").mockImplementation(function () {}), + error: vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}), + warn: vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}), } describe("BedrockEmbedder", () => { let embedder: BedrockEmbedder - let mockSend: MockedFunction + let mockSend: ReturnType Promise>> beforeEach(() => { clearAllMocks() @@ -82,11 +83,10 @@ describe("BedrockEmbedder", () => { mockSend = vitest.fn() // Set up the mock implementation - const MockedBedrockRuntimeClient = BedrockRuntimeClient as any + const MockedBedrockRuntimeClient = vi.mocked(BedrockRuntimeClient) MockedBedrockRuntimeClient.mockImplementation(function () { - return { - send: mockSend, - } + // This client double supplies only the send method used by the embedder. + return { send: mockSend } as unknown as BedrockRuntimeClient }) embedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0") @@ -172,7 +172,7 @@ describe("BedrockEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) expect(mockSend).toHaveBeenCalled() - const command = mockSend.mock.calls[0][0] as any + const command = mockSend.mock.calls[0][0] as InvokeModelCommand expect(command.input.modelId).toBe(testModelId) const bodyStr = typeof command.input.body === "string" @@ -277,7 +277,7 @@ describe("BedrockEmbedder", () => { const result = await novaMultimodalEmbedder.createEmbeddings(testTexts) expect(mockSend).toHaveBeenCalled() - const command = mockSend.mock.calls[0][0] as any + const command = mockSend.mock.calls[0][0] as InvokeModelCommand expect(command.input.modelId).toBe("amazon.nova-2-multimodal-embeddings-v1:0") const bodyStr = typeof command.input.body === "string" @@ -343,7 +343,7 @@ describe("BedrockEmbedder", () => { expect(mockSend).toHaveBeenCalledTimes(2) // Verify the request format for both texts - const firstCommand = mockSend.mock.calls[0][0] as any + const firstCommand = mockSend.mock.calls[0][0] as InvokeModelCommand const firstBodyStr = typeof firstCommand.input.body === "string" ? firstCommand.input.body @@ -361,7 +361,7 @@ describe("BedrockEmbedder", () => { }, }) - const secondCommand = mockSend.mock.calls[1][0] as any + const secondCommand = mockSend.mock.calls[1][0] as InvokeModelCommand const secondBodyStr = typeof secondCommand.input.body === "string" ? secondCommand.input.body @@ -459,7 +459,7 @@ describe("BedrockEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) // Verify warning was logged - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) + expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) // Verify only normal texts were processed expect(mockSend).toHaveBeenCalledTimes(2) @@ -472,7 +472,7 @@ describe("BedrockEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(console.warn).toHaveBeenCalledTimes(2) + expect(codeIndexLog.warn).toHaveBeenCalledTimes(2) expect(mockSend).not.toHaveBeenCalled() expect(result).toEqual({ embeddings: [], @@ -493,37 +493,16 @@ describe("BedrockEmbedder", () => { vitest.useRealTimers() }) - it("should retry on throttling errors with exponential backoff", async () => { - const testTexts = ["Hello world"] - const throttlingError = new Error("Rate limit exceeded") - throttlingError.name = "ThrottlingException" - - mockSend - .mockRejectedValueOnce(throttlingError) - .mockRejectedValueOnce(throttlingError) - .mockResolvedValueOnce({ - body: new TextEncoder().encode( - JSON.stringify({ - embedding: [0.1, 0.2, 0.3], - inputTextTokenCount: 2, - }), - ), - }) - - const resultPromise = embedder.createEmbeddings(testTexts) - - // Fast-forward through the delays - await vitest.advanceTimersByTimeAsync(INITIAL_RETRY_DELAY_MS) // First retry delay - await vitest.advanceTimersByTimeAsync(INITIAL_RETRY_DELAY_MS * 2) // Second retry delay - - const result = await resultPromise - - expect(mockSend).toHaveBeenCalledTimes(3) - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("Rate limit hit, retrying in")) - expect(result).toEqual({ - embeddings: [[0.1, 0.2, 0.3]], - usage: { promptTokens: 2, totalTokens: 2 }, + it("returns rate limits without internal retries", async () => { + const cause = Object.assign(new Error("rate limited"), { status: 429, name: "ThrottlingException" }) + mockSend.mockRejectedValue(cause) + await expect(embedder.createEmbeddings(["test"])).rejects.toMatchObject({ + classification: "rate-limit", + statusCode: 429, + cause, }) + expect(mockSend).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) }) it("should not retry on non-throttling errors", async () => { @@ -534,11 +513,11 @@ describe("BedrockEmbedder", () => { mockSend.mockRejectedValue(authError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Unauthorized", + "Failed to create embeddings: Unauthorized", ) expect(mockSend).toHaveBeenCalledTimes(1) - expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) }) @@ -553,10 +532,10 @@ describe("BedrockEmbedder", () => { mockSend.mockRejectedValue(apiError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: API connection failed", + "Failed to create embeddings: API connection failed", ) - expect(console.error).toHaveBeenCalledWith( + expect(codeIndexLog.error).toHaveBeenCalledWith( expect.stringContaining("Bedrock embedder error"), expect.any(Error), ) @@ -594,7 +573,7 @@ describe("BedrockEmbedder", () => { mockSend.mockRejectedValueOnce(authError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Invalid credentials", + "Failed to create embeddings: Invalid credentials", ) // Test AccessDeniedException @@ -603,7 +582,7 @@ describe("BedrockEmbedder", () => { mockSend.mockRejectedValueOnce(accessError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Access denied", + "Failed to create embeddings: Access denied", ) // Test ResourceNotFoundException @@ -612,7 +591,7 @@ describe("BedrockEmbedder", () => { mockSend.mockRejectedValueOnce(notFoundError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Model not found", + "Failed to create embeddings: Model not found", ) }) }) diff --git a/src/services/code-index/embedders/__tests__/cancellation.spec.ts b/src/services/code-index/embedders/__tests__/cancellation.spec.ts new file mode 100644 index 0000000000..db907c768a --- /dev/null +++ b/src/services/code-index/embedders/__tests__/cancellation.spec.ts @@ -0,0 +1,258 @@ +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) +vi.mock("../../../../utils/networkProxy", () => ({ createProxyRoutingRequestHandler: vi.fn() })) +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { instance: { captureEvent: vi.fn() } }, +})) +import { clearAllMocks } from "../../../../test-utils/reset" +import { OpenAI } from "openai" +import { BedrockRuntimeClient } from "@aws-sdk/client-bedrock-runtime" +import { OpenAiEmbedder } from "../openai" +import { OpenAICompatibleEmbedder } from "../openai-compatible" +import { OpenRouterEmbedder } from "../openrouter" +import { GeminiEmbedder } from "../gemini" +import { MistralEmbedder } from "../mistral" +import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" +import { CodeIndexOllamaEmbedder } from "../ollama" +import { BedrockEmbedder } from "../bedrock" +import type { IEmbedder } from "../../interfaces/embedder" +import { waitForRetry } from "../cancellation" + +const { request } = vi.hoisted(() => ({ request: vi.fn() })) + +vi.mock("openai", async (importOriginal) => ({ + ...(await importOriginal()), + OpenAI: vi.fn().mockImplementation(function () { + return { embeddings: { create: request } } + }), +})) +vi.mock("../../../../api/providers/openai-native", () => ({ + OpenAiNativeHandler: class { + constructor(protected options: object) {} + }, +})) +vi.mock("@aws-sdk/client-bedrock-runtime", () => ({ + BedrockRuntimeClient: vi.fn().mockImplementation(function () { + return { send: request } + }), + InvokeModelCommand: vi.fn().mockImplementation(function (input: unknown) { + return { input } + }), +})) +vi.mock("@aws-sdk/credential-providers", () => ({ + fromIni: vi.fn(), + fromNodeProviderChain: vi.fn(), +})) +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../../../../shared/embeddingModels", () => ({ + getDefaultModelId: () => "test-model", + getModelQueryPrefix: () => "", +})) +vi.mock("../../constants", async (importOriginal) => ({ + ...(await importOriginal()), + MAX_BATCH_TOKENS: 4, +})) + +type Provider = { + name: string + create: () => IEmbedder + transport: "sdk" | "fetch" | "ollama" | "bedrock" +} + +const providers: Provider[] = [ + { name: "OpenAI", create: () => new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), transport: "sdk" }, + { + name: "OpenAI-compatible", + create: () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key"), + transport: "sdk", + }, + { name: "OpenRouter", create: () => new OpenRouterEmbedder("test-key"), transport: "sdk" }, + { name: "Gemini", create: () => new GeminiEmbedder("test-key"), transport: "sdk" }, + { name: "Mistral", create: () => new MistralEmbedder("test-key"), transport: "sdk" }, + { name: "Vercel", create: () => new VercelAiGatewayEmbedder("test-key"), transport: "sdk" }, + { name: "Ollama", create: () => new CodeIndexOllamaEmbedder({}), transport: "ollama" }, + { name: "Bedrock", create: () => new BedrockEmbedder("us-east-1"), transport: "bedrock" }, + { + name: "OpenAI-compatible direct fetch", + create: () => new OpenAICompatibleEmbedder("https://example.test/v1/embeddings", "test-key"), + transport: "fetch", + }, +] + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (reason: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function successfulResponse(transport: Provider["transport"]) { + const data = { data: [{ embedding: [0.25, 0.5] }], embeddings: [[0.25, 0.5]] } + if (transport === "fetch" || transport === "ollama") { + return { ok: true, json: async () => data } + } + if (transport === "bedrock") { + return { body: new TextEncoder().encode(JSON.stringify({ embedding: [0.25, 0.5] })) } + } + return data +} + +beforeEach(() => { + vi.useFakeTimers() + request.mockReset() + vi.stubGlobal("fetch", request) + vi.spyOn(console, "warn").mockImplementation(() => {}) + vi.spyOn(console, "error").mockImplementation(() => {}) +}) + +afterEach(() => { + vi.unstubAllGlobals() + clearAllMocks() + vi.useRealTimers() +}) + +describe.each(providers)("$name cancellation", ({ create, transport }) => { + it("forwards the caller signal to the transport", async () => { + const controller = new AbortController() + request.mockResolvedValue(successfulResponse(transport)) + await create().createEmbeddings(["first"], undefined, controller.signal) + const options: { signal: AbortSignal; abortSignal: AbortSignal } = request.mock.calls[0][1] + if (transport === "bedrock") { + expect(options.abortSignal).toBe(controller.signal) + } else if (transport === "ollama") { + expect(options.signal.aborted).toBe(false) + controller.abort() + expect(options.signal.aborted).toBe(true) + expect(options.signal.reason).toBe(controller.signal.reason) + } else { + expect(options.signal).toBe(controller.signal) + } + }) + + it("refuses an already-aborted call without issuing a request", async () => { + const controller = new AbortController() + controller.abort(Object.freeze({ stopped: true })) + await expect(create().createEmbeddings(["first"], undefined, controller.signal)).rejects.toBe( + controller.signal.reason, + ) + expect(request).not.toHaveBeenCalled() + }) + + it.each(["success", "failure", "cancellation-error"])( + "discards late %s and issues no queued batch or item", + async (outcome) => { + const controller = new AbortController() + const pending = deferred() + const started = deferred() + request.mockImplementation(() => { + started.resolve() + return pending.promise + }) + const result = create().createEmbeddings(["first123", "second12", "third123"], undefined, controller.signal) + await started.promise + controller.abort(Object.freeze({ stopped: true })) + const requestOptions: { signal: AbortSignal; abortSignal: AbortSignal } = request.mock.calls[0][1] + const forwardedSignal = transport === "bedrock" ? requestOptions.abortSignal : requestOptions.signal + expect(forwardedSignal.aborted).toBe(true) + expect(forwardedSignal.reason).toBe(controller.signal.reason) + if (outcome === "success") { + pending.resolve(successfulResponse(transport)) + } else { + pending.reject( + outcome === "failure" ? new Error("Late failure") : new DOMException("Late abort", "AbortError"), + ) + } + await expect(result).rejects.toBe(controller.signal.reason) + expect(request).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }, + ) + it.each([ + [429, "rate-limit"], + [503, "server"], + [408, "permanent"], + [409, "permanent"], + [401, "permanent"], + ] as const)("classifies HTTP %d without retrying", async (status, classification) => { + const failure = Object.assign(new Error("request failed"), { + status, + headers: new Headers({ "retry-after": "2" }), + $metadata: { httpStatusCode: status }, + }) + if (transport === "fetch" || transport === "ollama") { + request.mockResolvedValue(new Response("request failed", { status, headers: failure.headers })) + } else { + request.mockRejectedValue(failure) + } + const embedder = create() + await expect(embedder.createEmbeddings(["first"])).rejects.toMatchObject({ + classification, + statusCode: status, + retryAfterMs: 2000, + }) + expect(request).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + if (transport === "sdk") { + expect(OpenAI).toHaveBeenLastCalledWith(expect.objectContaining({ maxRetries: 0 })) + } else if (transport === "bedrock") { + expect(BedrockRuntimeClient).toHaveBeenLastCalledWith(expect.objectContaining({ maxAttempts: 1 })) + } + }) + + it("classifies transport connection failures without retrying", async () => { + request.mockRejectedValue(Object.assign(new Error("connection failed"), { code: "ECONNRESET" })) + await expect(create().createEmbeddings(["first"])).rejects.toMatchObject({ classification: "connection" }) + expect(request).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }) +}) + +describe("cancellable wait", () => { + it("refuses a previously aborted signal", () => { + const controller = new AbortController() + controller.abort() + expect(() => waitForRetry(100, controller.signal)).toThrow(controller.signal.reason) + expect(vi.getTimerCount()).toBe(0) + }) + + it("removes its abort listener when the wait completes", async () => { + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, "removeEventListener") + const result = waitForRetry(100, controller.signal) + await vi.advanceTimersByTimeAsync(100) + await result + expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)) + expect(vi.getTimerCount()).toBe(0) + }) +}) + +it("Ollama keeps its timeout when a caller signal is supplied and cleans up after rejection", async () => { + const controller = new AbortController() + request.mockImplementation( + (_url: unknown, options: { signal: AbortSignal }) => + new Promise((_resolve, reject) => { + options.signal.addEventListener("abort", () => reject(options.signal.reason), { once: true }) + }), + ) + const result = new CodeIndexOllamaEmbedder({}).createEmbeddings(["first"], undefined, controller.signal) + const assertion = expect(result).rejects.toThrow("embeddings:validation.connectionFailed") + await vi.advanceTimersByTimeAsync(60_000) + await assertion + expect(controller.signal.aborted).toBe(false) + const options: { signal: AbortSignal } = request.mock.calls[0][1] + expect(options.signal.aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) +}) + +it("cancels a wait with the unchanged reason and removes its timer and listener", async () => { + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, "removeEventListener") + const reason = Object.freeze({ stopped: true }) + const pending = waitForRetry(60_000, controller.signal) + controller.abort(reason) + await expect(pending).rejects.toBe(reason) + expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts b/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts index 7128695e8f..b568436ce3 100644 --- a/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts +++ b/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts @@ -29,6 +29,15 @@ describe("EmbedderValidationManager", () => { }, ) + it("rethrows rate limits unchanged without permanent-error telemetry", async () => { + const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) + const error = Object.assign(new Error("limited"), { status: 429, headers: { "retry-after": "120" } }) + vi.mocked(embedder.validateConfiguration).mockRejectedValue(error) + + await expect(manager.validateEmbedder(embedder)).rejects.toBe(error) + expect(TelemetryService.instance.captureEvent).not.toHaveBeenCalled() + }) + it("preserves an exception's message and reports its stack", async () => { const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) const error = new Error("Connection failed") diff --git a/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts index 49086d675b..76d0411c14 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts @@ -1,217 +1,55 @@ -import type { MockedClass, MockedFunction } from "vitest" -import { OpenAI } from "openai" - -import { OpenAICompatibleEmbedder } from "../openai-compatible" -import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" - -// Mock the OpenAI SDK -vi.mock("openai") - -// Mock TelemetryService +vi.mock("../../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), error: vi.fn(), warn: vi.fn() } })) vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), - }, - }, -})) - -// Mock i18n -vi.mock("../../../../i18n", () => ({ - t: (key: string, params?: Record) => { - const translations: Record = { - "embeddings:rateLimitRetry": `Rate limit hit, retrying in ${params?.delayMs}ms (attempt ${params?.attempt}/${params?.maxRetries})`, - "embeddings:failedMaxAttempts": `Failed to create embeddings after ${params?.attempts} attempts`, - "embeddings:failedWithStatus": `Failed to create embeddings after ${params?.attempts} attempts: HTTP ${params?.statusCode} - ${params?.errorMessage}`, - "embeddings:failedWithError": `Failed to create embeddings after ${params?.attempts} attempts: ${params?.errorMessage}`, - } - return translations[key] || key - }, + TelemetryService: { instance: { captureEvent: vi.fn() } }, })) +import { OpenAI } from "openai" +import { OpenAICompatibleEmbedder } from "../openai-compatible" +import { OpenRouterEmbedder } from "../openrouter" -const MockedOpenAI = OpenAI as MockedClass - -describe("OpenAICompatibleEmbedder - Global Rate Limiting", () => { - let mockOpenAIInstance: any - let mockEmbeddingsCreate: MockedFunction - - const testBaseUrl = "https://api.openai.com/v1" - const testApiKey = "test-api-key" - const testModelId = "text-embedding-3-small" - - beforeEach(() => { - clearAllMocks() - vi.useFakeTimers() - vi.spyOn(console, "warn").mockImplementation(function () {}) - vi.spyOn(console, "error").mockImplementation(function () {}) - - // Setup mock OpenAI instance - mockEmbeddingsCreate = vi.fn() - mockOpenAIInstance = { - embeddings: { - create: mockEmbeddingsCreate, - }, - } - - MockedOpenAI.mockImplementation(function () { - return mockOpenAIInstance - }) - - // Reset global rate limit state - const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - ;(embedder as any).constructor.globalRateLimitState = { - isRateLimited: false, - rateLimitResetTime: 0, - consecutiveRateLimitErrors: 0, - lastRateLimitError: 0, - mutex: (embedder as any).constructor.globalRateLimitState.mutex, - } - }) - - afterEach(() => { - vi.useRealTimers() - restoreGlobals() - }) - - it("should apply global rate limiting across multiple batch requests", async () => { - const embedder1 = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - const embedder2 = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - - // First batch hits rate limit - const rateLimitError = new Error("Rate limit exceeded") as any - rateLimitError.status = 429 - - mockEmbeddingsCreate - .mockRejectedValueOnce(rateLimitError) // First attempt fails - .mockResolvedValue({ - data: [{ embedding: "base64encodeddata" }], - usage: { prompt_tokens: 10, total_tokens: 15 }, - }) - - // Start first batch request - const batch1Promise = embedder1.createEmbeddings(["test1"]) - - // Advance time slightly to let the first request fail and set global rate limit - await vi.advanceTimersByTimeAsync(100) - - // Start second batch request while global rate limit is active - const batch2Promise = embedder2.createEmbeddings(["test2"]) - - // Check that global rate limit was set - const state = (embedder1 as any).constructor.globalRateLimitState - expect(state.isRateLimited).toBe(true) - expect(state.consecutiveRateLimitErrors).toBe(1) - - // Advance time to complete rate limit delay (5 seconds base delay) - await vi.advanceTimersByTimeAsync(5000) - - // Both requests should complete - const [result1, result2] = await Promise.all([batch1Promise, batch2Promise]) - - expect(result1.embeddings).toHaveLength(1) - expect(result2.embeddings).toHaveLength(1) - - // The second embedder should have waited for the global rate limit - // No logging expected - we've removed it to prevent log flooding - }) - - it("should track consecutive rate limit errors", async () => { - const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - const state = (embedder as any).constructor.globalRateLimitState - - const rateLimitError = new Error("Rate limit exceeded") as any - rateLimitError.status = 429 - - // Test that consecutive errors increment when they happen quickly - // Mock multiple rate limit errors in a single request - mockEmbeddingsCreate - .mockRejectedValueOnce(rateLimitError) // First attempt - .mockRejectedValueOnce(rateLimitError) // Retry 1 - .mockResolvedValueOnce({ - data: [{ embedding: "base64encodeddata" }], - usage: { prompt_tokens: 10, total_tokens: 15 }, - }) - - const promise1 = embedder.createEmbeddings(["test1"]) - - // Wait for first attempt to fail - await vi.advanceTimersByTimeAsync(100) - expect(state.consecutiveRateLimitErrors).toBe(1) +import { ProviderPacerRegistry } from "../../provider-pacer" - // Wait for first retry (500ms) to also fail - await vi.advanceTimersByTimeAsync(500) +vi.mock("openai") +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) - // The state should show 2 consecutive errors now - // Note: The count might be 1 if the global rate limit kicked in before the second attempt - expect(state.consecutiveRateLimitErrors).toBeGreaterThanOrEqual(1) +const factories = [ + ["OpenAI-compatible", () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key")], + ["OpenRouter", () => new OpenRouterEmbedder("test-key")], +] as const - // Wait for the global rate limit and successful retry - await vi.advanceTimersByTimeAsync(20000) - await promise1 +beforeEach(() => { + vi.useFakeTimers() +}) +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) - // Verify the delay increases with consecutive errors - // Make another request immediately that also hits rate limit - mockEmbeddingsCreate.mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({ - data: [{ embedding: "base64encodeddata" }], - usage: { prompt_tokens: 10, total_tokens: 15 }, +describe.each(factories)("%s rate-limit boundary", (_name, createEmbedder) => { + it("shares a workspace pause across instances without adding wrapper retries", async () => { + const request = vi.fn() + // This client double supplies only embeddings.create, the embedder transport. + vi.mocked(OpenAI).mockImplementation(function () { + return { embeddings: { create: request } } as unknown as OpenAI }) - - // Store the current consecutive count before the next request - const previousCount = state.consecutiveRateLimitErrors - - const promise2 = embedder.createEmbeddings(["test2"]) - await vi.advanceTimersByTimeAsync(100) - - // Should have incremented from the previous count - expect(state.consecutiveRateLimitErrors).toBeGreaterThan(previousCount) - - // Complete the second request - await vi.advanceTimersByTimeAsync(20000) - await promise2 - }) - - it("should reset consecutive error count after time passes", async () => { - const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - const state = (embedder as any).constructor.globalRateLimitState - - // Manually set state to simulate previous errors - state.consecutiveRateLimitErrors = 3 - state.lastRateLimitError = Date.now() - 70000 // 70 seconds ago - - const rateLimitError = new Error("Rate limit exceeded") as any - rateLimitError.status = 429 - - mockEmbeddingsCreate.mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({ - data: [{ embedding: "base64encodeddata" }], - usage: { prompt_tokens: 10, total_tokens: 15 }, + const cause = Object.assign(new Error("rate limited"), { + status: 429, + headers: new Headers({ "retry-after": "60" }), }) - - // Trigger the updateGlobalRateLimitState method - await (embedder as any).updateGlobalRateLimitState(rateLimitError) - - // Should reset to 1 since more than 60 seconds passed - expect(state.consecutiveRateLimitErrors).toBe(1) - }) - - it("should not exceed maximum delay of 5 minutes", async () => { - const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - const state = (embedder as any).constructor.globalRateLimitState - - // Set state to simulate many consecutive errors - state.consecutiveRateLimitErrors = 10 // This would normally result in a very long delay - - const rateLimitError = new Error("Rate limit exceeded") as any - rateLimitError.status = 429 - - // Trigger the updateGlobalRateLimitState method - await (embedder as any).updateGlobalRateLimitState(rateLimitError) - - // Calculate the expected delay - const now = Date.now() - const delay = state.rateLimitResetTime - now - - // Should be capped at 5 minutes (300000ms) - expect(delay).toBeLessThanOrEqual(300000) - expect(delay).toBeGreaterThan(0) + request.mockRejectedValueOnce(cause).mockResolvedValue({ data: [{ embedding: [0.25] }] }) + const first = createEmbedder() + const second = createEmbedder() + const registry = new ProviderPacerRegistry() + const pacer = registry.get(first.rateLimitScope) + const firstResult = pacer.run("first.ts", undefined, () => first.createEmbeddings(["first"])) + await vi.advanceTimersByTimeAsync(0) + const secondResult = pacer.run("second.ts", undefined, () => second.createEmbeddings(["second"])) + await vi.advanceTimersByTimeAsync(59_999) + expect(request).toHaveBeenCalledOnce() + expect(first.rateLimitScope).toBe(second.rateLimitScope) + await vi.advanceTimersByTimeAsync(1) + await expect(firstResult).resolves.toMatchObject({ embeddings: [[0.25]] }) + await vi.advanceTimersByTimeAsync(375) + await expect(secondResult).resolves.toMatchObject({ embeddings: [[0.25]] }) + expect(request).toHaveBeenCalledTimes(3) }) }) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index 859f2ceae1..afe5621c5d 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -1,3 +1,5 @@ +import { codeIndexLog } from "../../code-index-log" +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import type { MockedClass, MockedFunction } from "vitest" import { OpenAI } from "openai" import { OpenAICompatibleEmbedder } from "../openai-compatible" @@ -21,13 +23,13 @@ vitest.mock("@roo-code/telemetry", () => ({ // Mock i18n vitest.mock("../../../../i18n", () => ({ - t: (key: string, params?: Record) => { + t: (key: string, params?: Record) => { const translations: Record = { "embeddings:authenticationFailed": "Failed to create embeddings: Authentication failed. Please check your API key.", - "embeddings:failedWithStatus": `Failed to create embeddings after ${params?.attempts} attempts: HTTP ${params?.statusCode} - ${params?.errorMessage}`, - "embeddings:failedWithError": `Failed to create embeddings after ${params?.attempts} attempts: ${params?.errorMessage}`, - "embeddings:failedMaxAttempts": `Failed to create embeddings after ${params?.attempts} attempts`, + "embeddings:failedWithStatus": `Failed to create embeddings: HTTP ${params?.statusCode} - ${params?.errorMessage}`, + "embeddings:failedWithError": `Failed to create embeddings: ${params?.errorMessage}`, + "embeddings:failedMaxAttempts": `Failed to create embeddings`, "embeddings:textExceedsTokenLimit": `Text at index ${params?.index} exceeds maximum token limit (${params?.itemTokens} > ${params?.maxTokens}). Skipping.`, "embeddings:rateLimitRetry": `Rate limit hit, retrying in ${params?.delayMs}ms (attempt ${params?.attempt}/${params?.maxRetries})`, "embeddings:unknownError": "Unknown error", @@ -55,8 +57,8 @@ const MockedOpenAI = OpenAI as MockedClass describe("OpenAICompatibleEmbedder", () => { let embedder: OpenAICompatibleEmbedder - let mockOpenAIInstance: any - let mockEmbeddingsCreate: MockedFunction + let mockOpenAIInstance: { embeddings: { create: typeof mockEmbeddingsCreate } } + let mockEmbeddingsCreate: ReturnType Promise>> const testBaseUrl = "https://api.example.com/v1" const testApiKey = "test-api-key" @@ -64,8 +66,8 @@ describe("OpenAICompatibleEmbedder", () => { beforeEach(() => { clearAllMocks() - vitest.spyOn(console, "warn").mockImplementation(function () {}) - vitest.spyOn(console, "error").mockImplementation(function () {}) + vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}) + vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}) // Setup mock OpenAI instance mockEmbeddingsCreate = vitest.fn() @@ -76,18 +78,9 @@ describe("OpenAICompatibleEmbedder", () => { } MockedOpenAI.mockImplementation(function () { - return mockOpenAIInstance + // The embedder only uses embeddings.create on this client double. + return mockOpenAIInstance as unknown as OpenAI }) - - // Reset global rate limit state to prevent interference between tests - const tempEmbedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) - ;(tempEmbedder as any).constructor.globalRateLimitState = { - isRateLimited: false, - rateLimitResetTime: 0, - consecutiveRateLimitErrors: 0, - lastRateLimitError: 0, - mutex: (tempEmbedder as any).constructor.globalRateLimitState.mutex, - } }) afterEach(() => { @@ -101,6 +94,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(MockedOpenAI).toHaveBeenCalledWith({ baseURL: testBaseUrl, apiKey: testApiKey, + maxRetries: 0, }) expect(embedder).toBeDefined() }) @@ -111,6 +105,7 @@ describe("OpenAICompatibleEmbedder", () => { expect(MockedOpenAI).toHaveBeenCalledWith({ baseURL: testBaseUrl, apiKey: testApiKey, + maxRetries: 0, }) expect(embedder).toBeDefined() }) @@ -376,7 +371,7 @@ describe("OpenAICompatibleEmbedder", () => { await embedder.createEmbeddings(testTexts) // Should warn about oversized text - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) + expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) // Should only process normal texts (1 call for 2 normal texts batched together) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) @@ -411,50 +406,22 @@ describe("OpenAICompatibleEmbedder", () => { vitest.useRealTimers() }) - it("should retry on rate limit errors with exponential backoff", async () => { - const testTexts = ["Hello world"] - const rateLimitError = { status: 429, message: "Rate limit exceeded" } - - // Create base64 encoded embedding for successful response - const testEmbedding = new Float32Array([0.25, 0.5, 0.75]) - const base64String = Buffer.from(testEmbedding.buffer).toString("base64") - - mockEmbeddingsCreate - .mockRejectedValueOnce(rateLimitError) - .mockRejectedValueOnce(rateLimitError) - .mockResolvedValueOnce({ - data: [{ embedding: base64String }], - usage: { prompt_tokens: 10, total_tokens: 15 }, - }) - - const resultPromise = embedder.createEmbeddings(testTexts) - - // First attempt fails immediately, triggering global rate limit (5s) - await vitest.advanceTimersByTimeAsync(100) - - // Wait for global rate limit delay - await vitest.advanceTimersByTimeAsync(5000) - - // Second attempt also fails, increasing delay - await vitest.advanceTimersByTimeAsync(100) - - // Wait for increased global rate limit delay (10s) - await vitest.advanceTimersByTimeAsync(10000) - - const result = await resultPromise - - expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(3) - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("Rate limit hit, retrying in")) - expect(result).toEqual({ - embeddings: [[0.25, 0.5, 0.75]], - usage: { promptTokens: 10, totalTokens: 15 }, + it("returns rate limits without internal retries", async () => { + const cause = Object.assign(new Error("rate limited"), { status: 429, name: "ThrottlingException" }) + mockEmbeddingsCreate.mockRejectedValue(cause) + await expect(embedder.createEmbeddings(["test"])).rejects.toMatchObject({ + classification: "rate-limit", + statusCode: 429, + cause, }) + expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) }) it("should not retry on non-rate-limit errors", async () => { const testTexts = ["Hello world"] const authError = new Error("Unauthorized") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) @@ -463,18 +430,18 @@ describe("OpenAICompatibleEmbedder", () => { ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) - expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) it("should throw error immediately on non-retryable errors", async () => { const testTexts = ["Hello world"] const serverError = new Error("Internal server error") - ;(serverError as any).status = 500 + Object.assign(serverError, { status: 500 }) mockEmbeddingsCreate.mockRejectedValue(serverError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: HTTP 500 - Internal server error", + "Failed to create embeddings: HTTP 500 - Internal server error", ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) @@ -492,10 +459,10 @@ describe("OpenAICompatibleEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(apiError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: API connection failed", + "Failed to create embeddings: API connection failed", ) - expect(console.error).toHaveBeenCalledWith( + expect(codeIndexLog.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI Compatible embedder error"), apiError, ) @@ -508,10 +475,10 @@ describe("OpenAICompatibleEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(batchError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Batch processing failed", + "Failed to create embeddings: Batch processing failed", ) - expect(console.error).toHaveBeenCalledWith( + expect(codeIndexLog.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI Compatible embedder error"), expect.any(Error), ) @@ -544,7 +511,7 @@ describe("OpenAICompatibleEmbedder", () => { it("should provide specific authentication error message", async () => { const testTexts = ["Hello world"] const authError = new Error("Invalid API key") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) @@ -556,12 +523,12 @@ describe("OpenAICompatibleEmbedder", () => { it("should provide detailed error message for HTTP errors", async () => { const testTexts = ["Hello world"] const httpError = new Error("Bad request") - ;(httpError as any).status = 400 + Object.assign(httpError, { status: 400 }) mockEmbeddingsCreate.mockRejectedValue(httpError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: HTTP 400 - Bad request", + "Failed to create embeddings: HTTP 400 - Bad request", ) }) @@ -572,7 +539,7 @@ describe("OpenAICompatibleEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(networkError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Network timeout", + "Failed to create embeddings: Network timeout", ) }) @@ -583,7 +550,7 @@ describe("OpenAICompatibleEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(weirdError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Custom error object", + "Failed to create embeddings: Custom error object", ) }) @@ -594,88 +561,28 @@ describe("OpenAICompatibleEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(unknownError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Unknown error", + "Failed to create embeddings: Unknown error", ) }) }) - /** - * Test to confirm OpenAI package bug with base64 encoding - * This test verifies that when we request encoding_format: "base64", - * the OpenAI package returns unparsed base64 strings as expected. - * This is the behavior we rely on in our workaround. - */ - describe("OpenAI package base64 behavior verification", () => { - it("should return unparsed base64 when encoding_format is base64", async () => { - const testTexts = ["Hello world"] - - // Create a real OpenAI instance to test the actual package behavior - const realOpenAI = new ((await vi.importActual("openai")) as any).OpenAI({ - baseURL: testBaseUrl, - apiKey: testApiKey, - }) - - // Create test embedding data as base64 using values that can be exactly represented in Float32 - const testEmbedding = new Float32Array([0.25, 0.5, 0.75, 1.0]) - const buffer = Buffer.from(testEmbedding.buffer) - const base64String = buffer.toString("base64") - - // Mock the raw API response that would come from OpenAI - const mockApiResponse = { - data: [ - { - object: "embedding", - embedding: base64String, // Raw base64 string from API - index: 0, - }, - ], - model: "text-embedding-3-small", - object: "list", - usage: { - prompt_tokens: 2, - total_tokens: 2, - }, - } - - // Mock the methodRequest method which is called by post() - const mockMethodRequest = vi.fn() - const mockAPIPromise = { - then: vi.fn().mockImplementation((callback) => { - return Promise.resolve(callback(mockApiResponse)) + it("keeps base64 strings unparsed at the SDK boundary", async () => { + const { OpenAI: ActualOpenAI } = await vi.importActual("openai") + const encodedEmbedding = Buffer.from(new Float32Array([0.25, 0.5]).buffer).toString("base64") + const client = new ActualOpenAI({ + apiKey: testApiKey, + maxRetries: 0, + fetch: async () => + new Response(JSON.stringify({ data: [{ embedding: encodedEmbedding }] }), { + headers: { "content-type": "application/json" }, }), - catch: vi.fn(), - finally: vi.fn(), - } - mockMethodRequest.mockReturnValue(mockAPIPromise) - - // Replace the methodRequest method on the client - ;(realOpenAI as any).post = vi.fn().mockImplementation((path, opts) => { - return mockMethodRequest("post", path, opts) - }) - - // Call the embeddings.create method with base64 encoding - const response = await realOpenAI.embeddings.create({ - input: testTexts, - model: "text-embedding-3-small", - encoding_format: "base64", - }) - - // Verify that the response contains the raw base64 string - // This confirms the OpenAI package doesn't parse base64 when explicitly requested - expect(response.data[0].embedding).toBe(base64String) - expect(typeof response.data[0].embedding).toBe("string") - - // Verify we can manually convert it back to the original float array - const returnedBuffer = Buffer.from(response.data[0].embedding as string, "base64") - const returnedFloat32Array = new Float32Array( - returnedBuffer.buffer, - returnedBuffer.byteOffset, - returnedBuffer.byteLength / 4, - ) - const returnedArray = Array.from(returnedFloat32Array) - - expect(returnedArray).toEqual([0.25, 0.5, 0.75, 1.0]) }) + const response = await client.embeddings.create({ + input: ["test"], + model: testModelId, + encoding_format: "base64", + }) + expect(response.data[0].embedding).toBe(encodedEmbedding) }) /** @@ -687,12 +594,8 @@ describe("OpenAICompatibleEmbedder", () => { const baseUrl = "https://api.openai.com/v1" // Helper to create mock fetch response - const createMockResponse = (data: any, status = 200, ok = true) => ({ - ok, - status, - json: vitest.fn().mockResolvedValue(data), - text: vitest.fn().mockResolvedValue(status === 200 ? "" : "Error message"), - }) + const createMockResponse = (data: unknown, status = 200, _ok = true) => + new Response(status === 200 ? JSON.stringify(data) : "Error message", { status }) // Helper to create base64 embedding const createBase64Embedding = (values: number[]) => { @@ -723,7 +626,7 @@ describe("OpenAICompatibleEmbedder", () => { ["http://localhost:8080", false], ])("should detect URL type correctly: %s -> %s", (url, expected) => { const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId) - const isFullUrl = (embedder as any).isFullEndpointUrl(url) + const isFullUrl = embedder["isFullEndpointUrl"](url) expect(isFullUrl).toBe(expected) }) @@ -738,7 +641,7 @@ describe("OpenAICompatibleEmbedder", () => { testUrls.forEach((url) => { const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId) - const isFullUrl = (embedder as any).isFullEndpointUrl(url) + const isFullUrl = embedder["isFullEndpointUrl"](url) expect(isFullUrl).toBe(false) }) }) @@ -753,7 +656,7 @@ describe("OpenAICompatibleEmbedder", () => { testUrls.forEach((url) => { const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId) - const isFullUrl = (embedder as any).isFullEndpointUrl(url) + const isFullUrl = embedder["isFullEndpointUrl"](url) expect(isFullUrl).toBe(false) }) }) @@ -769,7 +672,7 @@ describe("OpenAICompatibleEmbedder", () => { endpointUrls.forEach((url) => { const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId) - const isFullUrl = (embedder as any).isFullEndpointUrl(url) + const isFullUrl = embedder["isFullEndpointUrl"](url) expect(isFullUrl).toBe(true) }) }) @@ -786,7 +689,7 @@ describe("OpenAICompatibleEmbedder", () => { data: [{ embedding: base64String }], usage: { prompt_tokens: 10, total_tokens: 15 }, }) - ;(global.fetch as MockedFunction).mockResolvedValue(mockFetchResponse as any) + ;(global.fetch as MockedFunction).mockResolvedValue(mockFetchResponse) const azureResult = await azureEmbedder.createEmbeddings(testTexts) expect(global.fetch).toHaveBeenCalledWith( @@ -818,51 +721,23 @@ describe("OpenAICompatibleEmbedder", () => { it.each([ [401, "Authentication failed. Please check your API key."], - [500, "Failed to create embeddings after 3 attempts"], + [500, "Failed to create embeddings"], ])("should handle HTTP errors: %d", async (status, expectedMessage) => { const embedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId) const mockResponse = createMockResponse({}, status, false) - ;(global.fetch as MockedFunction).mockResolvedValue(mockResponse as any) + ;(global.fetch as MockedFunction).mockResolvedValue(mockResponse) await expect(embedder.createEmbeddings(["test"])).rejects.toThrow(expectedMessage) }) - it("should handle rate limiting with retries", async () => { - vitest.useFakeTimers() + it("returns direct-fetch rate limits without retrying", async () => { const embedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId) - const base64String = createBase64Embedding([0.1, 0.2, 0.3]) - - ;(global.fetch as MockedFunction) - .mockResolvedValueOnce(createMockResponse({}, 429, false) as any) - .mockResolvedValueOnce(createMockResponse({}, 429, false) as any) - .mockResolvedValueOnce( - createMockResponse({ - data: [{ embedding: base64String }], - usage: { prompt_tokens: 10, total_tokens: 15 }, - }) as any, - ) - - const resultPromise = embedder.createEmbeddings(["test"]) - - // First attempt fails, triggering global rate limit - await vitest.advanceTimersByTimeAsync(100) - - // Wait for global rate limit (5s) - await vitest.advanceTimersByTimeAsync(5000) - - // Second attempt also fails - await vitest.advanceTimersByTimeAsync(100) - - // Wait for increased global rate limit (10s) - await vitest.advanceTimersByTimeAsync(10000) - - const result = await resultPromise - - expect(global.fetch).toHaveBeenCalledTimes(3) - // Check that rate limit warnings were logged - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) - expectEmbeddingValues(result.embeddings[0], [0.1, 0.2, 0.3]) - vitest.useRealTimers() + vi.mocked(global.fetch).mockResolvedValue(new Response("rate limited", { status: 429 })) + await expect(embedder.createEmbeddings(["test"])).rejects.toMatchObject({ + classification: "rate-limit", + statusCode: 429, + }) + expect(global.fetch).toHaveBeenCalledTimes(1) }) it("should handle multiple embeddings and network errors", async () => { @@ -875,7 +750,7 @@ describe("OpenAICompatibleEmbedder", () => { data: [{ embedding: base64_1 }, { embedding: base64_2 }], usage: { prompt_tokens: 20, total_tokens: 30 }, }) - ;(global.fetch as MockedFunction).mockResolvedValue(mockResponse as any) + ;(global.fetch as MockedFunction).mockResolvedValue(mockResponse) const result = await embedder.createEmbeddings(["test1", "test2"]) expect(result.embeddings).toHaveLength(2) @@ -885,9 +760,7 @@ describe("OpenAICompatibleEmbedder", () => { // Test network error const networkError = new Error("Network failed") ;(global.fetch as MockedFunction).mockRejectedValue(networkError) - await expect(embedder.createEmbeddings(["test"])).rejects.toThrow( - "Failed to create embeddings after 3 attempts", - ) + await expect(embedder.createEmbeddings(["test"])).rejects.toThrow("Failed to create embeddings") }) }) }) @@ -946,9 +819,10 @@ describe("OpenAICompatibleEmbedder", () => { data: [{ embedding: [0.1, 0.2] }], usage: { prompt_tokens: 10, total_tokens: 15 }, }) + // Only embeddings.create is used; the SDK resource's private client is not needed. embedder["embeddingsClient"].embeddings = { create: mockCreate, - } as any + } as unknown as OpenAI.Embeddings await embedder.createEmbeddings(["test"]) @@ -1000,7 +874,7 @@ describe("OpenAICompatibleEmbedder", () => { usage: { prompt_tokens: 2, total_tokens: 2 }, }), text: async () => "", - } as any) + } as Response) const result = await embedder.validateConfiguration() @@ -1021,7 +895,7 @@ describe("OpenAICompatibleEmbedder", () => { embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) const authError = new Error("Invalid API key") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) const result = await embedder.validateConfiguration() @@ -1051,7 +925,7 @@ describe("OpenAICompatibleEmbedder", () => { status: 404, json: async () => ({ error: "Not found" }), text: async () => "Not found", - } as any) + } as Response) const result = await embedder.validateConfiguration() @@ -1063,20 +937,17 @@ describe("OpenAICompatibleEmbedder", () => { embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) const rateLimitError = new Error("Rate limit exceeded") - ;(rateLimitError as any).status = 429 + Object.assign(rateLimitError, { status: 429 }) mockEmbeddingsCreate.mockRejectedValue(rateLimitError) - const result = await embedder.validateConfiguration() - - expect(result.valid).toBe(false) - expect(result.error).toBe("embeddings:validation.serviceUnavailable") + await expect(embedder.validateConfiguration()).rejects.toBe(rateLimitError) }) it("should fail validation with generic error", async () => { embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId) const genericError = new Error("Unknown error") - ;(genericError as any).status = 500 + Object.assign(genericError, { status: 500 }) mockEmbeddingsCreate.mockRejectedValue(genericError) const result = await embedder.validateConfiguration() diff --git a/src/services/code-index/embedders/__tests__/openai.spec.ts b/src/services/code-index/embedders/__tests__/openai.spec.ts index 4ad8239b53..3fa2513175 100644 --- a/src/services/code-index/embedders/__tests__/openai.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai.spec.ts @@ -1,8 +1,10 @@ -import type { MockedClass, MockedFunction } from "vitest" +import { codeIndexLog } from "../../code-index-log" +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) +import type { MockedClass } from "vitest" import { OpenAI } from "openai" import { OpenAiEmbedder } from "../openai" -import { MAX_ITEM_TOKENS, INITIAL_RETRY_DELAY_MS } from "../../constants" +import { MAX_ITEM_TOKENS } from "../../constants" import { clearAllMocks } from "../../../../test-utils/reset" // Mock the OpenAI SDK @@ -19,13 +21,13 @@ vitest.mock("@roo-code/telemetry", () => ({ // Mock i18n vitest.mock("../../../../i18n", () => ({ - t: (key: string, params?: Record) => { + t: (key: string, params?: Record) => { const translations: Record = { "embeddings:authenticationFailed": "Failed to create embeddings: Authentication failed. Please check your OpenAI API key.", - "embeddings:failedWithStatus": `Failed to create embeddings after ${params?.attempts} attempts: HTTP ${params?.statusCode} - ${params?.errorMessage}`, - "embeddings:failedWithError": `Failed to create embeddings after ${params?.attempts} attempts: ${params?.errorMessage}`, - "embeddings:failedMaxAttempts": `Failed to create embeddings after ${params?.attempts} attempts`, + "embeddings:failedWithStatus": `Failed to create embeddings: HTTP ${params?.statusCode} - ${params?.errorMessage}`, + "embeddings:failedWithError": `Failed to create embeddings: ${params?.errorMessage}`, + "embeddings:failedMaxAttempts": `Failed to create embeddings`, "embeddings:textExceedsTokenLimit": `Text at index ${params?.index} exceeds maximum token limit (${params?.itemTokens} > ${params?.maxTokens}). Skipping.`, "embeddings:rateLimitRetry": `Rate limit hit, retrying in ${params?.delayMs}ms (attempt ${params?.attempt}/${params?.maxRetries})`, } @@ -35,13 +37,13 @@ vitest.mock("../../../../i18n", () => ({ // Mock console methods const consoleMocks = { - error: vitest.spyOn(console, "error").mockImplementation(function () {}), - warn: vitest.spyOn(console, "warn").mockImplementation(function () {}), + error: vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}), + warn: vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}), } describe("OpenAiEmbedder", () => { let embedder: OpenAiEmbedder - let mockEmbeddingsCreate: MockedFunction + let mockEmbeddingsCreate: ReturnType Promise>> let MockedOpenAI: MockedClass beforeEach(() => { @@ -52,9 +54,10 @@ describe("OpenAiEmbedder", () => { MockedOpenAI = OpenAI as MockedClass mockEmbeddingsCreate = vitest.fn() + // Only embeddings.create is used; the SDK resource's private client is not needed. MockedOpenAI.prototype.embeddings = { create: mockEmbeddingsCreate, - } as any + } as unknown as OpenAI.Embeddings embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-api-key", @@ -68,7 +71,7 @@ describe("OpenAiEmbedder", () => { describe("constructor", () => { it("should initialize with provided options", () => { - expect(MockedOpenAI).toHaveBeenCalledWith({ apiKey: "test-api-key" }) + expect(MockedOpenAI).toHaveBeenCalledWith({ apiKey: "test-api-key", maxRetries: 0 }) expect(embedder.embedderInfo.name).toBe("openai") }) @@ -77,7 +80,7 @@ describe("OpenAiEmbedder", () => { openAiEmbeddingModelId: "text-embedding-3-small", }) - expect(MockedOpenAI).toHaveBeenCalledWith({ apiKey: "not-provided" }) + expect(MockedOpenAI).toHaveBeenCalledWith({ apiKey: "not-provided", maxRetries: 0 }) }) it("should use default model if not specified", () => { @@ -102,10 +105,13 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: testModelId, + }, + undefined, + ) expect(result).toEqual({ embeddings: [[0.1, 0.2, 0.3]], usage: { promptTokens: 10, totalTokens: 15 }, @@ -122,10 +128,13 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: testModelId, + }, + undefined, + ) expect(result).toEqual({ embeddings: [ [0.1, 0.2, 0.3], @@ -146,10 +155,13 @@ describe("OpenAiEmbedder", () => { await embedder.createEmbeddings(testTexts, customModel) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: customModel, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: customModel, + }, + undefined, + ) }) it("should handle missing usage data gracefully", async () => { @@ -202,13 +214,16 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) // Verify warning was logged - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) + expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) // Verify only normal texts were processed - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: [normalText, "another normal"], - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: [normalText, "another normal"], + model: testModelId, + }, + undefined, + ) expect(result.embeddings).toHaveLength(2) }) @@ -253,7 +268,7 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(console.warn).toHaveBeenCalledTimes(2) + expect(codeIndexLog.warn).toHaveBeenCalledTimes(2) expect(mockEmbeddingsCreate).not.toHaveBeenCalled() expect(result).toEqual({ embeddings: [], @@ -274,38 +289,22 @@ describe("OpenAiEmbedder", () => { vitest.useRealTimers() }) - it("should retry on rate limit errors with exponential backoff", async () => { - const testTexts = ["Hello world"] - const rateLimitError = { status: 429, message: "Rate limit exceeded" } - - mockEmbeddingsCreate - .mockRejectedValueOnce(rateLimitError) - .mockRejectedValueOnce(rateLimitError) - .mockResolvedValueOnce({ - data: [{ embedding: [0.1, 0.2, 0.3] }], - usage: { prompt_tokens: 10, total_tokens: 15 }, - }) - - const resultPromise = embedder.createEmbeddings(testTexts) - - // Fast-forward through the delays - await vitest.advanceTimersByTimeAsync(INITIAL_RETRY_DELAY_MS) // First retry delay - await vitest.advanceTimersByTimeAsync(INITIAL_RETRY_DELAY_MS * 2) // Second retry delay - - const result = await resultPromise - - expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(3) - expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("Rate limit hit, retrying in")) - expect(result).toEqual({ - embeddings: [[0.1, 0.2, 0.3]], - usage: { promptTokens: 10, totalTokens: 15 }, + it("returns rate limits without internal retries", async () => { + const cause = Object.assign(new Error("rate limited"), { status: 429, name: "ThrottlingException" }) + mockEmbeddingsCreate.mockRejectedValue(cause) + await expect(embedder.createEmbeddings(["test"])).rejects.toMatchObject({ + classification: "rate-limit", + statusCode: 429, + cause, }) + expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) }) it("should not retry on non-rate-limit errors", async () => { const testTexts = ["Hello world"] const authError = new Error("Unauthorized") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) @@ -314,18 +313,18 @@ describe("OpenAiEmbedder", () => { ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) - expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) it("should throw error immediately on non-retryable errors", async () => { const testTexts = ["Hello world"] const serverError = new Error("Internal server error") - ;(serverError as any).status = 500 + Object.assign(serverError, { status: 500 }) mockEmbeddingsCreate.mockRejectedValue(serverError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: HTTP 500 - Internal server error", + "Failed to create embeddings: HTTP 500 - Internal server error", ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) @@ -343,10 +342,10 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(apiError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: API connection failed", + "Failed to create embeddings: API connection failed", ) - expect(console.error).toHaveBeenCalledWith( + expect(codeIndexLog.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI embedder error"), expect.any(Error), ) @@ -379,7 +378,7 @@ describe("OpenAiEmbedder", () => { it("should provide specific authentication error message", async () => { const testTexts = ["Hello world"] const authError = new Error("Invalid API key") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) @@ -391,12 +390,12 @@ describe("OpenAiEmbedder", () => { it("should provide detailed error message for HTTP errors", async () => { const testTexts = ["Hello world"] const httpError = new Error("Bad request") - ;(httpError as any).status = 400 + Object.assign(httpError, { status: 400 }) mockEmbeddingsCreate.mockRejectedValue(httpError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: HTTP 400 - Bad request", + "Failed to create embeddings: HTTP 400 - Bad request", ) }) @@ -407,7 +406,7 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(networkError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Network timeout", + "Failed to create embeddings: Network timeout", ) }) @@ -418,7 +417,7 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(weirdError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Custom error object", + "Failed to create embeddings: Custom error object", ) }) @@ -429,7 +428,7 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(unknownError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Unknown error", + "Failed to create embeddings: Unknown error", ) }) @@ -440,7 +439,7 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(stringError) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: Something went wrong", + "Failed to create embeddings: Something went wrong", ) }) @@ -472,7 +471,7 @@ describe("OpenAiEmbedder", () => { mockEmbeddingsCreate.mockRejectedValue(errorWithResponseStatus) await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow( - "Failed to create embeddings after 3 attempts: HTTP 403 - Request failed", + "Failed to create embeddings: HTTP 403 - Request failed", ) }) }) @@ -498,7 +497,7 @@ describe("OpenAiEmbedder", () => { it("should fail validation with authentication error", async () => { const authError = new Error("Invalid API key") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) const result = await embedder.validateConfiguration() @@ -509,13 +508,10 @@ describe("OpenAiEmbedder", () => { it("should fail validation with rate limit error", async () => { const rateLimitError = new Error("Rate limit exceeded") - ;(rateLimitError as any).status = 429 + Object.assign(rateLimitError, { status: 429 }) mockEmbeddingsCreate.mockRejectedValue(rateLimitError) - const result = await embedder.validateConfiguration() - - expect(result.valid).toBe(false) - expect(result.error).toBe("embeddings:validation.serviceUnavailable") + await expect(embedder.validateConfiguration()).rejects.toBe(rateLimitError) }) it("should fail validation with connection error", async () => { @@ -530,7 +526,7 @@ describe("OpenAiEmbedder", () => { it("should fail validation with generic error", async () => { const genericError = new Error("Unknown error") - ;(genericError as any).status = 500 + Object.assign(genericError, { status: 500 }) mockEmbeddingsCreate.mockRejectedValue(genericError) const result = await embedder.validateConfiguration() diff --git a/src/services/code-index/embedders/__tests__/openrouter.spec.ts b/src/services/code-index/embedders/__tests__/openrouter.spec.ts index d782ecd9cb..e6080b1d1b 100644 --- a/src/services/code-index/embedders/__tests__/openrouter.spec.ts +++ b/src/services/code-index/embedders/__tests__/openrouter.spec.ts @@ -1,4 +1,6 @@ -import type { MockedClass, MockedFunction } from "vitest" +import { codeIndexLog } from "../../code-index-log" +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) +import type { MockedClass } from "vitest" import { describe, it, expect, beforeEach, vi } from "vitest" import { OpenAI } from "openai" import { OpenRouterEmbedder, OPENROUTER_DEFAULT_PROVIDER_NAME } from "../openrouter" @@ -21,14 +23,14 @@ vi.mock("@roo-code/telemetry", () => ({ // Mock i18n vi.mock("../../../../i18n", () => ({ - t: (key: string, params?: Record) => { + t: (key: string, params?: Record) => { const translations: Record = { "embeddings:validation.apiKeyRequired": "validation.apiKeyRequired", "embeddings:authenticationFailed": "Failed to create embeddings: Authentication failed. Please check your OpenRouter API key.", - "embeddings:failedWithStatus": `Failed to create embeddings after ${params?.attempts} attempts: HTTP ${params?.statusCode} - ${params?.errorMessage}`, - "embeddings:failedWithError": `Failed to create embeddings after ${params?.attempts} attempts: ${params?.errorMessage}`, - "embeddings:failedMaxAttempts": `Failed to create embeddings after ${params?.attempts} attempts`, + "embeddings:failedWithStatus": `Failed to create embeddings: HTTP ${params?.statusCode} - ${params?.errorMessage}`, + "embeddings:failedWithError": `Failed to create embeddings: ${params?.errorMessage}`, + "embeddings:failedMaxAttempts": `Failed to create embeddings`, "embeddings:textExceedsTokenLimit": `Text at index ${params?.index} exceeds maximum token limit (${params?.itemTokens} > ${params?.maxTokens}). Skipping.`, "embeddings:rateLimitRetry": `Rate limit hit, retrying in ${params?.delayMs}ms (attempt ${params?.attempt}/${params?.maxRetries})`, } @@ -40,13 +42,13 @@ const MockedOpenAI = OpenAI as MockedClass describe("OpenRouterEmbedder", () => { const mockApiKey = "test-api-key" - let mockEmbeddingsCreate: MockedFunction - let mockOpenAIInstance: any + let mockEmbeddingsCreate: ReturnType Promise>> + let mockOpenAIInstance: { embeddings: { create: typeof mockEmbeddingsCreate } } beforeEach(() => { clearAllMocks() - vi.spyOn(console, "warn").mockImplementation(function () {}) - vi.spyOn(console, "error").mockImplementation(function () {}) + vi.spyOn(codeIndexLog, "warn").mockImplementation(function () {}) + vi.spyOn(codeIndexLog, "error").mockImplementation(function () {}) // Setup mock OpenAI instance mockEmbeddingsCreate = vi.fn() @@ -57,7 +59,8 @@ describe("OpenRouterEmbedder", () => { } MockedOpenAI.mockImplementation(function () { - return mockOpenAIInstance + // The embedder only uses embeddings.create on this client double. + return mockOpenAIInstance as unknown as OpenAI }) }) @@ -92,6 +95,7 @@ describe("OpenRouterEmbedder", () => { expect(MockedOpenAI).toHaveBeenCalledWith({ baseURL: "https://openrouter.ai/api/v1", apiKey: mockApiKey, + maxRetries: 0, defaultHeaders: { "HTTP-Referer": "https://github.com/Zoo-Code-Org/Zoo-Code", "X-Title": "Zoo Code", @@ -329,7 +333,7 @@ describe("OpenRouterEmbedder", () => { it("should handle validation failure", async () => { const authError = new Error("Invalid API key") - ;(authError as any).status = 401 + Object.assign(authError, { status: 401 }) mockEmbeddingsCreate.mockRejectedValue(authError) diff --git a/src/services/code-index/embedders/__tests__/paced-transport-writer.ts b/src/services/code-index/embedders/__tests__/paced-transport-writer.ts new file mode 100644 index 0000000000..37c078ec46 --- /dev/null +++ b/src/services/code-index/embedders/__tests__/paced-transport-writer.ts @@ -0,0 +1,82 @@ +import path from "node:path" +import { setTimeout as sleep } from "node:timers/promises" +import { FileIndexWriter } from "../../file-completion" +import { FileFailureRegistry } from "../../file-failure-registry" +import { IndexWriteGate } from "../../index-write-gate" +import { ProviderPacerRegistry } from "../../provider-pacer" +import type { CacheManager } from "../../cache-manager" +import type { GenerationAllocator } from "../../generation-allocator" +import type { IEmbedder, IVectorStore, CodeBlock } from "../../interfaces" + +export function pacedTransportWriter(store?: IVectorStore) { + const workspace = path.resolve("/workspace") + const file = path.join(workspace, "test.ts") + const failures = new FileFailureRegistry() + const pacers = new ProviderPacerRegistry() + const cache = { + invalidateFile: vi.fn().mockResolvedValue(undefined), + completeFile: vi.fn(), + completeRemoval: vi.fn(), + } + const vectorStore: IVectorStore = store ?? { + rateLimitScope: "qdrant:test", + initialize: vi.fn().mockResolvedValue(false), + upsertPoints: vi.fn().mockResolvedValue(undefined), + deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), + deletePointsByMultipleFilePaths: vi + .fn() + .mockResolvedValue(undefined), + countPointsForFile: vi.fn().mockResolvedValue({ current: 1, other: 0 }), + search: vi.fn().mockResolvedValue([]), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), + collectionExists: vi.fn().mockResolvedValue(true), + hasIndexedData: vi.fn().mockResolvedValue(true), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + } + const gate = new IndexWriteGate() + // The cache supplies invalidation/completion; the allocator supplies only allocate. + const writer = new FileIndexWriter( + cache as unknown as CacheManager, + vectorStore, + gate, + { allocate: async () => 1 } as unknown as GenerationAllocator, + workspace, + failures, + pacers, + ) + const block: CodeBlock = { + file_path: file, + identifier: null, + type: "function", + start_line: 1, + end_line: 1, + content: "test", + fileHash: "revision", + segmentHash: "segment", + } + return { + writer, + gate, + pacers, + failures, + cache, + file, + replace: (embedder: IEmbedder, signal = new AbortController().signal) => + writer.replace(file, "revision", [block], embedder, signal), + } +} + +export async function waitForTransport(assertion: () => void): Promise { + const deadline = performance.now() + 2000 + while (true) { + try { + assertion() + return + } catch (error) { + if (performance.now() >= deadline) throw error + await sleep(1) + } + } +} diff --git a/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts b/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts new file mode 100644 index 0000000000..27f793909b --- /dev/null +++ b/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts @@ -0,0 +1,130 @@ +import "../../__tests__/mock-code-index-log" +import { OpenAiEmbedder } from "../openai" +import { OpenAICompatibleEmbedder } from "../openai-compatible" +import { OpenRouterEmbedder } from "../openrouter" +import { GeminiEmbedder } from "../gemini" +import { MistralEmbedder } from "../mistral" +import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" +import { CodeIndexOllamaEmbedder } from "../ollama" +import { pacedTransportWriter } from "./paced-transport-writer" +import { codeIndexLog } from "../../code-index-log" + +vi.mock("../../../../api/providers/openai-native", () => ({ + OpenAiNativeHandler: class { + constructor(protected options: unknown) {} + }, +})) +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) + +const factories = [ + ["OpenAI", () => new OpenAiEmbedder({ openAiNativeApiKey: "test-key" })], + ["compatible SDK", () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key")], + ["compatible fetch", () => new OpenAICompatibleEmbedder("https://example.test/embeddings", "test-key")], + ["OpenRouter", () => new OpenRouterEmbedder("test-key")], + ["Gemini", () => new GeminiEmbedder("test-key")], + ["Mistral", () => new MistralEmbedder("test-key")], + ["Vercel", () => new VercelAiGatewayEmbedder("test-key")], + ["Ollama", () => new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "https://example.test" })], +] as const + +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date("2026-11-11T12:00:00Z")) + vi.spyOn(Math, "random").mockReturnValue(0) + vi.mocked(codeIndexLog.info).mockClear() +}) +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +function response(status = 200, headers: Record = {}) { + return new Response( + JSON.stringify( + status === 200 + ? { data: [{ embedding: [1, 0] }], embeddings: [[1, 0]] } + : { error: { message: "limited" } }, + ), + { + status, + headers: { "content-type": "application/json", ...headers }, + }, + ) +} + +describe.each(factories)("T25 %s paced transport", (_name, createEmbedder) => { + it.each( + [2, 60, 120].flatMap((seconds) => [ + { header: "retry-after", value: String(seconds), delay: seconds * 1000 }, + { header: "retry-after-ms", value: String(seconds * 1000), delay: seconds * 1000 }, + { + header: "retry-after", + value: new Date(Date.parse("2026-11-11T12:00:00Z") + seconds * 1000).toUTCString(), + delay: seconds * 1000, + }, + ]), + )("honors the entire $header: $value", async ({ header, value, delay }) => { + const request = vi + .fn() + .mockImplementationOnce(async () => response(429, { [header]: value })) + .mockImplementation(async () => response()) + vi.stubGlobal("fetch", request) + const { replace, failures, cache } = pacedTransportWriter() + const pending = replace(createEmbedder()) + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync(delay - 1) + expect(request).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + await pending + expect(request).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() + }) + + it.each([undefined, "invalid", "-1", "Tue, 10 Nov 2026 12:00:00 GMT"])( + "recovers after four limits with growing fallback for %s and no budget charge", + async (header) => { + let issued = 0 + const request = vi.fn(async () => + ++issued <= 4 ? response(429, header === undefined ? {} : { "retry-after": header }) : response(), + ) + vi.stubGlobal("fetch", request) + const { replace, failures, cache } = pacedTransportWriter() + const pending = replace(createEmbedder()) + await vi.advanceTimersByTimeAsync(0) + for (const [index, delay] of [500, 1000, 2000, 4000].entries()) { + expect(request).toHaveBeenCalledTimes(index + 1) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(delay - 1) + expect(request).toHaveBeenCalledTimes(index + 1) + await vi.advanceTimersByTimeAsync(1) + } + await pending + expect(request).toHaveBeenCalledTimes(5) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() + expect(codeIndexLog.info).toHaveBeenCalledTimes(4) + }, + ) + + it("cancels a provider wait without another transport request or failure entry", async () => { + const request = vi.fn(async () => response(429, { "retry-after": "120" })) + vi.stubGlobal("fetch", request) + const controller = new AbortController() + const { replace, failures, pacers } = pacedTransportWriter() + const pending = replace(createEmbedder(), controller.signal).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledOnce() + expect(pacers.waitingPaths()).toEqual(["test.ts"]) + expect(failures.list()).toEqual([]) + controller.abort() + expect(await pending).toBe(controller.signal.reason) + await vi.advanceTimersByTimeAsync(240_000) + expect(request).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + }) +}) diff --git a/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts b/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts new file mode 100644 index 0000000000..6616bbf72f --- /dev/null +++ b/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts @@ -0,0 +1,71 @@ +import "../../__tests__/mock-code-index-log" +import { OpenAiEmbedder } from "../openai" +import { OpenAICompatibleEmbedder } from "../openai-compatible" +import { OpenRouterEmbedder } from "../openrouter" +import { GeminiEmbedder } from "../gemini" +import { MistralEmbedder } from "../mistral" +import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" +import { CodeIndexOllamaEmbedder } from "../ollama" +import { BedrockEmbedder } from "../bedrock" +import { ProviderPacerRegistry } from "../../provider-pacer" + +vi.mock("../../../../api/providers/openai-native", () => ({ + OpenAiNativeHandler: class { + constructor(protected options: unknown) {} + }, +})) +vi.mock("@aws-sdk/client-bedrock-runtime", () => ({ BedrockRuntimeClient: vi.fn(), InvokeModelCommand: vi.fn() })) +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) + +const factories = [ + ["openai", (key: string) => new OpenAiEmbedder({ openAiNativeApiKey: key })], + ["openai-compatible", (key: string) => new OpenAICompatibleEmbedder("https://example.test/v1", key)], + ["openrouter", (key: string) => new OpenRouterEmbedder(key)], + ["gemini", (key: string) => new GeminiEmbedder(key)], + ["mistral", (key: string) => new MistralEmbedder(key)], + ["vercel-ai-gateway", (key: string) => new VercelAiGatewayEmbedder(key)], +] as const + +it.each(factories)("%s shares identical allowances and isolates credentials", (provider, createEmbedder) => { + const first = createEmbedder("first-secret") + const recreated = createEmbedder("first-secret") + const other = createEmbedder("second-secret") + expect(first.rateLimitScope).toMatch(new RegExp(`^${provider}:[a-f0-9]{64}$`)) + expect(first.rateLimitScope).toBe(recreated.rateLimitScope) + expect(first.rateLimitScope).not.toBe(other.rateLimitScope) + expect(first.rateLimitScope).not.toContain("first-secret") + const registry = new ProviderPacerRegistry() + expect(registry.get(first.rateLimitScope)).toBe(registry.get(recreated.rateLimitScope)) + expect(registry.get(first.rateLimitScope)).not.toBe(registry.get(other.rateLimitScope)) +}) + +it("normalizes equivalent URLs while isolating different endpoints", () => { + const first = new OpenAICompatibleEmbedder("https://EXAMPLE.test:443/v1/", "key") + const same = new OpenAICompatibleEmbedder("https://example.test/v1", "key") + const other = new OpenAICompatibleEmbedder("https://example.test/v2", "key") + expect(first.rateLimitScope).toBe(same.rateLimitScope) + expect(first.rateLimitScope).not.toBe(other.rateLimitScope) +}) + +it("Ollama scopes depend only on the normalized endpoint", () => { + const first = new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "http://LOCALHOST:11434/" }) + const same = new CodeIndexOllamaEmbedder({ ollamaModelId: "another-model" }) + const other = new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "http://localhost:11435" }) + expect(first.rateLimitScope).toBe(same.rateLimitScope) + expect(first.rateLimitScope).not.toBe(other.rateLimitScope) +}) + +it("Bedrock isolates region and profile, not model", () => { + const first = new BedrockEmbedder("us-east-1", "profile") + expect(first.rateLimitScope).toBe(new BedrockEmbedder("us-east-1", "profile", "other-model").rateLimitScope) + expect(first.rateLimitScope).not.toBe(new BedrockEmbedder("us-west-2", "profile").rateLimitScope) + expect(first.rateLimitScope).not.toBe(new BedrockEmbedder("us-east-1", "other-profile").rateLimitScope) + expect(first.rateLimitScope).toMatch(/^bedrock:[a-f0-9]{64}$/) + expect(first.rateLimitScope).not.toContain("profile") +}) + +it("wrapped providers retain their own provider identity", () => { + const gemini = new GeminiEmbedder("key") + const compatible = new OpenAICompatibleEmbedder("https://generativelanguage.googleapis.com/v1beta/openai/", "key") + expect(gemini.rateLimitScope).not.toBe(compatible.rateLimitScope) +}) diff --git a/src/services/code-index/embedders/bedrock.ts b/src/services/code-index/embedders/bedrock.ts index 0e153f3c2c..3496dd4552 100644 --- a/src/services/code-index/embedders/bedrock.ts +++ b/src/services/code-index/embedders/bedrock.ts @@ -1,13 +1,10 @@ +import { codeIndexLog } from "../code-index-log" +import { makeRateLimitScope } from "../provider-pacer" import { BedrockRuntimeClient, InvokeModelCommand, InvokeModelCommandInput } from "@aws-sdk/client-bedrock-runtime" import { fromIni, fromNodeProviderChain } from "@aws-sdk/credential-providers" import { createProxyRoutingRequestHandler } from "../../../utils/networkProxy" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces" -import { - MAX_BATCH_TOKENS, - MAX_ITEM_TOKENS, - MAX_BATCH_RETRIES as MAX_RETRIES, - INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS, -} from "../constants" +import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId } from "../../../shared/embeddingModels" import { Package } from "../../../shared/package" import { t } from "../../../i18n" @@ -16,9 +13,10 @@ import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" /** - * Amazon Bedrock implementation of the embedder interface with batching and rate limiting + * Amazon Bedrock implementation of the embedder interface with batching */ export class BedrockEmbedder implements IEmbedder { + readonly rateLimitScope: string private bedrockClient: BedrockRuntimeClient private readonly defaultModelId: string @@ -37,6 +35,13 @@ export class BedrockEmbedder implements IEmbedder { throw new Error("Region is required for AWS Bedrock embedder") } + this.rateLimitScope = makeRateLimitScope( + "bedrock", + undefined, + undefined, + JSON.stringify([region, profile ?? ""]), + ) + // Initialize the Bedrock client with credentials // If profile is specified, use it; otherwise use default credential chain const credentials = this.profile ? fromIni({ profile: this.profile }) : fromNodeProviderChain() @@ -48,6 +53,7 @@ export class BedrockEmbedder implements IEmbedder { this.bedrockClient = new BedrockRuntimeClient({ userAgentAppId: `ZooCode#${Package.version}`, + maxAttempts: 1, region: this.region, credentials, ...(requestHandler && { requestHandler }), @@ -57,12 +63,13 @@ export class BedrockEmbedder implements IEmbedder { } /** - * Creates embeddings for the given texts with batching and rate limiting + * Creates embeddings for the given texts with batching * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId const allEmbeddings: number[][] = [] @@ -70,6 +77,7 @@ export class BedrockEmbedder implements IEmbedder { const remainingTexts = [...texts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -79,7 +87,7 @@ export class BedrockEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -105,7 +113,7 @@ export class BedrockEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -115,73 +123,43 @@ export class BedrockEmbedder implements IEmbedder { return { embeddings: allEmbeddings, usage } } - /** - * Helper method to handle batch embedding with retries and exponential backoff - * @param batchTexts Array of texts to embed in this batch - * @param model Model identifier to use - * @returns Promise resolving to embeddings and usage statistics - */ - private async _embedBatchWithRetries( + private async _embedBatch( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { - for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { - try { - const embeddings: number[][] = [] - let totalPromptTokens = 0 - let totalTokens = 0 - - // Process each text in the batch - // Note: Amazon Titan models typically don't support batch embedding in a single request - // So we process them individually - for (const text of batchTexts) { - const embedding = await this._invokeEmbeddingModel(text, model) - embeddings.push(embedding.embedding) - totalPromptTokens += embedding.inputTextTokenCount || 0 - totalTokens += embedding.inputTextTokenCount || 0 - } - - return { - embeddings, - usage: { - promptTokens: totalPromptTokens, - totalTokens, - }, - } - } catch (error: any) { - const hasMoreAttempts = attempts < MAX_RETRIES - 1 - - // Check if it's a rate limit error - if (error.name === "ThrottlingException" && hasMoreAttempts) { - const delayMs = INITIAL_DELAY_MS * Math.pow(2, attempts) - console.warn( - t("embeddings:rateLimitRetry", { - delayMs, - attempt: attempts + 1, - maxRetries: MAX_RETRIES, - }), - ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) - continue - } - - // Capture telemetry before reformatting the error - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "BedrockEmbedder:_embedBatchWithRetries", - attempt: attempts + 1, - }) - - // Log the error for debugging - console.error(`Bedrock embedder error (attempt ${attempts + 1}/${MAX_RETRIES}):`, error) + try { + signal?.throwIfAborted() + const embeddings: number[][] = [] + let totalPromptTokens = 0 + let totalTokens = 0 + + // Amazon Titan models typically need one request per text. + for (const text of batchTexts) { + signal?.throwIfAborted() + const embedding = await this._invokeEmbeddingModel(text, model, signal) + embeddings.push(embedding.embedding) + totalPromptTokens += embedding.inputTextTokenCount || 0 + totalTokens += embedding.inputTextTokenCount || 0 + } - // Format and throw the error - throw formatEmbeddingError(error, MAX_RETRIES) + return { + embeddings, + usage: { + promptTokens: totalPromptTokens, + totalTokens, + }, } + } catch (error) { + signal?.throwIfAborted() + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "BedrockEmbedder:_embedBatch", + }) + codeIndexLog.error("Bedrock embedder error:", error) + throw formatEmbeddingError(error) } - - throw new Error(t("embeddings:failedMaxAttempts", { attempts: MAX_RETRIES })) } /** @@ -193,6 +171,7 @@ export class BedrockEmbedder implements IEmbedder { private async _invokeEmbeddingModel( text: string, model: string, + signal?: AbortSignal, ): Promise<{ embedding: number[]; inputTextTokenCount?: number }> { let requestBody: any const modelId = model @@ -245,7 +224,9 @@ export class BedrockEmbedder implements IEmbedder { const command = new InvokeModelCommand(params) - const response = await this.bedrockClient.send(command) + signal?.throwIfAborted() + const response = await this.bedrockClient.send(command, ...(signal ? [{ abortSignal: signal }] : [])) + signal?.throwIfAborted() // Parse the response const responseBody = JSON.parse(new TextDecoder().decode(response.body)) diff --git a/src/services/code-index/embedders/cancellation.ts b/src/services/code-index/embedders/cancellation.ts new file mode 100644 index 0000000000..a5e0d61d9b --- /dev/null +++ b/src/services/code-index/embedders/cancellation.ts @@ -0,0 +1,15 @@ +export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timeout) + signal?.removeEventListener("abort", onAbort) + reject(signal?.reason) + } + const timeout = setTimeout(() => { + signal?.removeEventListener("abort", onAbort) + resolve() + }, delayMs) + signal?.addEventListener("abort", onAbort, { once: true }) + }) +} diff --git a/src/services/code-index/embedders/embedder-validation-manager.ts b/src/services/code-index/embedders/embedder-validation-manager.ts index 84dd73aac6..92fd31e662 100644 --- a/src/services/code-index/embedders/embedder-validation-manager.ts +++ b/src/services/code-index/embedders/embedder-validation-manager.ts @@ -2,12 +2,14 @@ import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" import { t } from "../../../i18n" import type { IEmbedder } from "../interfaces" +import { classifyIndexingError } from "../shared/indexing-errors" export class EmbedderValidationManager { public async validateEmbedder(embedder: IEmbedder): Promise<{ valid: boolean; error?: string }> { try { return await embedder.validateConfiguration() } catch (error) { + if (classifyIndexingError(error) === "rate-limit") throw error TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/gemini.ts b/src/services/code-index/embedders/gemini.ts index 03bfc35aae..29f8e1f2b6 100644 --- a/src/services/code-index/embedders/gemini.ts +++ b/src/services/code-index/embedders/gemini.ts @@ -1,4 +1,5 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" +import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { GEMINI_MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -16,6 +17,8 @@ import { TelemetryService } from "@roo-code/telemetry" * migrated to gemini-embedding-001 for backward compatibility. */ export class GeminiEmbedder implements IEmbedder { + readonly rateLimitScope: string + private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/" private static readonly DEFAULT_MODEL = "gemini-embedding-001" @@ -47,6 +50,8 @@ export class GeminiEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } + this.rateLimitScope = makeRateLimitScope("gemini", GeminiEmbedder.GEMINI_BASE_URL, apiKey) + // Migrate deprecated models to their replacements silently const migratedModelId = modelId ? GeminiEmbedder.migrateModelId(modelId) : undefined @@ -68,12 +73,14 @@ export class GeminiEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } catch (error) { + signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/mistral.ts b/src/services/code-index/embedders/mistral.ts index c23bcbba1d..174b9424aa 100644 --- a/src/services/code-index/embedders/mistral.ts +++ b/src/services/code-index/embedders/mistral.ts @@ -1,4 +1,5 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" +import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -13,6 +14,8 @@ import { TelemetryService } from "@roo-code/telemetry" * - codestral-embed-2505 (dimension: 1536) */ export class MistralEmbedder implements IEmbedder { + readonly rateLimitScope: string + private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly MISTRAL_BASE_URL = "https://api.mistral.ai/v1" private static readonly DEFAULT_MODEL = "codestral-embed-2505" @@ -28,6 +31,8 @@ export class MistralEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } + this.rateLimitScope = makeRateLimitScope("mistral", MistralEmbedder.MISTRAL_BASE_URL, apiKey) + // Use provided model or default this.modelId = modelId || MistralEmbedder.DEFAULT_MODEL @@ -46,12 +51,14 @@ export class MistralEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } catch (error) { + signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/ollama.ts b/src/services/code-index/embedders/ollama.ts index 654c41240f..56710209fe 100644 --- a/src/services/code-index/embedders/ollama.ts +++ b/src/services/code-index/embedders/ollama.ts @@ -1,3 +1,6 @@ +import { codeIndexLog } from "../code-index-log" +import { EmbeddingError } from "../shared/indexing-errors" +import { makeRateLimitScope } from "../provider-pacer" import { ApiHandlerOptions } from "../../../shared/api" import { EmbedderInfo, EmbeddingResponse, IEmbedder } from "../interfaces" import { getModelQueryPrefix } from "../../../shared/embeddingModels" @@ -15,6 +18,7 @@ const OLLAMA_VALIDATION_TIMEOUT_MS = 30000 // 30 seconds for validation requests * Implements the IEmbedder interface using a local Ollama instance. */ export class CodeIndexOllamaEmbedder implements IEmbedder { + readonly rateLimitScope: string private readonly baseUrl: string private readonly defaultModelId: string @@ -26,6 +30,7 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { baseUrl = baseUrl.replace(/\/+$/, "") this.baseUrl = baseUrl + this.rateLimitScope = makeRateLimitScope("ollama", baseUrl) this.defaultModelId = options.ollamaModelId || "nomic-embed-text:latest" } @@ -35,9 +40,10 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { * @param model - Optional model ID to override the default. * @returns A promise that resolves to an EmbeddingResponse containing the embeddings and usage data. */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId - const url = `${this.baseUrl}/api/embed` // Endpoint as specified + const url = `${this.baseUrl}/api/embed` // Apply model-specific query prefix if required const queryPrefix = getModelQueryPrefix("ollama", modelToUse) @@ -50,7 +56,7 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -64,14 +70,11 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { }) : texts - try { - // Note: Standard Ollama API uses 'prompt' for single text, not 'input' for array. - // Implementing based on user's specific request structure. - - // Add timeout to prevent indefinite hanging - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal + try { const response = await fetch(url, { method: "POST", headers: { @@ -79,11 +82,11 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { }, body: JSON.stringify({ model: modelToUse, - input: processedTexts, // Using 'input' as requested + input: processedTexts, }), - signal: controller.signal, + signal: requestSignal, }) - clearTimeout(timeoutId) + signal?.throwIfAborted() if (!response.ok) { let errorBody = t("embeddings:ollama.couldNotReadErrorBody") @@ -92,18 +95,19 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { } catch { // Ignore error reading body } - throw new Error( + throw new EmbeddingError( t("embeddings:ollama.requestFailed", { status: response.status, statusText: response.statusText, errorBody, }), + response, ) } const data = await response.json() + signal?.throwIfAborted() - // Extract embeddings using 'embeddings' key as requested const embeddings = data.embeddings if (!embeddings || !Array.isArray(embeddings)) { throw new Error(t("embeddings:ollama.invalidResponseStructure")) @@ -113,27 +117,29 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { embeddings: embeddings, } } catch (error: any) { - // Capture telemetry before reformatting the error + signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, location: "OllamaEmbedder:createEmbeddings", }) + codeIndexLog.error("Ollama embedding failed:", error) - // Log the original error for debugging purposes - console.error("Ollama embedding failed:", error) - - // Handle specific error types with better messages if (error.name === "AbortError") { - throw new Error(t("embeddings:validation.connectionFailed")) + throw new EmbeddingError(t("embeddings:validation.connectionFailed"), error, { + classification: controller.signal.aborted ? "connection" : "permanent", + }) } else if (error.message?.includes("fetch failed") || error.code === "ECONNREFUSED") { - throw new Error(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl })) + throw new EmbeddingError(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl }), error) } else if (error.code === "ENOTFOUND") { - throw new Error(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl })) + throw new EmbeddingError(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl }), error) } - // Re-throw a more specific error for the caller - throw new Error(t("embeddings:ollama.embeddingFailed", { message: error.message })) + throw new EmbeddingError(t("embeddings:ollama.embeddingFailed", { message: error.message }), error, { + retryAfterMs: error instanceof EmbeddingError ? error.retryAfterMs : undefined, + }) + } finally { + clearTimeout(timeoutId) } } diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 5cbbf67efc..341ec28357 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -1,22 +1,17 @@ +import { codeIndexLog } from "../code-index-log" +import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" -import { - MAX_BATCH_TOKENS, - MAX_ITEM_TOKENS, - MAX_BATCH_RETRIES as MAX_RETRIES, - INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS, -} from "../constants" +import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId, getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" import { withValidationErrorHandling, HttpError, formatEmbeddingError } from "../shared/validation-helpers" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" -import { Mutex } from "async-mutex" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" interface EmbeddingItem { embedding: string | number[] - [key: string]: any } interface OpenAIEmbeddingResponse { @@ -28,11 +23,12 @@ interface OpenAIEmbeddingResponse { } /** - * OpenAI Compatible implementation of the embedder interface with batching and rate limiting. + * OpenAI Compatible implementation of the embedder interface with batching. * This embedder allows using any OpenAI-compatible API endpoint by specifying a custom baseURL. */ export class OpenAICompatibleEmbedder implements IEmbedder { + readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string private readonly baseUrl: string @@ -40,16 +36,6 @@ export class OpenAICompatibleEmbedder implements IEmbedder { private readonly isFullUrl: boolean private readonly maxItemTokens: number - // Global rate limiting state shared across all instances - private static globalRateLimitState = { - isRateLimited: false, - rateLimitResetTime: 0, - consecutiveRateLimitErrors: 0, - lastRateLimitError: 0, - // Mutex to ensure thread-safe access to rate limit state - mutex: new Mutex(), - } - /** * Creates a new OpenAI Compatible embedder * @param baseUrl The base URL for the OpenAI-compatible API endpoint @@ -67,11 +53,13 @@ export class OpenAICompatibleEmbedder implements IEmbedder { this.baseUrl = baseUrl this.apiKey = apiKey + this.rateLimitScope = makeRateLimitScope("openai-compatible", baseUrl, apiKey) // Wrap OpenAI client creation to handle invalid API key characters try { this.embeddingsClient = new OpenAI({ baseURL: baseUrl, + maxRetries: 0, apiKey: apiKey, }) } catch (error) { @@ -86,12 +74,13 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } /** - * Creates embeddings for the given texts with batching and rate limiting + * Creates embeddings for the given texts with batching * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -105,7 +94,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -124,6 +113,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -133,7 +123,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > this.maxItemTokens) { - console.warn( + codeIndexLog.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -159,7 +149,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -203,9 +193,11 @@ export class OpenAICompatibleEmbedder implements IEmbedder { url: string, batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise { const response = await fetch(url, { method: "POST", + ...(signal ? { signal } : {}), headers: { "Content-Type": "application/json", // Azure OpenAI uses 'api-key' header, while OpenAI uses 'Authorization' @@ -235,6 +227,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } const error = new Error(`HTTP ${status}: ${errorText}`) as HttpError error.status = status || response?.status || 0 + error.headers = response?.headers throw error } @@ -247,113 +240,69 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } } - /** - * Helper method to handle batch embedding with retries and exponential backoff - * @param batchTexts Array of texts to embed in this batch - * @param model Model identifier to use - * @returns Promise resolving to embeddings and usage statistics - */ - private async _embedBatchWithRetries( + private async _embedBatch( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { // Use cached value for performance const isFullUrl = this.isFullUrl - for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { - // Check global rate limit before attempting request - await this.waitForGlobalRateLimit() - - try { - let response: OpenAIEmbeddingResponse + try { + signal?.throwIfAborted() + let response: OpenAIEmbeddingResponse - if (isFullUrl) { - // Use direct HTTP request for full endpoint URLs - response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model) - } else { - // Use OpenAI SDK for base URLs - response = (await this.embeddingsClient.embeddings.create({ + if (isFullUrl) { + response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model, signal) + } else { + response = (await this.embeddingsClient.embeddings.create( + { input: batchTexts, model: model, - // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 - // when processing numeric arrays, which breaks compatibility with models using larger dimensions. - // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. + // Base64 with local decoding bypasses the OpenAI v4.78.1 numeric-array parser's + // 256-dimension truncation, which breaks larger models. encoding_format: "base64", - })) as OpenAIEmbeddingResponse - } + }, + ...(signal ? [{ signal }] : []), + )) as OpenAIEmbeddingResponse + } - // Convert base64 embeddings to float32 arrays - const processedEmbeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding === "string") { - const buffer = Buffer.from(item.embedding, "base64") + signal?.throwIfAborted() + const processedEmbeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding === "string") { + const buffer = Buffer.from(item.embedding, "base64") - // Create Float32Array view over the buffer - const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) - return { - ...item, - embedding: Array.from(float32Array), - } + return { + ...item, + embedding: Array.from(float32Array), } - return item - }) - - // Replace the original data with processed embeddings - response.data = processedEmbeddings - - const embeddings = response.data.map((item) => item.embedding as number[]) - - return { - embeddings: embeddings, - usage: { - promptTokens: response.usage?.prompt_tokens || 0, - totalTokens: response.usage?.total_tokens || 0, - }, } - } catch (error) { - // Capture telemetry before error is reformatted - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "OpenAICompatibleEmbedder:_embedBatchWithRetries", - attempt: attempts + 1, - }) + return item + }) - const hasMoreAttempts = attempts < MAX_RETRIES - 1 - - // Check if it's a rate limit error - const httpError = error as HttpError - if (httpError?.status === 429) { - // Update global rate limit state - await this.updateGlobalRateLimitState(httpError) - - if (hasMoreAttempts) { - // Calculate delay based on global rate limit state - const baseDelay = INITIAL_DELAY_MS * Math.pow(2, attempts) - const globalDelay = await this.getGlobalRateLimitDelay() - const delayMs = Math.max(baseDelay, globalDelay) - - console.warn( - t("embeddings:rateLimitRetry", { - delayMs, - attempt: attempts + 1, - maxRetries: MAX_RETRIES, - }), - ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) - continue - } - } + response.data = processedEmbeddings - // Log the error for debugging - console.error(`OpenAI Compatible embedder error (attempt ${attempts + 1}/${MAX_RETRIES}):`, error) + const embeddings = response.data.map((item) => item.embedding as number[]) - // Format and throw the error - throw formatEmbeddingError(error, MAX_RETRIES) + return { + embeddings: embeddings, + usage: { + promptTokens: response.usage?.prompt_tokens || 0, + totalTokens: response.usage?.total_tokens || 0, + }, } + } catch (error) { + signal?.throwIfAborted() + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "OpenAICompatibleEmbedder:_embedBatch", + }) + codeIndexLog.error("OpenAI Compatible embedder error:", error) + throw formatEmbeddingError(error) } - - throw new Error(t("embeddings:failedMaxAttempts", { attempts: MAX_RETRIES })) } /** @@ -410,87 +359,4 @@ export class OpenAICompatibleEmbedder implements IEmbedder { name: "openai-compatible", } } - - /** - * Waits if there's an active global rate limit - */ - private async waitForGlobalRateLimit(): Promise { - const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire() - try { - const state = OpenAICompatibleEmbedder.globalRateLimitState - - if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { - const waitTime = state.rateLimitResetTime - Date.now() - // Silent wait - no logging to prevent flooding - release() // Release mutex before waiting - await new Promise((resolve) => setTimeout(resolve, waitTime)) - return - } - - // Reset rate limit if time has passed - if (state.isRateLimited && state.rateLimitResetTime <= Date.now()) { - state.isRateLimited = false - state.consecutiveRateLimitErrors = 0 - } - } finally { - // Only release if we haven't already - try { - release() - } catch { - // Already released - } - } - } - - /** - * Updates global rate limit state when a 429 error occurs - */ - private async updateGlobalRateLimitState(_error: HttpError): Promise { - const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire() - try { - const state = OpenAICompatibleEmbedder.globalRateLimitState - const now = Date.now() - - // Increment consecutive rate limit errors - if (now - state.lastRateLimitError < 60000) { - // Within 1 minute - state.consecutiveRateLimitErrors++ - } else { - state.consecutiveRateLimitErrors = 1 - } - - state.lastRateLimitError = now - - // Calculate exponential backoff based on consecutive errors - const baseDelay = 5000 // 5 seconds base - const maxDelay = 300000 // 5 minutes max - const exponentialDelay = Math.min(baseDelay * Math.pow(2, state.consecutiveRateLimitErrors - 1), maxDelay) - - // Set global rate limit - state.isRateLimited = true - state.rateLimitResetTime = now + exponentialDelay - - // Silent rate limit activation - no logging to prevent flooding - } finally { - release() - } - } - - /** - * Gets the current global rate limit delay - */ - private async getGlobalRateLimitDelay(): Promise { - const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire() - try { - const state = OpenAICompatibleEmbedder.globalRateLimitState - - if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { - return state.rateLimitResetTime - Date.now() - } - - return 0 - } finally { - release() - } - } } diff --git a/src/services/code-index/embedders/openai.ts b/src/services/code-index/embedders/openai.ts index 4ff8f27e47..5119873c00 100644 --- a/src/services/code-index/embedders/openai.ts +++ b/src/services/code-index/embedders/openai.ts @@ -1,24 +1,22 @@ +import { codeIndexLog } from "../code-index-log" +import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { OpenAiNativeHandler } from "../../../api/providers/openai-native" import { ApiHandlerOptions } from "../../../shared/api" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces" -import { - MAX_BATCH_TOKENS, - MAX_ITEM_TOKENS, - MAX_BATCH_RETRIES as MAX_RETRIES, - INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS, -} from "../constants" +import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" -import { withValidationErrorHandling, formatEmbeddingError, HttpError } from "../shared/validation-helpers" +import { withValidationErrorHandling, formatEmbeddingError } from "../shared/validation-helpers" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" /** - * OpenAI implementation of the embedder interface with batching and rate limiting + * OpenAI implementation of the embedder interface with batching */ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { + readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string @@ -29,10 +27,11 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { constructor(options: ApiHandlerOptions & { openAiEmbeddingModelId?: string }) { super(options) const apiKey = this.options.openAiNativeApiKey ?? "not-provided" + this.rateLimitScope = makeRateLimitScope("openai", "https://api.openai.com/v1", apiKey) // Wrap OpenAI client creation to handle invalid API key characters try { - this.embeddingsClient = new OpenAI({ apiKey }) + this.embeddingsClient = new OpenAI({ apiKey, maxRetries: 0 }) } catch (error) { // Use the error handler to transform ByteString conversion errors throw handleOpenAIError(error, "OpenAI") @@ -42,12 +41,13 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { } /** - * Creates embeddings for the given texts with batching and rate limiting + * Creates embeddings for the given texts with batching * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -61,7 +61,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -80,6 +80,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -89,7 +90,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -115,7 +116,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -125,65 +126,39 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { return { embeddings: allEmbeddings, usage } } - /** - * Helper method to handle batch embedding with retries and exponential backoff - * @param batchTexts Array of texts to embed in this batch - * @param model Model identifier to use - * @returns Promise resolving to embeddings and usage statistics - */ - private async _embedBatchWithRetries( + private async _embedBatch( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { - for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { - try { - const response = await this.embeddingsClient.embeddings.create({ + try { + signal?.throwIfAborted() + const response = await this.embeddingsClient.embeddings.create( + { input: batchTexts, model: model, - }) - - return { - embeddings: response.data.map((item) => item.embedding), - usage: { - promptTokens: response.usage?.prompt_tokens || 0, - totalTokens: response.usage?.total_tokens || 0, - }, - } - } catch (error: any) { - const hasMoreAttempts = attempts < MAX_RETRIES - 1 - - // Check if it's a rate limit error - const httpError = error as HttpError - if (httpError?.status === 429 && hasMoreAttempts) { - const delayMs = INITIAL_DELAY_MS * Math.pow(2, attempts) - console.warn( - t("embeddings:rateLimitRetry", { - delayMs, - attempt: attempts + 1, - maxRetries: MAX_RETRIES, - }), - ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) - continue - } - - // Capture telemetry before reformatting the error - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "OpenAiEmbedder:_embedBatchWithRetries", - attempt: attempts + 1, - }) - - // Log the error for debugging - console.error(`OpenAI embedder error (attempt ${attempts + 1}/${MAX_RETRIES}):`, error) - - // Format and throw the error - throw formatEmbeddingError(error, MAX_RETRIES) + }, + signal ? { signal } : undefined, + ) + + signal?.throwIfAborted() + return { + embeddings: response.data.map((item) => item.embedding), + usage: { + promptTokens: response.usage?.prompt_tokens || 0, + totalTokens: response.usage?.total_tokens || 0, + }, } + } catch (error) { + signal?.throwIfAborted() + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "OpenAiEmbedder:_embedBatch", + }) + codeIndexLog.error("OpenAI embedder error:", error) + throw formatEmbeddingError(error) } - - throw new Error(t("embeddings:failedMaxAttempts", { attempts: MAX_RETRIES })) } /** diff --git a/src/services/code-index/embedders/openrouter.ts b/src/services/code-index/embedders/openrouter.ts index 258ccdb4eb..a2b7d0f0f8 100644 --- a/src/services/code-index/embedders/openrouter.ts +++ b/src/services/code-index/embedders/openrouter.ts @@ -1,17 +1,13 @@ +import { codeIndexLog } from "../code-index-log" +import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" -import { - MAX_BATCH_TOKENS, - MAX_ITEM_TOKENS, - MAX_BATCH_RETRIES as MAX_RETRIES, - INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS, -} from "../constants" +import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId, getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" -import { withValidationErrorHandling, HttpError, formatEmbeddingError } from "../shared/validation-helpers" +import { withValidationErrorHandling, formatEmbeddingError } from "../shared/validation-helpers" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" -import { Mutex } from "async-mutex" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" // Default provider name when no specific provider is selected @@ -19,7 +15,6 @@ export const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]" interface EmbeddingItem { embedding: string | number[] - [key: string]: any } interface OpenRouterEmbeddingResponse { @@ -31,27 +26,18 @@ interface OpenRouterEmbeddingResponse { } /** - * OpenRouter implementation of the embedder interface with batching and rate limiting. + * OpenRouter implementation of the embedder interface with batching. * OpenRouter provides an OpenAI-compatible API that gives access to hundreds of models * through a single endpoint, automatically handling fallbacks and cost optimization. */ export class OpenRouterEmbedder implements IEmbedder { + readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string private readonly maxItemTokens: number private readonly baseUrl: string = "https://openrouter.ai/api/v1" private readonly specificProvider?: string - // Global rate limiting state shared across all instances - private static globalRateLimitState = { - isRateLimited: false, - rateLimitResetTime: 0, - consecutiveRateLimitErrors: 0, - lastRateLimitError: 0, - // Mutex to ensure thread-safe access to rate limit state - mutex: new Mutex(), - } - /** * Creates a new OpenRouter embedder * @param apiKey The API key for authentication @@ -64,6 +50,8 @@ export class OpenRouterEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } + this.rateLimitScope = makeRateLimitScope("openrouter", this.baseUrl, apiKey) + // Only set specificProvider if it's not the default value this.specificProvider = specificProvider && specificProvider !== OPENROUTER_DEFAULT_PROVIDER_NAME ? specificProvider : undefined @@ -72,6 +60,7 @@ export class OpenRouterEmbedder implements IEmbedder { try { this.embeddingsClient = new OpenAI({ baseURL: this.baseUrl, + maxRetries: 0, apiKey: apiKey, defaultHeaders: { "HTTP-Referer": "https://github.com/Zoo-Code-Org/Zoo-Code", @@ -88,12 +77,13 @@ export class OpenRouterEmbedder implements IEmbedder { } /** - * Creates embeddings for the given texts with batching and rate limiting + * Creates embeddings for the given texts with batching * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -107,7 +97,7 @@ export class OpenRouterEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - console.warn( + codeIndexLog.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -126,6 +116,7 @@ export class OpenRouterEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -135,7 +126,7 @@ export class OpenRouterEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > this.maxItemTokens) { - console.warn( + codeIndexLog.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -161,7 +152,7 @@ export class OpenRouterEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -171,116 +162,72 @@ export class OpenRouterEmbedder implements IEmbedder { return { embeddings: allEmbeddings, usage } } - /** - * Helper method to handle batch embedding with retries and exponential backoff - * @param batchTexts Array of texts to embed in this batch - * @param model Model identifier to use - * @returns Promise resolving to embeddings and usage statistics - */ - private async _embedBatchWithRetries( + private async _embedBatch( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { - for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { - // Check global rate limit before attempting request - await this.waitForGlobalRateLimit() - - try { - // Build the request parameters - const requestParams: any = { - input: batchTexts, - model: model, - // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 - // when processing numeric arrays, which breaks compatibility with models using larger dimensions. - // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. - encoding_format: "base64", - } + try { + signal?.throwIfAborted() + const requestParams: OpenAI.EmbeddingCreateParams & { + provider?: { order: string[]; only: string[]; allow_fallbacks: boolean } + } = { + input: batchTexts, + model: model, + // Base64 with local decoding bypasses the OpenAI v4.78.1 numeric-array parser's + // 256-dimension truncation, which breaks larger models. + encoding_format: "base64", + } - // Add provider routing if a specific provider is set - if (this.specificProvider) { - requestParams.provider = { - order: [this.specificProvider], - only: [this.specificProvider], - allow_fallbacks: false, - } + if (this.specificProvider) { + requestParams.provider = { + order: [this.specificProvider], + only: [this.specificProvider], + allow_fallbacks: false, } + } - const response = (await this.embeddingsClient.embeddings.create( - requestParams, - )) as OpenRouterEmbeddingResponse + const response = (await this.embeddingsClient.embeddings.create( + requestParams, + ...(signal ? [{ signal }] : []), + )) as OpenRouterEmbeddingResponse - // Convert base64 embeddings to float32 arrays - const processedEmbeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding === "string") { - const buffer = Buffer.from(item.embedding, "base64") + signal?.throwIfAborted() + const processedEmbeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding === "string") { + const buffer = Buffer.from(item.embedding, "base64") - // Create Float32Array view over the buffer - const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) - return { - ...item, - embedding: Array.from(float32Array), - } + return { + ...item, + embedding: Array.from(float32Array), } - return item - }) - - // Replace the original data with processed embeddings - response.data = processedEmbeddings - - const embeddings = response.data.map((item) => item.embedding as number[]) - - return { - embeddings: embeddings, - usage: { - promptTokens: response.usage?.prompt_tokens || 0, - totalTokens: response.usage?.total_tokens || 0, - }, } - } catch (error) { - // Capture telemetry before error is reformatted - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "OpenRouterEmbedder:_embedBatchWithRetries", - attempt: attempts + 1, - }) + return item + }) - const hasMoreAttempts = attempts < MAX_RETRIES - 1 - - // Check if it's a rate limit error - const httpError = error as HttpError - if (httpError?.status === 429) { - // Update global rate limit state - await this.updateGlobalRateLimitState(httpError) - - if (hasMoreAttempts) { - // Calculate delay based on global rate limit state - const baseDelay = INITIAL_DELAY_MS * Math.pow(2, attempts) - const globalDelay = await this.getGlobalRateLimitDelay() - const delayMs = Math.max(baseDelay, globalDelay) - - console.warn( - t("embeddings:rateLimitRetry", { - delayMs, - attempt: attempts + 1, - maxRetries: MAX_RETRIES, - }), - ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) - continue - } - } + response.data = processedEmbeddings - // Log the error for debugging - console.error(`OpenRouter embedder error (attempt ${attempts + 1}/${MAX_RETRIES}):`, error) + const embeddings = response.data.map((item) => item.embedding as number[]) - // Format and throw the error - throw formatEmbeddingError(error, MAX_RETRIES) + return { + embeddings: embeddings, + usage: { + promptTokens: response.usage?.prompt_tokens || 0, + totalTokens: response.usage?.total_tokens || 0, + }, } + } catch (error) { + signal?.throwIfAborted() + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "OpenRouterEmbedder:_embedBatch", + }) + codeIndexLog.error("OpenRouter embedder error:", error) + throw formatEmbeddingError(error) } - - throw new Error(t("embeddings:failedMaxAttempts", { attempts: MAX_RETRIES })) } /** @@ -295,7 +242,9 @@ export class OpenRouterEmbedder implements IEmbedder { const modelToUse = this.defaultModelId // Build the request parameters - const requestParams: any = { + const requestParams: OpenAI.EmbeddingCreateParams & { + provider?: { order: string[]; only: string[]; allow_fallbacks: boolean } + } = { input: testTexts, model: modelToUse, encoding_format: "base64", @@ -343,88 +292,4 @@ export class OpenRouterEmbedder implements IEmbedder { name: "openrouter", } } - - /** - * Waits if there's an active global rate limit - */ - private async waitForGlobalRateLimit(): Promise { - const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire() - let mutexReleased = false - - try { - const state = OpenRouterEmbedder.globalRateLimitState - - if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { - const waitTime = state.rateLimitResetTime - Date.now() - // Silent wait - no logging to prevent flooding - release() - mutexReleased = true - await new Promise((resolve) => setTimeout(resolve, waitTime)) - return - } - - // Reset rate limit if time has passed - if (state.isRateLimited && state.rateLimitResetTime <= Date.now()) { - state.isRateLimited = false - state.consecutiveRateLimitErrors = 0 - } - } finally { - // Only release if we haven't already - if (!mutexReleased) { - release() - } - } - } - - /** - * Updates global rate limit state when a 429 error occurs - */ - private async updateGlobalRateLimitState(_error: HttpError): Promise { - const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire() - try { - const state = OpenRouterEmbedder.globalRateLimitState - const now = Date.now() - - // Increment consecutive rate limit errors - if (now - state.lastRateLimitError < 60000) { - // Within 1 minute - state.consecutiveRateLimitErrors++ - } else { - state.consecutiveRateLimitErrors = 1 - } - - state.lastRateLimitError = now - - // Calculate exponential backoff based on consecutive errors - const baseDelay = 5000 // 5 seconds base - const maxDelay = 300000 // 5 minutes max - const exponentialDelay = Math.min(baseDelay * Math.pow(2, state.consecutiveRateLimitErrors - 1), maxDelay) - - // Set global rate limit - state.isRateLimited = true - state.rateLimitResetTime = now + exponentialDelay - - // Silent rate limit activation - no logging to prevent flooding - } finally { - release() - } - } - - /** - * Gets the current global rate limit delay - */ - private async getGlobalRateLimitDelay(): Promise { - const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire() - try { - const state = OpenRouterEmbedder.globalRateLimitState - - if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { - return state.rateLimitResetTime - Date.now() - } - - return 0 - } finally { - release() - } - } } diff --git a/src/services/code-index/embedders/vercel-ai-gateway.ts b/src/services/code-index/embedders/vercel-ai-gateway.ts index ec0888cd8b..f2314d6c37 100644 --- a/src/services/code-index/embedders/vercel-ai-gateway.ts +++ b/src/services/code-index/embedders/vercel-ai-gateway.ts @@ -1,4 +1,5 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" +import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -22,6 +23,8 @@ import { TelemetryService } from "@roo-code/telemetry" * - mistral/mistral-embed (dimension: 1024) */ export class VercelAiGatewayEmbedder implements IEmbedder { + readonly rateLimitScope: string + private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1" private static readonly DEFAULT_MODEL = "openai/text-embedding-3-large" @@ -37,6 +40,12 @@ export class VercelAiGatewayEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } + this.rateLimitScope = makeRateLimitScope( + "vercel-ai-gateway", + VercelAiGatewayEmbedder.VERCEL_AI_GATEWAY_BASE_URL, + apiKey, + ) + // Use provided model or default this.modelId = modelId || VercelAiGatewayEmbedder.DEFAULT_MODEL @@ -55,12 +64,14 @@ export class VercelAiGatewayEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } catch (error) { + signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/file-completion.ts b/src/services/code-index/file-completion.ts new file mode 100644 index 0000000000..9f0cbb5f2f --- /dev/null +++ b/src/services/code-index/file-completion.ts @@ -0,0 +1,311 @@ +import { v5 as uuidv5 } from "uuid" +import type { CacheManager } from "./cache-manager" +import { GenerationAllocator } from "./generation-allocator" +import { IndexWriteGate } from "./index-write-gate" +import type { CodeBlock, IEmbedder, IVectorStore } from "./interfaces" +import { BATCH_SEGMENT_THRESHOLD, QDRANT_CODE_BLOCK_NAMESPACE } from "./constants" +import { FileFailureRegistry, REMOVAL_REVISION, type FileFailureActivity } from "./file-failure-registry" +import { ProviderPacerRegistry } from "./provider-pacer" +import { retryWithPolicy } from "./shared/retry-policy" +import { generateRelativeFilePath } from "./shared/get-relative-path" +import { throwIfAborted } from "./parser-errors" + +export class FileCompletion { + public registrationComplete = false + public outstanding = 0 + public failed = false + private finishing?: Promise + + constructor(private readonly finish: () => Promise) {} + + register(): void { + if (this.registrationComplete) throw new Error("File registration already complete") + this.outstanding++ + } + + async settled(success: boolean): Promise { + this.failed ||= !success + this.outstanding-- + await this.tryComplete() + } + + async registered(): Promise { + this.registrationComplete = true + await this.tryComplete() + } + + private async tryComplete(): Promise { + if (!this.registrationComplete || this.outstanding !== 0 || this.failed) return + this.finishing ??= this.finish().catch((error) => { + this.failed = true + throw error + }) + await this.finishing + } +} + +export class FileIndexWriter { + private readonly owners = new WeakMap() + constructor( + private readonly cache: CacheManager, + private readonly store: IVectorStore, + private readonly gate: IndexWriteGate, + private readonly generations: GenerationAllocator, + private readonly workspacePath: string, + private readonly failures = new FileFailureRegistry(), + private readonly pacers = new ProviderPacerRegistry(), + ) {} + + private async run(signal: AbortSignal, operation: () => Promise, files: string[] = []): Promise { + const owner = this.owners.get(signal) ?? signal + // Track revision cancellation separately without treating a late successful save as a rejection. + const outcome = await this.gate.run( + signal, + () => + this.gate + .run( + owner, + () => { + throwIfAborted(signal) + return operation() + }, + files, + ) + .then( + (value) => ({ value, error: undefined }), + (error: unknown) => ({ value: undefined, error }), + ), + files, + ) + if (outcome.error !== undefined) throw outcome.error + return outcome.value as T + } + + async retryTransient( + operation: () => Promise, + options: { + signal: AbortSignal + activity: FileFailureActivity + file: string + revision: string + rateLimitScope?: string + }, + ): Promise { + const { signal, activity, file, revision, rateLimitScope = this.store.rateLimitScope } = options + const relativePath = generateRelativeFilePath(file, this.workspacePath) + const previous = this.failures.getFailure(relativePath, revision, activity) + if (previous?.state === "exhausted") throw Object.assign(new Error(previous.reason), previous) + return await retryWithPolicy( + () => + this.pacers + .get(rateLimitScope) + .run(relativePath, signal, () => + this.run(signal, operation, activity === "upsert" || activity === "delete" ? [file] : []), + ), + { + signal, + activity, + budget: this.failures.getRetryBudget(relativePath, revision, activity), + onFailure: ({ error, attempt, willRetry, classification, ...delay }) => { + this.failures.record( + relativePath, + revision, + activity, + error instanceof Error ? error.message : String(error), + attempt, + willRetry ? "retrying" : "exhausted", + classification, + delay, + ) + }, + }, + ) + } + + private async withRevision( + filePath: string, + revision: string, + parent: AbortSignal, + operation: (signal: AbortSignal) => Promise, + owner = parent, + ): Promise { + throwIfAborted(parent) + const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + const revisionSignal = this.failures.renew(relativePath, revision) + const signal = AbortSignal.any([parent, revisionSignal]) + this.owners.set(signal, owner) + try { + this.failures.assertRetryable(relativePath, revision) + return await operation(signal) + } catch (error) { + if (!signal.aborted && !this.failures.hasRevisionFailure(relativePath, revision)) { + this.failures.record( + relativePath, + revision, + revision === REMOVAL_REVISION ? "delete" : "upsert", + error instanceof Error ? error.message : String(error), + 1, + "exhausted", + ) + } + throw error + } finally { + const failure = this.failures.get(relativePath) + if (!revisionSignal.aborted && failure?.revision === revision && failure.state === "retrying") { + this.failures.record( + relativePath, + revision, + failure.activity, + failure.reason, + failure.attempt, + "exhausted", + failure.classification, + ) + } + } + } + + async remove(filePath: string, signal: AbortSignal, owner = signal): Promise { + return this.withRevision( + filePath, + REMOVAL_REVISION, + signal, + (current) => this.removeCurrent(filePath, current), + owner, + ) + } + + private async removeCurrent(filePath: string, signal: AbortSignal): Promise { + const generation = await this.generations.allocate(signal) + await this.run(signal, () => this.cache.invalidateFile(filePath)) + await this.retryTransient(() => this.store.deletePointsByFilePath(filePath, generation), { + signal, + activity: "delete", + file: filePath, + revision: REMOVAL_REVISION, + }) + throwIfAborted(signal) + this.cache.completeRemoval(filePath) + this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath), REMOVAL_REVISION) + } + + async replace( + filePath: string, + hash: string, + blocks: CodeBlock[], + embedder: IEmbedder, + signal: AbortSignal, + batchSize = BATCH_SEGMENT_THRESHOLD, + onComplete?: (blockCount: number) => void, + owner = signal, + ): Promise { + return this.withRevision( + filePath, + hash, + signal, + (current) => this.replaceCurrent(filePath, hash, blocks, embedder, current, batchSize, onComplete), + owner, + ) + } + + private async replaceCurrent( + filePath: string, + hash: string, + blocks: CodeBlock[], + embedder: IEmbedder, + signal: AbortSignal, + batchSize: number, + onComplete?: (blockCount: number) => void, + ): Promise { + const generation = await this.generations.allocate(signal) + await this.run(signal, () => this.cache.invalidateFile(filePath)) + const retryOptions = { signal, file: filePath, revision: hash } + const selected = blocks.filter((block) => block.content.trim()) + const verify = async () => { + throwIfAborted(signal) + const counts = await this.retryTransient(() => this.store.countPointsForFile(filePath, generation), { + ...retryOptions, + activity: "verify", + }) + throwIfAborted(signal) + if (counts.current !== selected.length || counts.other !== 0) { + const reason = `Index verification failed for ${filePath}: expected ${selected.length}, current ${counts.current}, other ${counts.other}` + this.failures.record( + generateRelativeFilePath(filePath, this.workspacePath), + hash, + "verify", + reason, + 1, + "exhausted", + ) + throw new Error(reason) + } + } + const completion = new FileCompletion(async () => { + await this.retryTransient(() => this.store.deletePointsByFilePath(filePath, generation), { + ...retryOptions, + activity: "delete", + }) + await verify() + const waitedForPriorWrites = await this.gate.waitForPriorWrites(filePath, signal) + if (this.gate.needsReconciliation(filePath)) { + do { + await this.gate.waitForGrace(filePath, signal) + await verify() + } while (this.gate.graceDeadline(filePath) > Date.now()) + } else if (waitedForPriorWrites) { + await verify() + } + throwIfAborted(signal) + this.cache.completeFile(filePath, hash) + this.gate.reconciled(filePath) + this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath), hash) + onComplete?.(selected.length) + }) + const batches: CodeBlock[][] = [] + for (let offset = 0; offset < selected.length; offset += Math.max(1, batchSize)) { + completion.register() + batches.push(selected.slice(offset, offset + Math.max(1, batchSize))) + } + await completion.registered() + for (const batch of batches) { + try { + throwIfAborted(signal) + const { embeddings } = await this.retryTransient( + async () => { + const result = await embedder.createEmbeddings( + batch.map((block) => block.content.trim()), + undefined, + signal, + ) + if (result.embeddings.length !== batch.length) + throw new Error(`Incomplete embeddings for ${filePath}`) + return result + }, + { ...retryOptions, activity: "embed", rateLimitScope: embedder.rateLimitScope }, + ) + throwIfAborted(signal) + const points = batch.map((block, index) => ({ + id: uuidv5(`${block.segmentHash}:${generation}`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: embeddings[index], + payload: { + filePath: generateRelativeFilePath(filePath, this.workspacePath), + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + segmentHash: block.segmentHash, + generation, + }, + })) + await this.retryTransient(() => this.store.upsertPoints(points), { + ...retryOptions, + activity: "upsert", + }) + } catch (error) { + await completion.settled(false) + throw error + } + await completion.settled(true) + } + } +} diff --git a/src/services/code-index/file-failure-registry.ts b/src/services/code-index/file-failure-registry.ts new file mode 100644 index 0000000000..95fff45e12 --- /dev/null +++ b/src/services/code-index/file-failure-registry.ts @@ -0,0 +1,169 @@ +import type { Disposable, Event } from "vscode" +import type { IndexingErrorClassification } from "./shared/indexing-errors" +import type { RetryBudget, RetryFailure } from "./shared/retry-policy" + +export type FileFailureActivity = "read" | "parse" | "embed" | "upsert" | "delete" | "verify" +export type FileFailureState = "retrying" | "exhausted" +export const REMOVAL_REVISION = "removed" +export const UNKNOWN_REVISION = "unread" + +export interface FailedFile { + path: string + revision: string + activity: FileFailureActivity + reason: string + classification: IndexingErrorClassification + attempt: number + state: FileFailureState + delayMs?: number + delaySource?: RetryFailure["delaySource"] +} + +type Revision = { + revision: string + controller: AbortController + failures: Map + budgets: Map +} + +export class FileFailureRegistry { + private readonly entries = new Map() + private readonly revisions = new Map() + private readonly changes = new Set<() => void>() + private readonly records = new Set<(failure: FailedFile) => void>() + + readonly onDidChange: Event = (listener, thisArgument, disposables) => { + const callback = () => listener.call(thisArgument, undefined) + this.changes.add(callback) + const disposable: Disposable = { + dispose: () => { + this.changes.delete(callback) + }, + } + disposables?.push(disposable) + return disposable + } + + readonly onDidRecord: Event = (listener, thisArgument, disposables) => { + const callback = (failure: FailedFile) => listener.call(thisArgument, { ...failure }) + this.records.add(callback) + const disposable: Disposable = { + dispose: () => { + this.records.delete(callback) + }, + } + disposables?.push(disposable) + return disposable + } + + getRevision(path: string): string | undefined { + return this.revisions.get(path)?.revision + } + + get(path: string): FailedFile | undefined { + const failure = this.entries.get(path) + return failure && { ...failure } + } + + has(path: string): boolean { + return this.entries.has(path) + } + + hasRevisionFailure(path: string, revision: string): boolean { + const current = this.revisions.get(path) + return current?.revision === revision && current.failures.size > 0 + } + + assertRetryable(path: string, revision: string): void { + const current = this.revisions.get(path) + if (current?.revision !== revision) return + for (const failure of current.failures.values()) { + if (failure.state === "exhausted") throw Object.assign(new Error(failure.reason), failure) + } + } + + renew(path: string, revision: string): AbortSignal { + let current = this.revisions.get(path) + if (!current || current.revision !== revision) { + current?.controller.abort() + current = { revision, controller: new AbortController(), failures: new Map(), budgets: new Map() } + this.revisions.set(path, current) + const failure = this.entries.get(path) + if (failure) { + this.entries.set(path, { ...failure, revision, state: "exhausted" }) + this.changed() + } + } + return current.controller.signal + } + + isCurrent(path: string, revision: string): boolean { + const current = this.revisions.get(path) + return current?.revision === revision && !current.controller.signal.aborted + } + + getFailure(path: string, revision: string, activity: FileFailureActivity): FailedFile | undefined { + const current = this.revisions.get(path) + const failure = current?.revision === revision ? current.failures.get(activity) : undefined + return failure && { ...failure } + } + + getRetryBudget(path: string, revision: string, activity: FileFailureActivity): RetryBudget { + const current = this.revisions.get(path) + if (current?.revision !== revision) return { retriesUsed: 0 } + let budget = current.budgets.get(activity) + if (!budget) { + budget = { retriesUsed: 0 } + current.budgets.set(activity, budget) + } + return budget + } + + isExhausted(path: string, revision: string, activity: FileFailureActivity): boolean { + return this.getFailure(path, revision, activity)?.state === "exhausted" + } + + record( + path: string, + revision: string, + activity: FileFailureActivity, + reason: string, + attempt: number, + state: FileFailureState, + classification: IndexingErrorClassification = "permanent", + delay?: Pick, + ): void { + if (!this.isCurrent(path, revision)) return + const failure = { path, revision, activity, reason, attempt, state, classification, ...delay } + this.revisions.get(path)!.failures.set(activity, failure) + this.entries.set(path, failure) + for (const listener of this.records) listener(failure) + this.changed() + } + + clear(path: string, revision?: string): void { + if (revision !== undefined && !this.isCurrent(path, revision)) return + this.revisions.get(path)?.failures.clear() + this.revisions.get(path)?.budgets.clear() + if (this.entries.delete(path)) this.changed() + } + + resetAllBudgets(): void { + for (const current of this.revisions.values()) { + current.controller.abort() + current.controller = new AbortController() + current.failures.clear() + current.budgets.clear() + } + for (const [path, failure] of this.entries) this.entries.set(path, { ...failure, state: "exhausted" }) + this.changed() + } + + list(): FailedFile[] { + return [...this.entries.values()].map((failure) => ({ ...failure })) + } + + private changed(): void { + for (const listener of this.changes) listener() + } +} diff --git a/src/services/code-index/generation-allocator.ts b/src/services/code-index/generation-allocator.ts new file mode 100644 index 0000000000..8e7a25ed04 --- /dev/null +++ b/src/services/code-index/generation-allocator.ts @@ -0,0 +1,58 @@ +import type { CacheManager } from "./cache-manager" +import type { IVectorStore } from "./interfaces" +import { IndexWriteGate } from "./index-write-gate" +import { throwIfAborted } from "./parser-errors" + +const GENERATION_BLOCK_SIZE = 1024 + +export class GenerationAllocator { + private nextGeneration: number + private reservedThrough: number + private allocationTail: Promise = Promise.resolve() + private initialized = false + + constructor(private readonly cacheManager: CacheManager) { + this.reservedThrough = cacheManager.generationWatermark ?? 0 + this.nextGeneration = this.reservedThrough + 1 + } + + async initialize(vectorStore: IVectorStore, signal: AbortSignal, gate: IndexWriteGate): Promise { + if (this.initialized) return + throwIfAborted(signal) + if (this.cacheManager.generationWatermark === undefined) { + if (await vectorStore.hasIndexedData()) { + await gate.run(signal, () => vectorStore.clearCollection()) + } + throwIfAborted(signal) + await this.cacheManager.clearCacheFile() + } + await this.reserve(signal) + this.initialized = true + } + + private async reserve(signal?: AbortSignal): Promise { + throwIfAborted(signal) + const watermark = + Math.max(this.reservedThrough, this.cacheManager.generationWatermark ?? 0) + GENERATION_BLOCK_SIZE + if (!Number.isSafeInteger(watermark)) throw new Error("Index generation counter exhausted; clear index storage") + await this.cacheManager.reserveGenerations(watermark) + throwIfAborted(signal) + this.reservedThrough = watermark + } + + allocate(signal?: AbortSignal): Promise { + const allocation = this.allocationTail + .catch(() => {}) + .then(async () => { + throwIfAborted(signal) + if (this.nextGeneration > this.reservedThrough) await this.reserve(signal) + throwIfAborted(signal) + return this.nextGeneration++ + }) + this.allocationTail = allocation.then( + () => {}, + () => {}, + ) + return allocation + } +} diff --git a/src/services/code-index/index-write-gate.ts b/src/services/code-index/index-write-gate.ts new file mode 100644 index 0000000000..c3951d980f --- /dev/null +++ b/src/services/code-index/index-write-gate.ts @@ -0,0 +1,176 @@ +import { abortError, throwIfAborted } from "./parser-errors" +import { INDEX_STOP_LIMIT_MS } from "./constants" + +export const LATE_WRITE_GRACE_MS = 30_000 + +type IssuedWrite = { pending: Promise; files: string[]; abandoned?: boolean } + +export class IndexWriteGate { + private readonly issued = new Map>() + private readonly deadlines = new Map() + private readonly uncertainByAttempt = new WeakMap>() + private readonly owners = new WeakMap() + + private readonly stopDeadlines = new WeakMap() + private readonly observedOwners = new WeakSet() + + own(signal: AbortSignal, owner: AbortSignal): void { + this.owners.set(signal, owner) + } + + stopAt(signal: AbortSignal, expiresAt: number): void { + this.stopDeadlines.set(signal, expiresAt) + } + + graceDeadline(filePath: string): number { + return this.deadlines.get(filePath) ?? 0 + } + + recover(files: string[], startedAt = Date.now()): void { + for (const filePath of files) { + this.deadlines.set(filePath, Math.max(this.deadlines.get(filePath) ?? 0, startedAt + LATE_WRITE_GRACE_MS)) + } + } + + needsReconciliation(filePath: string): boolean { + return this.deadlines.has(filePath) + } + + reconciled(filePath: string): void { + this.deadlines.delete(filePath) + } + + hasPending(signal: AbortSignal): boolean { + return (this.issued.get(signal)?.size ?? 0) > 0 + } + + async settled(signal: AbortSignal): Promise { + while (this.hasPending(signal)) { + await Promise.allSettled([...this.issued.get(signal)!].map((write) => write.pending)) + } + } + + uncertainFiles(signal: AbortSignal): string[] { + return [...(this.uncertainByAttempt.get(signal) ?? [])] + } + + private markUncertain(signal: AbortSignal, files: string[], startedAt = Date.now()): void { + let uncertain = this.uncertainByAttempt.get(signal) + if (!uncertain) { + uncertain = new Set() + this.uncertainByAttempt.set(signal, uncertain) + } + for (const filePath of files) uncertain.add(filePath) + this.recover(files, startedAt) + } + + abandon(signal: AbortSignal, startedAt = Date.now()): void { + for (const write of this.issued.get(signal) ?? []) { + if (write.abandoned) continue + write.abandoned = true + this.markUncertain(signal, write.files, startedAt) + } + } + + async waitForPriorWrites(filePath: string, signal: AbortSignal): Promise { + const currentOwner = this.owners.get(signal) ?? signal + let waited = false + while (true) { + throwIfAborted(signal) + const pending: Promise[] = [] + let remaining = Infinity + for (const [owner, writes] of this.issued) { + if (owner === currentOwner || !owner.aborted) continue + const relevant = [...writes].filter((write) => !write.abandoned && write.files.includes(filePath)) + if (relevant.length === 0) continue + const stopDeadline = this.stopDeadlines.get(owner) + if (stopDeadline === undefined) throw new Error("Missing indexing stop deadline") + if (stopDeadline <= Date.now()) { + this.abandon(owner, stopDeadline) + continue + } + remaining = Math.min(remaining, stopDeadline - Date.now()) + pending.push(...relevant.map((write) => write.pending)) + } + if (pending.length === 0) return waited + waited = true + await new Promise((resolve, reject) => { + const finish = (error?: Error) => { + clearTimeout(timer) + signal.removeEventListener("abort", onAbort) + if (error) reject(error) + else resolve() + } + const onAbort = () => finish(abortError()) + const timer = setTimeout(() => finish(), remaining) + signal.addEventListener("abort", onAbort, { once: true }) + void Promise.allSettled(pending).then(() => finish()) + if (signal.aborted) onAbort() + }) + } + } + + async waitForGrace(filePath: string, signal: AbortSignal): Promise { + while (true) { + throwIfAborted(signal) + const remaining = (this.deadlines.get(filePath) ?? 0) - Date.now() + if (remaining <= 0) return + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer) + signal.removeEventListener("abort", onAbort) + reject(abortError()) + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + resolve() + }, remaining) + signal.addEventListener("abort", onAbort, { once: true }) + if (signal.aborted) onAbort() + }) + } + } + + async run( + signal: AbortSignal, + write: () => Result | Promise, + files: string[] = [], + ): Promise { + throwIfAborted(signal) + const owner = this.owners.get(signal) ?? signal + if (!this.observedOwners.has(owner)) { + this.observedOwners.add(owner) + const onAbort = () => { + if (!this.stopDeadlines.has(owner)) this.stopAt(owner, Date.now() + INDEX_STOP_LIMIT_MS) + } + if (owner.aborted) onAbort() + else owner.addEventListener("abort", onAbort, { once: true }) + } + let writes = this.issued.get(owner) + if (!writes) { + writes = new Set() + this.issued.set(owner, writes) + } + let issued = false + let succeeded = false + const pending = Promise.resolve().then(() => { + throwIfAborted(signal) + issued = true + return write() + }) + const record = { pending, files: [...files] } + writes.add(record) + try { + const result = await pending + succeeded = true + throwIfAborted(signal) + return result + } catch (error) { + if (issued && !succeeded) this.markUncertain(owner, files) + throw error + } finally { + writes.delete(record) + if (writes.size === 0) this.issued.delete(owner) + } + } +} diff --git a/src/services/code-index/interfaces/cache.ts b/src/services/code-index/interfaces/cache.ts index 01931a3a8b..5bfaab5b50 100644 --- a/src/services/code-index/interfaces/cache.ts +++ b/src/services/code-index/interfaces/cache.ts @@ -4,4 +4,16 @@ export interface ICacheManager { deleteHash(filePath: string): void flush(): Promise getAllHashes(): Record + isTracked(filePath: string): boolean + getOpenFiles(): Record + getTrackedPaths(): string[] + invalidateFile(filePath: string): Promise + completeFile(filePath: string, hash: string): void + completeRemoval(filePath: string): void + saveNow(): Promise + clearCacheFile(): Promise + readonly loadSucceeded: boolean + readonly loadedAt: number + readonly generationWatermark: number | undefined + reserveGenerations(watermark: number): Promise } diff --git a/src/services/code-index/interfaces/directory-scanner-factory.ts b/src/services/code-index/interfaces/directory-scanner-factory.ts index 0e55a220bb..94abdd235a 100644 --- a/src/services/code-index/interfaces/directory-scanner-factory.ts +++ b/src/services/code-index/interfaces/directory-scanner-factory.ts @@ -1,6 +1,10 @@ import type { Ignore } from "ignore" import type { CacheManager } from "../cache-manager" import type { ICodeParser, IDirectoryScanner, IEmbedder, IVectorStore } from "./index" +import type { FileFailureRegistry } from "../file-failure-registry" +import type { IndexWriteGate } from "../index-write-gate" +import type { GenerationAllocator } from "../generation-allocator" +import type { ProviderPacerRegistry } from "../provider-pacer" export interface DirectoryScannerFactoryOptions { embedder: IEmbedder @@ -8,6 +12,12 @@ export interface DirectoryScannerFactoryOptions { parser: ICodeParser cacheManager: CacheManager ignoreInstance: Ignore + failures?: FileFailureRegistry + writeGate?: IndexWriteGate + generations?: GenerationAllocator + workspacePath?: string + isCurrentAttempt?: () => boolean + providerPacers?: ProviderPacerRegistry } export interface IDirectoryScannerFactory { diff --git a/src/services/code-index/interfaces/embedder.ts b/src/services/code-index/interfaces/embedder.ts index bd148bb22d..f6a2f79a91 100644 --- a/src/services/code-index/interfaces/embedder.ts +++ b/src/services/code-index/interfaces/embedder.ts @@ -3,13 +3,15 @@ * This interface is implemented by both OpenAI and Ollama embedders. */ export interface IEmbedder { + readonly rateLimitScope: string + /** * Creates embeddings for the given texts. * @param texts Array of text strings to create embeddings for * @param model Optional model ID to use for embeddings * @returns Promise resolving to an EmbeddingResponse */ - createEmbeddings(texts: string[], model?: string): Promise + createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise /** * Validates the embedder configuration by testing connectivity and credentials. diff --git a/src/services/code-index/interfaces/file-processor.ts b/src/services/code-index/interfaces/file-processor.ts index 8ecdc518c8..81fc92902d 100644 --- a/src/services/code-index/interfaces/file-processor.ts +++ b/src/services/code-index/interfaces/file-processor.ts @@ -1,31 +1,15 @@ -import * as vscode from "vscode" -import { PointStruct } from "./vector-store" +import type * as vscode from "vscode" +import type { PointStruct } from "./vector-store" +import type { ParserFatalError } from "../parser-errors" -/** - * Interface for code file parser - */ -export interface ICodeParser { - /** - * Parses a code file into code blocks - * @param filePath Path to the file to parse - * @param options Optional parsing options - * @returns Promise resolving to array of code blocks - */ - parseFile( - filePath: string, - options?: { - minBlockLines?: number - maxBlockLines?: number - content?: string - fileHash?: string - }, - ): Promise -} +export type { ICodeParser, CodeBlock } from "../parser-types" /** * Interface for directory scanner */ export interface IDirectoryScanner { + readonly hasIncompleteFiles?: boolean + /** * Scans a directory for code blocks * @param directoryPath Path to the directory to scan @@ -44,6 +28,7 @@ export interface IDirectoryScanner { skipped: number } totalBlockCount: number + incompleteFiles?: string[] }> } @@ -54,7 +39,9 @@ export interface IFileWatcher extends vscode.Disposable { /** * Initializes the file watcher */ - initialize(): Promise + initialize(signal?: AbortSignal): Promise + readonly onParserFatal?: vscode.Event + readonly isProcessing?: boolean /** * Event emitted when a batch of files begins processing. @@ -100,18 +87,3 @@ export interface FileProcessingResult { newHash?: string pointsToUpsert?: PointStruct[] } - -/** - * Common types used across the code-index service - */ - -export interface CodeBlock { - file_path: string - identifier: string | null - type: string - start_line: number - end_line: number - content: string - fileHash: string - segmentHash: string -} diff --git a/src/services/code-index/interfaces/file-watcher-factory.ts b/src/services/code-index/interfaces/file-watcher-factory.ts index 94fc8882fc..97861fa2c0 100644 --- a/src/services/code-index/interfaces/file-watcher-factory.ts +++ b/src/services/code-index/interfaces/file-watcher-factory.ts @@ -2,7 +2,11 @@ import type * as vscode from "vscode" import type { Ignore } from "ignore" import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import type { CacheManager } from "../cache-manager" -import type { IEmbedder, IFileWatcher, IVectorStore } from "./index" +import type { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./index" +import type { FileFailureRegistry } from "../file-failure-registry" +import type { IndexWriteGate } from "../index-write-gate" +import type { GenerationAllocator } from "../generation-allocator" +import type { ProviderPacerRegistry } from "../provider-pacer" export interface FileWatcherFactoryOptions { workspacePath: string @@ -12,6 +16,12 @@ export interface FileWatcherFactoryOptions { cacheManager: CacheManager ignoreInstance: Ignore rooIgnoreController?: RooIgnoreController + parser?: ICodeParser + failures?: FileFailureRegistry + writeGate?: IndexWriteGate + generations?: GenerationAllocator + isCurrentAttempt?: () => boolean + providerPacers?: ProviderPacerRegistry } export interface IFileWatcherFactory { diff --git a/src/services/code-index/interfaces/manager.ts b/src/services/code-index/interfaces/manager.ts index cab376e1b6..dcecf8bc6d 100644 --- a/src/services/code-index/interfaces/manager.ts +++ b/src/services/code-index/interfaces/manager.ts @@ -9,3 +9,5 @@ export type EmbedderProvider = | "bedrock" | "openrouter" | "semble" + +export type { IndexingStatus as IndexProgressUpdate, ProviderWaitStatus } from "@roo-code/types" diff --git a/src/services/code-index/interfaces/vector-store.ts b/src/services/code-index/interfaces/vector-store.ts index 7946563fd5..254476d4fc 100644 --- a/src/services/code-index/interfaces/vector-store.ts +++ b/src/services/code-index/interfaces/vector-store.ts @@ -8,6 +8,8 @@ export type PointStruct = { } export interface IVectorStore { + readonly rateLimitScope: string + /** * Initializes the vector store * @returns Promise resolving to boolean indicating if a new collection was created @@ -36,16 +38,17 @@ export interface IVectorStore { ): Promise /** - * Deletes points by file path - * @param filePath Path of the file to delete points for + * Deletes legacy points and generations below the mandatory cutoff for this exact file. */ - deletePointsByFilePath(filePath: string): Promise + deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise /** - * Deletes points by multiple file paths - * @param filePaths Array of file paths to delete points for + * Applies the same generation fence to every exact file path. */ - deletePointsByMultipleFilePaths(filePaths: string[]): Promise + deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise + + /** Other includes legacy points without a generation. Both counts are exact. */ + countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> /** * Clears all points from the collection diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 26ce8de5cc..9cf3e21ff0 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -1,11 +1,10 @@ import * as vscode from "vscode" import { ContextProxy } from "../../core/config/ContextProxy" -import { VectorStoreSearchResult } from "./interfaces" -import { IndexingState } from "./interfaces/manager" +import type { IndexingState } from "./interfaces/manager" import { CodeIndexConfigManager } from "./config-manager" -import type { CodeIndexStateManager } from "./state-manager" +import { CodeIndexStateManager } from "./state-manager" import { CodeIndexServiceFactory } from "./service-factory" -import { CodeIndexSearchService } from "./search-service" +import { CodeIndexSearchService, type CodeIndexSearchResponse } from "./search-service" import { CodeIndexOrchestrator } from "./orchestrator" import { CacheManager } from "./cache-manager" import { SembleProvider } from "./semble" @@ -13,57 +12,88 @@ import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import fs from "fs/promises" import ignore from "ignore" import path from "path" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" +import { t } from "../../i18n" +import { IndexWriteGate } from "./index-write-gate" +import { GenerationAllocator } from "./generation-allocator" +import { FileFailureRegistry, type FailedFile } from "./file-failure-registry" +import { ParseWorkerClient } from "./parse-worker-client" +import { codeIndexLog } from "./code-index-log" +import { initializeCodeIndexHostLogging } from "./code-index-log-host" +import { ProviderPacerRegistry } from "./provider-pacer" +import { classifyIndexingError } from "./shared/indexing-errors" +import { formatProviderWaitMessage } from "./shared/provider-wait-message" +type FailedFileStatus = Pick +type FailedFileSummary = FailedFileStatus & { count: number } export class CodeIndexManager { - // Specialized class instances - private _configManager: CodeIndexConfigManager | undefined - private readonly _stateManager: CodeIndexStateManager - private _serviceFactory: CodeIndexServiceFactory | undefined - private _orchestrator: CodeIndexOrchestrator | undefined - private _searchService: CodeIndexSearchService | undefined - private _cacheManager: CacheManager | undefined - private _sembleProvider: SembleProvider | undefined - - // Flag to prevent race conditions during error recovery + private static readonly diagnosticOwners = new Set() + private _configManager?: CodeIndexConfigManager + private _serviceFactory?: CodeIndexServiceFactory + private _orchestrator?: CodeIndexOrchestrator + private _searchService?: CodeIndexSearchService + private _cacheManager?: CacheManager + private cacheInitialization?: Promise + private _sembleProvider?: SembleProvider + private sembleState?: CodeIndexStateManager + private sembleSubscription?: vscode.Disposable + private parser?: ParseWorkerClient private _isRecoveringFromError = false - - private readonly workspacePath: string - private readonly _folderUri: vscode.Uri - private readonly context: vscode.ExtensionContext - - /** @internal — construct only via {@link CodeIndexManagerRegistry} */ + private parserFailed = false + private parserFailureMessage = "" + private servicesStopped = false + private disabledSinceFailure = false + private readonly writeGate = new IndexWriteGate() + private generationAllocator?: GenerationAllocator + private currentAttempt = 0 + private readonly idleStop = Promise.resolve() + private clearing: Promise = Promise.resolve() + private startInFlight?: Promise + private servicesRestartPending = false + private startupFailureMessage?: string + private startupEpoch = 0 + public readonly fileFailureRegistry = new FileFailureRegistry() + public readonly providerPacerRegistry = new ProviderPacerRegistry() + private readonly progressEmitter = new vscode.EventEmitter>() + private readonly statusSubscriptions: vscode.Disposable[] = [] + + /** @internal — construct only via CodeIndexManagerRegistry. */ public constructor( - workspacePath: string, - folderUri: vscode.Uri, - context: vscode.ExtensionContext, - stateManager: CodeIndexStateManager, + private readonly workspacePath: string, + private readonly _folderUri: vscode.Uri, + private readonly context: vscode.ExtensionContext, + private readonly _stateManager: CodeIndexStateManager, ) { - this.workspacePath = workspacePath - this._folderUri = folderUri - this.context = context - this._stateManager = stateManager + initializeCodeIndexHostLogging() + CodeIndexManager.diagnosticOwners.add(this) + this.configureDiagnostics() + this.statusSubscriptions.push( + this._stateManager.onProgressUpdate(() => this.progressEmitter.fire(this.getCurrentStatus())), + this.fileFailureRegistry.onDidChange(() => this.progressEmitter.fire(this.getCurrentStatus())), + this.providerPacerRegistry.onDidChange(() => this.progressEmitter.fire(this.getCurrentStatus())), + ) } - // --- Public API --- - - /** - * Returns the workspaceState key for per-folder indexing enablement, - * keyed by the real workspace folder URI so local/remote schemes cannot collide. - */ private _workspaceEnabledKey(): string { return "codeIndexWorkspaceEnabled:" + this._folderUri.toString(true) } public get isWorkspaceEnabled(): boolean { - const explicit = this.context.workspaceState.get(this._workspaceEnabledKey(), undefined) - if (explicit !== undefined) return explicit - return this.autoEnableDefault + return ( + this.context.workspaceState.get(this._workspaceEnabledKey(), undefined) ?? + this.autoEnableDefault + ) } public async setWorkspaceEnabled(enabled: boolean): Promise { + const wasEnabled = this.isWorkspaceEnabled await this.context.workspaceState.update(this._workspaceEnabledKey(), enabled) + if (!enabled) { + this.disabledSinceFailure = true + void this.stopWatcher() + } else if (!wasEnabled) { + this.renewAfterDisable() + } + this.progressEmitter.fire(this.getCurrentStatus()) } public get autoEnableDefault(): boolean { @@ -75,28 +105,18 @@ export class CodeIndexManager { } public get onProgressUpdate() { - return this._stateManager.onProgressUpdate + return this.progressEmitter.event } - private assertInitialized() { - if (this._sembleProvider) { - // When semble is active, we don't need orchestrator/searchService - return - } + private assertInitialized(): void { + if (this._sembleProvider) return if (!this._configManager || !this._orchestrator || !this._searchService || !this._cacheManager) { throw new Error("CodeIndexManager not initialized. Call initialize() first.") } } public get state(): IndexingState { - if (!this.isFeatureEnabled) { - return "Standby" - } - if (this._sembleProvider) { - return this._sembleProvider.state - } - this.assertInitialized() - return this._orchestrator!.state + return this.getCurrentStatus().systemStatus } public get isFeatureEnabled(): boolean { @@ -107,7 +127,6 @@ export class CodeIndexManager { return this._configManager?.isFeatureConfigured ?? false } - /** Whether configuration loading has succeeded, independently of service or indexing readiness. */ public get isConfigurationLoaded(): boolean { return this._configManager?.isConfigurationLoaded ?? false } @@ -121,301 +140,411 @@ export class CodeIndexManager { } } - /** - * Initializes the manager with configuration and dependent services. - * Must be called before using any other methods. - * @returns Object indicating if a restart is needed - */ - public async initialize(contextProxy: ContextProxy): Promise<{ requiresRestart: boolean }> { - // 1. ConfigManager Initialization and Configuration Loading - if (!this._configManager) { - this._configManager = new CodeIndexConfigManager(contextProxy) + private renewAfterDisable(): void { + this.parserFailed = false + this.startupFailureMessage = undefined + this.disabledSinceFailure = false + this.fileFailureRegistry.resetAllBudgets() + } + + private async initializeCache(): Promise { + if (!this.cacheInitialization) { + const cache = (this._cacheManager ??= new CacheManager(this.context, this.workspacePath)) + this.cacheInitialization = cache + .initialize() + .then(() => { + this.writeGate.recover(Object.keys(cache.getOpenFiles()), cache.loadedAt) + this.generationAllocator = new GenerationAllocator(cache) + }) + .catch((error: unknown) => { + this.cacheInitialization = undefined + throw error + }) } - // Load configuration once to get current state and restart requirements - const { requiresRestart } = await this._configManager.loadConfiguration() + await this.cacheInitialization + } - // 2. Check if feature is enabled - if (!this.isFeatureEnabled) { - if (this._orchestrator) { - this._orchestrator.stopWatcher() - } - if (this._sembleProvider) { - this._sembleProvider.stopIndexing() - } + public async initialize(contextProxy: ContextProxy, autoStart = true): Promise<{ requiresRestart: boolean }> { + const epoch = this.startupEpoch + const configuration = (this._configManager ??= new CodeIndexConfigManager(contextProxy)) + const { requiresRestart } = await configuration.loadConfiguration() + if (configuration !== this._configManager || epoch !== this.startupEpoch) return { requiresRestart } + this.configureDiagnostics() + if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { + this.disabledSinceFailure = true + void this.stopWatcher() return { requiresRestart } } - - // 3. Check if workspace is available - const workspacePath = this.workspacePath - if (!workspacePath) { + if (!this.workspacePath) { this._stateManager.setSystemState("Standby", "No workspace folder open") return { requiresRestart } } - - // 4. Check workspace-level enablement (before creating expensive services) - if (!this.isWorkspaceEnabled) { - this._stateManager.setSystemState("Standby", "Indexing not enabled for this workspace") + if (this.disabledSinceFailure) this.renewAfterDisable() + if (autoStart && (this.parserFailed || this.startupFailureMessage)) return { requiresRestart } + if (!autoStart) { + await this.initializeCache() return { requiresRestart } } - - // 5. CacheManager Initialization - if (!this._cacheManager) { - this._cacheManager = new CacheManager(this.context, this.workspacePath) - await this._cacheManager.initialize() - } - - // 6. Determine if Core Services Need Recreation - const needsServiceRecreation = (!this._serviceFactory && !this._sembleProvider) || requiresRestart - - if (needsServiceRecreation) { - await this._recreateServices() + if (this.startInFlight) { + this.servicesRestartPending ||= requiresRestart + await this.startInFlight + } else if ((!this._serviceFactory && !this._sembleProvider) || requiresRestart || this.servicesStopped) { + await this.runStartup(async (startupEpoch) => { + await this.initializeCache() + if (startupEpoch === this.startupEpoch) await this._recreateServices() + }) } + return { requiresRestart } + } - // 7. Handle Indexing Start/Restart + /** Resolves after startup, not after the background scan. */ + public startIndexing(contextProxy?: ContextProxy): Promise { + if (this.startInFlight) return this.startInFlight if (this._sembleProvider) { - // For semble, start indexing if needed - const shouldStartIndexing = requiresRestart || needsServiceRecreation - if (shouldStartIndexing) { - await this._sembleProvider.startIndexing() - } - } else { - const shouldStartOrRestartIndexing = - requiresRestart || - (needsServiceRecreation && (!this._orchestrator || this._orchestrator.state !== "Indexing")) - - if (shouldStartOrRestartIndexing) { - this.startIndexingInBackground() - } + return this.runStartup(async (epoch) => { + await this.clearing + if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return + this.startupFailureMessage = undefined + this.servicesStopped = false + this.subscribeSembleState() + }) } - - return { requiresRestart } + if ( + this.isFeatureEnabled && + this.isWorkspaceEnabled && + this._orchestrator?.isScanning && + !this._orchestrator.stopped + ) { + return this._orchestrator.startupResult + } + return this.runStartup(async (epoch) => { + if (contextProxy) await this.initialize(contextProxy, false) + if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return + this.parserFailed = false + this.startupFailureMessage = undefined + await this.initializeCache() + if (epoch !== this.startupEpoch) return + await this._recreateServices() + if (epoch !== this.startupEpoch) return + this.fileFailureRegistry.resetAllBudgets() + this.assertInitialized() + }) } - private startIndexingInBackground(): void { - void this._orchestrator?.startIndexing().catch((error) => { - console.error("[CodeIndexManager] Background indexing failed:", error) + private runStartup(operation: (epoch: number) => Promise): Promise { + if (this.startInFlight) return this.startInFlight + const epoch = this.startupEpoch + const startup = Promise.resolve().then(async () => { + try { + if (epoch !== this.startupEpoch) return + await operation(epoch) + while ( + epoch === this.startupEpoch && + !this.parserFailed && + this.isFeatureEnabled && + this.isWorkspaceEnabled + ) { + if (this.servicesRestartPending) { + this.servicesRestartPending = false + await this._recreateServices() + continue + } + await this.startIndexingInBackground() + if (!this.servicesRestartPending) break + } + } catch (error) { + if (epoch !== this.startupEpoch) return + this.startupFailureMessage = codeIndexLog.redact(error instanceof Error ? error.message : String(error)) + this._stateManager.setSystemState("Error", this.startupFailureMessage) + codeIndexLog.error("Indexing startup failed", { error, attempt: this.currentAttempt }) + throw error + } }) + this.startInFlight = startup.finally(() => { + this.startInFlight = undefined + this.servicesRestartPending = false + }) + return this.startInFlight } - /** - * Initiates the indexing process (initial scan and starts watcher). - * Automatically recovers from error state if needed before starting. - * - * @important This method should NEVER be awaited as it starts a long-running background process. - * The indexing will continue asynchronously and progress will be reported through events. - */ - public async startIndexing(): Promise { - if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { - return - } - - // Delegate to semble provider if active + private async startIndexingInBackground(): Promise { + if (this.servicesStopped || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return if (this._sembleProvider) { await this._sembleProvider.startIndexing() return } - - // Check if we're in error state and recover if needed - const currentStatus = this.getCurrentStatus() - if (currentStatus.systemStatus === "Error") { - await this.recoverFromError() - - // After recovery, we need to reinitialize since recoverFromError clears all services - // This will be handled by the caller (webviewMessageHandler) checking isInitialized - return - } - - this.assertInitialized() - await this._orchestrator!.startIndexing() + const orchestrator = this._orchestrator + if (!orchestrator) return + const attempt = this.currentAttempt + void orchestrator.startIndexing().catch((error: unknown) => { + if (attempt !== this.currentAttempt || orchestrator.stopped) return + codeIndexLog.error("Background scan failed", { error, attempt }) + }) + await orchestrator.startupResult } - /** - * Stops any in-progress indexing operation and the file watcher. - */ - public stopIndexing(): void { - if (this._sembleProvider) { - this._sembleProvider.stopIndexing() - return - } - if (this._orchestrator) { - this._orchestrator.stopIndexing() - } + public stopIndexing(): Promise { + this.startupEpoch++ + this.servicesStopped = true + this._sembleProvider?.stopIndexing() + this.parser?.stop() + if (this._orchestrator) return this._orchestrator.stopIndexing() + this.currentAttempt++ + return this.idleStop } - /** - * Stops the file watcher and potentially cleans up resources. - */ - public stopWatcher(): void { - if (!this.isFeatureEnabled) { - return - } - if (this._orchestrator) { - this._orchestrator.stopWatcher() - } + public stopWatcher(): Promise { + this.startupEpoch++ + this.servicesStopped = true + this._sembleProvider?.stopIndexing() + this.parser?.stop() + if (this._orchestrator) return this._orchestrator.stopWatcher() + this.currentAttempt++ + return this.idleStop } - /** - * Recovers from error state by clearing the error and resetting internal state. - * This allows the manager to be re-initialized after a recoverable error. - * - * This method clears all service instances (configManager, serviceFactory, orchestrator, searchService) - * to force a complete re-initialization on the next operation. This ensures a clean slate - * after recovering from errors such as network failures or configuration issues. - * - * @remarks - * - Safe to call even when not in error state (idempotent) - * - Does not restart indexing automatically - call initialize() after recovery - * - Service instances will be recreated on next initialize() call - * - Prevents race conditions from multiple concurrent recovery attempts - */ + /** Drops services without clearing the fatal-parser latch or retained workspace data. */ public async recoverFromError(): Promise { - // Prevent race conditions from multiple rapid recovery attempts - if (this._isRecoveringFromError) { - return - } - + if (this._isRecoveringFromError) return this._isRecoveringFromError = true try { - // Clear error state + void this.stopWatcher() + this.currentAttempt++ this._stateManager.setSystemState("Standby", "") } catch (error) { - // Log error but continue with recovery - clearing service instances is more important - console.error("Failed to clear error state during recovery:", error) + codeIndexLog.error("Failed to clear error state during recovery", { error }) } finally { - // Force re-initialization by clearing service instances - // This ensures a clean slate even if state update failed this._configManager = undefined this._serviceFactory = undefined this._orchestrator = undefined this._searchService = undefined - this._sembleProvider = undefined - - // Reset the flag after recovery is complete + this.disposeSemble() this._isRecoveringFromError = false } } - /** - * Cleans up the manager instance. - */ + private subscribeSembleState(): void { + this.sembleSubscription?.dispose() + const state = this.sembleState + const epoch = this.startupEpoch + const attempt = this.currentAttempt + this.sembleSubscription = state?.onProgressUpdate(() => { + if (epoch !== this.startupEpoch || attempt !== this.currentAttempt) return + const status = state.getCurrentStatus() + this._stateManager.setSystemState(status.systemStatus, status.message) + }) + } + + private disposeSemble(): void { + this.sembleSubscription?.dispose() + this.sembleSubscription = undefined + this._sembleProvider?.dispose() + this._sembleProvider = undefined + this.sembleState?.dispose() + this.sembleState = undefined + } + public dispose(): void { - this.stopIndexing() - if (this._sembleProvider) { - this._sembleProvider.dispose() - this._sembleProvider = undefined - } + void this.stopWatcher() + this.currentAttempt++ + this.disposeSemble() + this.statusSubscriptions.forEach((subscription) => subscription.dispose()) + this.progressEmitter.dispose() this._stateManager.dispose() + CodeIndexManager.diagnosticOwners.delete(this) } - /** - * Clears all index data by stopping the watcher, clearing the Qdrant collection, - * and deleting the cache file. - */ public async clearIndexData(): Promise { - if (!this.isFeatureEnabled) { - return - } - if (this._sembleProvider) { - await this._sembleProvider.clearIndexData() - return - } + if (!this.isFeatureEnabled) return this.assertInitialized() - await this._orchestrator!.clearIndexData() - await this._cacheManager!.clearCacheFile() + this.startupEpoch++ + this.servicesStopped = true + const clearing = this._sembleProvider + ? this._sembleProvider.clearIndexData() + : this._orchestrator!.clearIndexData() + this.clearing = clearing.catch(() => {}) + await clearing } - // --- Private Helpers --- - public getCurrentStatus() { const status = this._stateManager.getCurrentStatus() + const stopped = + this.servicesStopped || !!this._orchestrator?.stopped || !this.isWorkspaceEnabled || !this.isFeatureEnabled + const failedFiles: FailedFileStatus[] = this.fileFailureRegistry.list().map((failure) => ({ + path: codeIndexLog.redact( + path.isAbsolute(failure.path) ? path.relative(this.workspacePath, failure.path) : failure.path, + ), + activity: failure.activity, + reason: codeIndexLog.redact(failure.reason), + state: stopped ? "exhausted" : failure.state, + })) + const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles[0] + const failedFileSummary: FailedFileSummary | undefined = representative + ? { ...representative, count: failedFiles.length } + : undefined + const providerWaits = this.providerPacerRegistry.providerWaits().map((wait) => ({ + ...wait, + waitingFiles: wait.waitingFiles.map((file) => + codeIndexLog.redact(path.isAbsolute(file) ? path.relative(this.workspacePath, file) : file), + ), + })) + const waiting = !stopped && this.providerPacerRegistry.hasWaitingRequests() + let systemStatus: IndexingState = status.systemStatus + let message = status.message + if (this.parserFailed || this.startupFailureMessage) { + systemStatus = "Error" + message = this.parserFailed ? this.parserFailureMessage : this.startupFailureMessage! + } else if (systemStatus !== "Error" && systemStatus !== "Stopping") { + const retrying = failedFiles.some((failure) => failure.state === "retrying") + if (stopped) { + systemStatus = "Standby" + } else if (waiting) { + systemStatus = "Indexing" + message = formatProviderWaitMessage(providerWaits) + } else if (this._orchestrator?.isScanning || retrying || systemStatus === "Indexing") { + systemStatus = "Indexing" + if (retrying) message = t("embeddings:orchestrator.indexingFilesRetrying") + } else if (this._orchestrator?.isNaturallySettledPartial) { + systemStatus = "Standby" + message = t("embeddings:orchestrator.indexingFilesFailed") + } else if (systemStatus !== "Indexed") { + systemStatus = "Standby" + } + } return { ...status, + systemStatus, + message: codeIndexLog.redact(message), + failedFiles, + failedFileSummary, + providerWaits, workspacePath: this.workspacePath, workspaceEnabled: this.isWorkspaceEnabled, autoEnableDefault: this.autoEnableDefault, } } - public async searchIndex(query: string, directoryPrefix?: string): Promise { + public async searchIndex( + query: string, + directoryPrefix?: string, + signal?: AbortSignal, + ): Promise { if (!this.isFeatureEnabled) { - return [] + const failedPaths = this.fileFailureRegistry.list().map((failure) => failure.path) + const waitingPaths = this.providerPacerRegistry.waitingPaths() + return { + results: [], + incomplete: failedPaths.length > 0 || waitingPaths.length > 0, + failedPaths, + waitingPaths, + } } if (this._sembleProvider) { - return this._sembleProvider.searchIndex(query, directoryPrefix) + return { + results: await this._sembleProvider.searchIndex(query, directoryPrefix), + incomplete: false, + failedPaths: [], + waitingPaths: [], + } } this.assertInitialized() - return this._searchService!.searchIndex(query, directoryPrefix) + return this._searchService!.searchIndex(query, directoryPrefix, signal) } - /** - * Private helper method to recreate services with current configuration. - * Used by both initialize() and handleSettingsChange(). - */ private async _recreateServices(): Promise { - // Stop watcher if it exists - if (this._orchestrator) { - this.stopWatcher() - } - // Dispose existing semble provider if switching away - if (this._sembleProvider) { - this._sembleProvider.dispose() - this._sembleProvider = undefined - } - // Clear existing services to ensure clean state + const epoch = this.startupEpoch + // Keep the clearing owner's attempt valid until its deletion has finished. + await this.clearing + if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return + const attempt = ++this.currentAttempt + const isCurrentAttempt = () => attempt === this.currentAttempt + const isCurrentStartup = () => isCurrentAttempt() && epoch === this.startupEpoch + this._stateManager.setDiagnosticContext({ source: "manager", attempt }) + void this._orchestrator?.stopWatcher() + this.parser?.stop() + this.parser = undefined this._orchestrator = undefined this._searchService = undefined - - // Branch: if provider is "semble", create SembleProvider instead of external services + this._serviceFactory = undefined + this.disposeSemble() if (this._configManager!.currentEmbedderProvider === "semble") { - this._sembleProvider = new SembleProvider(this.workspacePath, this.context, this._stateManager) - await this._sembleProvider.initialize() + const state = new CodeIndexStateManager() + this.sembleState = state + this.subscribeSembleState() + const provider = new SembleProvider(this.workspacePath, this.context, state) + this._sembleProvider = provider + await provider.initialize() + if (!isCurrentStartup()) { + if (this._sembleProvider === provider) this.disposeSemble() + else provider.dispose() + return + } + this.servicesStopped = false return } - - // (Re)Initialize service factory for external providers - this._serviceFactory = new CodeIndexServiceFactory( - this._configManager!, - this.workspacePath, - this._cacheManager!, - ) - const ignoreInstance = ignore() - const workspacePath = this.workspacePath - - if (!workspacePath) { - this._stateManager.setSystemState("Standby", "") - return - } - - // Create .gitignore instance - const ignorePath = path.join(workspacePath, ".gitignore") try { - const content = await fs.readFile(ignorePath, "utf8") - ignoreInstance.add(content) + ignoreInstance.add(await fs.readFile(path.join(this.workspacePath, ".gitignore"), "utf8")) ignoreInstance.add(".gitignore") } catch (error) { - // Should never happen: reading file failed even though it exists - console.error("Unexpected error loading .gitignore:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "_recreateServices", - }) + codeIndexLog.debug("Could not load .gitignore", { error }) } - - // Create RooIgnoreController instance - const rooIgnoreController = new RooIgnoreController(workspacePath) + if (!isCurrentStartup()) return + const rooIgnoreController = new RooIgnoreController(this.workspacePath) await rooIgnoreController.initialize() - - // (Re)Create shared service instances - const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices( + if (!isCurrentStartup()) return + const parser = new ParseWorkerClient() + this.parser = parser + const factory = new CodeIndexServiceFactory( + this._configManager!, + this.workspacePath, + this._cacheManager!, + this.writeGate, + this.generationAllocator!, + this.fileFailureRegistry, + parser, + isCurrentAttempt, + this.providerPacerRegistry, + ) + this._serviceFactory = factory + const { embedder, vectorStore, scanner, fileWatcher } = factory.createServices( this.context, this._cacheManager!, ignoreInstance, rooIgnoreController, ) - - // (Re)Initialize orchestrator + const pacer = this.providerPacerRegistry.get(embedder.rateLimitScope) + let deferredValidation = pacer.state().pausedUntil !== undefined + let validationResult: { valid: boolean; error?: string } = { valid: true } + if (!deferredValidation) { + const issueSeq = pacer.markIssued() + try { + validationResult = await factory.validateEmbedder(embedder) + } catch (error) { + if (!isCurrentStartup()) { + parser.stop() + fileWatcher.dispose() + return + } + if (classifyIndexingError(error) !== "rate-limit") { + parser.stop() + fileWatcher.dispose() + throw error + } + pacer.reportRateLimit(error, issueSeq) + deferredValidation = true + } + } + if (!isCurrentStartup()) { + parser.stop() + fileWatcher.dispose() + return + } + if (!validationResult.valid) { + parser.stop() + fileWatcher.dispose() + throw new Error(validationResult.error || "Embedder configuration validation failed") + } this._orchestrator = new CodeIndexOrchestrator( this._configManager!, this._stateManager, @@ -424,62 +553,82 @@ export class CodeIndexManager { vectorStore, scanner, fileWatcher, + parser, + this.writeGate, + this.generationAllocator!, + this.fileFailureRegistry, + isCurrentAttempt, + (_error, message) => { + if (!isCurrentAttempt()) return + this.parserFailed = true + this.parserFailureMessage = codeIndexLog.redact(message) + this.servicesStopped = true + this.progressEmitter.fire(this.getCurrentStatus()) + }, + this.providerPacerRegistry, + deferredValidation + ? async (signal) => { + const result = await pacer.run(undefined, signal, () => factory.validateEmbedder(embedder)) + if (!result.valid) throw new Error(result.error || "Embedder configuration validation failed") + } + : undefined, ) - - // (Re)Initialize search service this._searchService = new CodeIndexSearchService( this._configManager!, this._stateManager, embedder, vectorStore, + this.fileFailureRegistry, + this.providerPacerRegistry, + () => this.getCurrentStatus(), ) - - // Clear any error state after successful recreation + this.servicesStopped = false this._stateManager.setSystemState("Standby", "") } - /** - * Handle code index settings changes. - * This method should be called when code index settings are updated - * to ensure the CodeIndexConfigManager picks up the new configuration. - * If the configuration changes require a restart, the service will be restarted. - */ - public async handleSettingsChange(): Promise { - if (this._configManager) { - const { requiresRestart } = await this._configManager.loadConfiguration() - - const isFeatureEnabled = this.isFeatureEnabled - const isFeatureConfigured = this.isFeatureConfigured - - // If feature is disabled, stop the service (including any active scan) - if (!isFeatureEnabled) { - this.stopIndexing() - this._stateManager.setSystemState("Standby", "Code indexing is disabled") - return - } - - if (requiresRestart && isFeatureEnabled && isFeatureConfigured) { - try { - // Ensure cacheManager is initialized before recreating services - if (!this._cacheManager) { - this._cacheManager = new CacheManager(this.context, this.workspacePath) - await this._cacheManager.initialize() - } + private configureDiagnostics(): void { + const owners = [...CodeIndexManager.diagnosticOwners] + codeIndexLog.configure({ + workspacePaths: owners.map((manager) => manager.workspacePath), + secrets: owners.flatMap((manager) => { + const config = manager._configManager?.getConfig() + return [ + config?.openAiOptions?.openAiNativeApiKey, + config?.openAiCompatibleOptions?.apiKey, + config?.geminiOptions?.apiKey, + config?.mistralOptions?.apiKey, + config?.vercelAiGatewayOptions?.apiKey, + config?.openRouterOptions?.apiKey, + config?.qdrantApiKey, + ].filter((secret): secret is string => !!secret) + }), + }) + } - // Recreate services with new configuration - await this._recreateServices() - } catch (error) { - // Error state already set in _recreateServices - console.error("Failed to recreate services:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "handleSettingsChange", - }) - // Re-throw the error so the caller knows validation failed - throw error - } - } + public async handleSettingsChange(): Promise { + const configuration = this._configManager + if (!configuration) return + const epoch = this.startupEpoch + const { requiresRestart } = await configuration.loadConfiguration() + if (configuration !== this._configManager || epoch !== this.startupEpoch) return + this.configureDiagnostics() + if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { + this.disabledSinceFailure = true + void this.stopWatcher() + return + } + if (this.disabledSinceFailure) this.renewAfterDisable() + if (this.parserFailed) return + if (this.startInFlight) { + this.servicesRestartPending ||= requiresRestart + return this.startInFlight + } + if ((requiresRestart || this.servicesStopped || this.startupFailureMessage) && this.isFeatureConfigured) { + return this.runStartup(async (startupEpoch) => { + this.startupFailureMessage = undefined + await this.initializeCache() + if (startupEpoch === this.startupEpoch) await this._recreateServices() + }) } } } diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 445db2364e..10e5cbd148 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -1,22 +1,41 @@ import * as vscode from "vscode" import * as path from "path" -import { CodeIndexConfigManager } from "./config-manager" -import { CodeIndexStateManager, IndexingState } from "./state-manager" -import { IFileWatcher, IVectorStore, BatchProcessingSummary } from "./interfaces" -import { DirectoryScanner } from "./processors" -import { CacheManager } from "./cache-manager" +import type { CodeIndexConfigManager } from "./config-manager" +import type { CodeIndexStateManager, IndexingState } from "./state-manager" +import type { IFileWatcher, IVectorStore, IDirectoryScanner } from "./interfaces" +import type { CacheManager } from "./cache-manager" import { CodeIndexScanExecutor } from "./code-index-scan-executor" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" import { t } from "../../i18n" +import { ParserFatalError, abortError } from "./parser-errors" +import { ParseWorkerClient } from "./parse-worker-client" +import { IndexWriteGate } from "./index-write-gate" +import { GenerationAllocator } from "./generation-allocator" +import { FileFailureRegistry } from "./file-failure-registry" +import { ProviderPacerRegistry } from "./provider-pacer" +import { INDEX_STOP_LIMIT_MS } from "./constants" +import { codeIndexLog } from "./code-index-log" +import { formatProviderWaitMessage } from "./shared/provider-wait-message" -/** - * Manages the code indexing workflow, coordinating between different services and managers. - */ export class CodeIndexOrchestrator { - private _fileWatcherSubscriptions: vscode.Disposable[] = [] - private _isProcessing: boolean = false - private _abortController: AbortController | null = null + private watcherSubscriptions: vscode.Disposable[] = [] + private scanning = false + private controller?: AbortController + private scanPromise?: Promise + private stopPromise?: Promise + private readonly idleStop = Promise.resolve() + public startupResult: Promise = Promise.resolve() + private resolveStartup?: () => void + private rejectStartup?: (error: unknown) => void + private failureSubscription?: vscode.Disposable + private pacerSubscription?: vscode.Disposable + private watcherBatchPending = false + private scanPartial = false + private batchPartial = false + private naturallySettled = false + public stopped = false + private parserFailure?: ParserFatalError + private collectionIncomplete = false + private recoveryCompletion?: Promise private readonly scanExecutor: CodeIndexScanExecutor constructor( @@ -25,263 +44,380 @@ export class CodeIndexOrchestrator { workspacePath: string, private readonly cacheManager: CacheManager, private readonly vectorStore: IVectorStore, - scanner: DirectoryScanner, + private readonly scanner: IDirectoryScanner, private readonly fileWatcher: IFileWatcher, + private readonly parser: ParseWorkerClient = new ParseWorkerClient(), + private readonly writeGate = new IndexWriteGate(), + private readonly generations = new GenerationAllocator(cacheManager), + private readonly failures = new FileFailureRegistry(), + private readonly isCurrentAttempt: () => boolean = () => true, + private readonly onFatal?: (error: ParserFatalError, message: string) => void, + private readonly providerPacers = new ProviderPacerRegistry(), + private readonly beforeScan?: (signal: AbortSignal) => Promise, ) { - this.scanExecutor = new CodeIndexScanExecutor(workspacePath, scanner, vectorStore, stateManager) + this.scanExecutor = new CodeIndexScanExecutor(workspacePath, scanner, stateManager, isCurrentAttempt) } - /** - * Starts the file watcher if not already running. - */ - private async _startWatcher(): Promise { - if (!this.configManager.isFeatureConfigured) { - throw new Error("Cannot start watcher: Service not configured.") - } + private isLive(signal: AbortSignal): boolean { + return !signal.aborted && this.isCurrentAttempt() + } - this.stateManager.setSystemState("Indexing", "Initializing file watcher...") + private checkLive(signal: AbortSignal): void { + if (!this.isLive(signal)) throw abortError() + } - try { - await this.fileWatcher.initialize() + private write(signal: AbortSignal, operation: () => Result | Promise): Promise { + return this.writeGate.run(signal, () => { + this.checkLive(signal) + return operation() + }) + } - this._fileWatcherSubscriptions = [ - this.fileWatcher.onDidStartBatchProcessing((_filePaths: string[]) => {}), - this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { - if (totalInBatch > 0 && this.stateManager.state !== "Indexing") { - this.stateManager.setSystemState("Indexing", "Processing file changes...") - } - this.stateManager.reportFileQueueProgress( - processedInBatch, - totalInBatch, - currentFile ? path.basename(currentFile) : undefined, - ) - if (processedInBatch === totalInBatch) { - // Covers (N/N) and (0/0) - if (totalInBatch > 0) { - // Batch with items completed - this.stateManager.setSystemState("Indexed", "File changes processed. Index up-to-date.") - } else { - if (this.stateManager.state === "Indexing") { - // Only transition if it was "Indexing" - this.stateManager.setSystemState("Indexed", "Index up-to-date. File queue empty.") - } - } - } - }), - this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { - if (summary.batchError) { - console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) - } - }), - ] - } catch (error) { - console.error("[CodeIndexOrchestrator] Failed to start file watcher:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "_startWatcher", - }) - throw error - } + private failStop(error: ParserFatalError): void { + const signal = this.controller?.signal + if (!signal || !this.isLive(signal) || this.parserFailure) return + this.parserFailure = error + const message = t("embeddings:orchestrator.parserFailed", { + filePath: error.filePath, + category: error.category, + }) + this.stateManager.setSystemState("Error", message, "error") + if (this.isLive(signal)) this.onFatal?.(error, message) + void this.stop(true) } - /** - * Updates the status of a file in the state manager. - */ + private hasIncompleteFiles(): boolean { + return ( + this.failures.list().length > 0 || + this.providerPacers.waitingPaths().length > 0 || + this.watcherBatchPending || + this.scanPartial || + this.batchPartial || + (this.stopped && !!this.scanner.hasIncompleteFiles) || + Object.keys(this.cacheManager.getOpenFiles()).length > 0 + ) + } - /** - * Initiates the indexing process (initial scan and starts watcher). - */ - public async startIndexing(): Promise { - // Check if workspace is available first - if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) { - this.stateManager.setSystemState("Error", t("embeddings:orchestrator.indexingRequiresWorkspace")) - console.warn("[CodeIndexOrchestrator] Start rejected: No workspace folder open.") + private updateRecoveryStatus(signal: AbortSignal, source = "changed-file-batch"): void { + if ( + !this.isLive(signal) || + this.stopped || + this.stateManager.state === "Error" || + this.stateManager.state === "Stopping" + ) return + const failures = this.failures.list() + const retrying = failures.some((failure) => failure.state === "retrying") + const message = t( + retrying ? "embeddings:orchestrator.indexingFilesRetrying" : "embeddings:orchestrator.indexingFilesFailed", + ) + if (this.providerPacers.hasWaitingRequests()) { + this.stateManager.setSystemState( + "Indexing", + formatProviderWaitMessage(this.providerPacers.providerWaits()), + source, + ) + } else if (this.scanning || this.watcherBatchPending || retrying) { + this.stateManager.setSystemState("Indexing", failures.length > 0 ? message : undefined, source) + } else if (this.hasIncompleteFiles()) { + if (this.naturallySettled) this.stateManager.setSystemState("Standby", message, source) + } else if (this.collectionIncomplete) { + this.recoveryCompletion ??= this.completeRecovery(signal, source).finally(() => { + this.recoveryCompletion = undefined + }) + } else { + this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted"), source) } + } - if (!this.configManager.isFeatureConfigured) { - this.stateManager.setSystemState("Standby", "Missing configuration. Save your settings to start indexing.") - console.warn("[CodeIndexOrchestrator] Start rejected: Missing configuration.") - return + private async completeRecovery(signal: AbortSignal, source: string): Promise { + try { + await this.write(signal, async () => { + if ( + this.stopped || + this.stateManager.state === "Error" || + this.stateManager.state === "Stopping" || + this.hasIncompleteFiles() || + this.scanning + ) + return + await this.vectorStore.markIndexingComplete() + if (this.isLive(signal)) this.collectionIncomplete = false + }) + if (this.isLive(signal) && !this.collectionIncomplete) this.updateRecoveryStatus(signal, source) + } catch (error) { + if (!this.isLive(signal) || this.stopped) return + codeIndexLog.error("Recovery completion failed", { error }) + this.stateManager.setSystemState("Error", String(error), "error") + void this.stop(true) } + } - if ( - this._isProcessing || - (this.stateManager.state !== "Standby" && - this.stateManager.state !== "Error" && - this.stateManager.state !== "Indexed") - ) { - console.warn( - `[CodeIndexOrchestrator] Start rejected: Already processing or in state ${this.stateManager.state}.`, - ) - return + private async startWatcher(signal: AbortSignal): Promise { + this.checkLive(signal) + this.watcherSubscriptions = [ + this.fileWatcher.onDidStartBatchProcessing(() => { + if (!this.isLive(signal)) return + this.watcherBatchPending = true + this.naturallySettled = false + this.stateManager.setSystemState("Indexing", "Processing file changes...", "changed-file-batch") + }), + this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { + if (!this.isLive(signal)) return + this.stateManager.reportFileQueueProgress( + processedInBatch, + totalInBatch, + currentFile ? path.basename(currentFile) : undefined, + "changed-file-batch", + ) + }), + this.fileWatcher.onDidFinishBatchProcessing((summary) => { + if (!this.isLive(signal)) return + this.watcherBatchPending = false + this.naturallySettled = true + this.batchPartial = + !!summary.batchError || + summary.processedFiles.some((file) => file.status === "error" || file.status === "local_error") + // Registry-backed failures recover through the registry, not a sticky batch flag. + if (this.failures.list().length > 0) this.batchPartial = false + this.updateRecoveryStatus(signal) + }), + ] + if (this.fileWatcher.onParserFatal) { + this.watcherSubscriptions.push(this.fileWatcher.onParserFatal((error) => this.failStop(error))) } + await this.fileWatcher.initialize(signal) + this.checkLive(signal) + } - this._isProcessing = true - this._abortController = new AbortController() - const signal = this._abortController.signal - this.stateManager.setSystemState("Indexing", "Initializing services...") + public startIndexing(): Promise { + if (this.stopped || !this.isCurrentAttempt()) return Promise.resolve() + if (this.scanning) return this.scanPromise ?? Promise.resolve() + if (this.controller) return Promise.resolve() + if (!vscode.workspace.workspaceFolders?.length) { + const message = t("embeddings:orchestrator.indexingRequiresWorkspace") + this.stateManager.setSystemState("Error", message, "error") + this.startupResult = Promise.reject(new Error(message)) + void this.startupResult.catch(() => {}) + return Promise.resolve() + } + if (!this.configManager.isFeatureConfigured) { + this.stateManager.setSystemState( + "Standby", + "Missing configuration. Save your settings to start indexing.", + "start-request", + ) + return Promise.resolve() + } + this.scanning = true + this.controller = new AbortController() + this.startupResult = new Promise((resolve, reject) => { + this.resolveStartup = resolve + this.rejectStartup = reject + }) + void this.startupResult.catch(() => {}) + const signal = this.controller.signal + this.failureSubscription = this.failures.onDidChange(() => + this.updateRecoveryStatus(signal, this.scanning ? "scan" : "changed-file-batch"), + ) + this.pacerSubscription = this.providerPacers.onDidChange(() => + this.updateRecoveryStatus(signal, this.scanning ? "scan" : "changed-file-batch"), + ) + this.stateManager.setSystemState("Indexing", "Initializing services...", "start-request") + this.scanPromise = this.runScan(signal) + return this.scanPromise + } - // Only clean up collections created by this run; existing data must survive failed retries. + private async runScan(signal: AbortSignal): Promise { let clearIndexOnError = false - try { - const collectionCreated = await this.vectorStore.initialize() - - clearIndexOnError = collectionCreated - - if (collectionCreated) { - await this.cacheManager.clearCacheFile() + if (this.beforeScan) { + this.resolveStartup?.() + await this.beforeScan(signal) + this.checkLive(signal) } - - // Check if the collection already has indexed data - // If it does, we can skip the full scan and just start the watcher + clearIndexOnError = await this.write(signal, () => this.vectorStore.initialize()) + this.checkLive(signal) + if (clearIndexOnError) await this.write(signal, () => this.cacheManager.clearCacheFile()) + this.checkLive(signal) + await this.generations.initialize(this.vectorStore, signal, this.writeGate) + this.checkLive(signal) const hasExistingData = await this.vectorStore.hasIndexedData() - - if (hasExistingData && !collectionCreated) { - if (!(await this.scanExecutor.runIncrementalScan(signal))) { - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) - return - } - - await this._startWatcher() - - // Mark indexing as complete after successful incremental scan - await this.vectorStore.markIndexingComplete() - - this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) - } else { - if (!(await this.scanExecutor.runFullScan(signal))) { - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) - return - } - - await this._startWatcher() - - // Mark indexing as complete after successful full scan - await this.vectorStore.markIndexingComplete() - - this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) + this.checkLive(signal) + await this.write(signal, () => this.vectorStore.markIndexingIncomplete()) + this.checkLive(signal) + this.collectionIncomplete = true + this.resolveStartup?.() + const completed = + hasExistingData && !clearIndexOnError + ? await this.scanExecutor.runIncrementalScan(signal) + : await this.scanExecutor.runFullScan(signal) + this.checkLive(signal) + if (!completed) { + void this.stop(true) + return } - } catch (error: any) { - // Handle abort gracefully — not an error, just a user-initiated stop - if (error?.name === "AbortError" || signal.aborted) { - console.log("[CodeIndexOrchestrator] Indexing aborted by user.") - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) + this.scanPartial = this.scanExecutor.partial && this.failures.list().length === 0 + await this.startWatcher(signal) + this.checkLive(signal) + this.naturallySettled = true + } catch (error) { + this.rejectStartup?.(error) + if (!this.isLive(signal)) return + if (error instanceof ParserFatalError) { + this.failStop(error) return } - - console.error("[CodeIndexOrchestrator] Error during indexing:", error) - // Scanner/provider errors and stacks may contain local paths or other private data. - // Keep details local; send only a fixed failure category across the telemetry boundary. - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: "Indexing failed", - location: "startIndexing", - }) + if (error instanceof Error && error.name === "AbortError") { + void this.stop(true) + return + } + codeIndexLog.error("Initial scan failed", { error }) if (clearIndexOnError) { try { - await this.vectorStore.clearCollection() + await this.write(signal, () => this.vectorStore.clearCollection()) + await this.write(signal, () => this.cacheManager.clearCacheFile()) } catch (cleanupError) { - console.error("[CodeIndexOrchestrator] Failed to clean up after error:", cleanupError) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: "Index cleanup failed", - location: "startIndexing.cleanup", - }) + if (this.isLive(signal)) codeIndexLog.error("Index cleanup failed", { error: cleanupError }) } - // Indexing started but failed mid-way - clear cache to avoid cache-Qdrant mismatch - await this.cacheManager.clearCacheFile() - console.log( - "[CodeIndexOrchestrator] Indexing failed after starting. Clearing cache to avoid inconsistency.", - ) - } else { - console.log("[CodeIndexOrchestrator] Preserving existing index and cache for a retry.") } - + if (!this.isLive(signal)) return this.stateManager.setSystemState( "Error", t("embeddings:orchestrator.failedDuringInitialScan", { - errorMessage: error.message || t("embeddings:orchestrator.unknownError"), + errorMessage: error instanceof Error ? error.message : t("embeddings:orchestrator.unknownError"), }), + "error", ) - this.stopWatcher() + void this.stop(true) } finally { - this._isProcessing = false - this._abortController = null + this.resolveStartup?.() + this.scanning = false + this.updateRecoveryStatus(signal, "scan-end") + if (this.recoveryCompletion) await this.recoveryCompletion } } - /** - * Stops any in-progress indexing by aborting the scan and stopping the file watcher. - */ - public stopIndexing(): void { - if (this._abortController) { - this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStoppedPartial")) - this._abortController.abort() - this._abortController = null - } - this.stopWatcher() + public stopIndexing(): Promise { + return this.stop(false) } - /** - * Stops the file watcher and cleans up resources. - */ - public stopWatcher(): void { - this.fileWatcher.dispose() - this._fileWatcherSubscriptions.forEach((sub) => sub.dispose()) - this._fileWatcherSubscriptions = [] + public stopWatcher(): Promise { + return this.stop(true) + } - if (this.stateManager.state !== "Error" && this.stateManager.state !== "Stopping") { - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.fileWatcherStopped")) + private persistUncertain(signal?: AbortSignal): Promise { + if (!this.isCurrentAttempt()) return Promise.resolve() + const paths = signal ? this.writeGate.uncertainFiles(signal) : [] + return Promise.all(paths.map((filePath) => this.cacheManager.invalidateFile(filePath))) + .then(() => { + if (this.isCurrentAttempt()) return this.cacheManager.flush() + }) + .catch((error: unknown) => { + if (this.isCurrentAttempt()) codeIndexLog.error("Stopped index cache persistence failed", { error }) + }) + } + + private stop(force: boolean): Promise { + if (this.stopPromise) return this.stopPromise + const signal = this.controller?.signal + const active = !!signal && !signal.aborted + if (!active && !force && !this.parserFailure) return this.idleStop + const preserveError = this.stateManager.state === "Error" || !!this.parserFailure + const expiresAt = Date.now() + INDEX_STOP_LIMIT_MS + if (signal) this.writeGate.stopAt(signal, expiresAt) + this.stopPromise = Promise.resolve().then(async () => { + let timer: ReturnType | undefined + let workerStopped = false + const workerExit = this.parser.stopped.then(() => { + workerStopped = true + }) + const settled = Promise.allSettled([ + this.scanPromise, + signal ? this.writeGate.settled(signal) : undefined, + workerExit, + ]).then(async () => { + await this.persistUncertain(signal) + return false + }) + const expired = await Promise.race([ + settled, + new Promise((resolve) => { + timer = setTimeout(() => resolve(true), Math.max(0, expiresAt - Date.now())) + }), + ]) + clearTimeout(timer) + if (expired && signal) { + this.writeGate.abandon(signal, expiresAt) + // Invalidation mutates the retained cache synchronously; disk I/O cannot extend the stop deadline. + void this.persistUncertain(signal) + } + if (expired && !workerStopped) this.parser.logStopTimeout() + if (!active || !this.isCurrentAttempt() || preserveError || this.state === "Error") return + this.stateManager.setSystemState( + "Standby", + t( + expired || this.hasIncompleteFiles() + ? "embeddings:orchestrator.indexingStoppedPartial" + : "embeddings:orchestrator.indexingStopped", + ), + "stop", + ) + }) + this.stopped = true + this.naturallySettled = false + this.controller?.abort() + this.resolveStartup?.() + this.failureSubscription?.dispose() + this.failureSubscription = undefined + this.pacerSubscription?.dispose() + this.pacerSubscription = undefined + this.parser.stop() + this.fileWatcher.dispose() + this.watcherSubscriptions.forEach((subscription) => subscription.dispose()) + this.watcherSubscriptions = [] + if (active && !preserveError && this.isCurrentAttempt()) { + this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStopping"), "stop") } - this._isProcessing = false + return this.stopPromise } - /** - * Clears all index data by stopping the watcher, clearing the vector store, - * and resetting the cache file. - */ public async clearIndexData(): Promise { - this._isProcessing = true - + await this.stopWatcher() + const signal = new AbortController().signal try { - await this.stopWatcher() - - try { - if (this.configManager.isFeatureConfigured) { - await this.vectorStore.deleteCollection() - } else { - console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") - } - } catch (error: any) { - console.error("[CodeIndexOrchestrator] Failed to clear vector collection:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "clearIndexData", - }) - this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) + this.checkLive(signal) + if (this.configManager.isFeatureConfigured) { + await this.write(signal, () => this.vectorStore.deleteCollection()) } + await this.write(signal, () => this.cacheManager.clearCacheFile()) + if (this.isLive(signal)) + this.stateManager.setSystemState("Standby", "Index data cleared successfully.", "rebuild") + } catch (error) { + if (!this.isLive(signal)) return + this.stateManager.setSystemState("Error", `Failed to clear index data: ${String(error)}`, "error") + throw error + } + } - await this.cacheManager.clearCacheFile() + public get isScanning(): boolean { + return this.scanning + } - if (this.stateManager.state !== "Error") { - this.stateManager.setSystemState("Standby", "Index data cleared successfully.") - } - } finally { - this._isProcessing = false - } + public get isActive(): boolean { + return !!this.controller && this.isLive(this.controller.signal) && !this.stopped + } + + public get isNaturallySettledPartial(): boolean { + return ( + this.isActive && + this.naturallySettled && + !this.scanning && + !this.watcherBatchPending && + this.hasIncompleteFiles() + ) } - /** - * Gets the current state of the indexing system. - */ public get state(): IndexingState { return this.stateManager.state } diff --git a/src/services/code-index/parse-worker-client.ts b/src/services/code-index/parse-worker-client.ts new file mode 100644 index 0000000000..4a9d129808 --- /dev/null +++ b/src/services/code-index/parse-worker-client.ts @@ -0,0 +1,189 @@ +import { codeIndexLog } from "./code-index-log" +import { Worker } from "node:worker_threads" +import * as path from "node:path" +import { CodeParser } from "./processors/parser" +import type { CodeBlock, ICodeParser } from "./parser-types" +import type { ParseRequest, ParseReply } from "./parse-worker-protocol" +import { abortError, ParserFatalError, isParserAbort } from "./parser-errors" +import { PARSE_TIMEOUT_MS } from "./constants" + +export type { ParseRequest, ParseReply } from "./parse-worker-protocol" + +export interface ParseWorker { + postMessage(request: ParseRequest): void + on(event: "message", listener: (reply: ParseReply) => void): unknown + on(event: "error", listener: (error: Error) => void): unknown + on(event: "exit", listener: (code: number) => void): unknown + terminate(): Promise +} + +interface PendingParse { + request: ParseRequest + resolve: (blocks: CodeBlock[]) => void + reject: (error: Error) => void + signal?: AbortSignal + cancel: () => void +} + +export class ParseWorkerClient implements ICodeParser { + private worker?: ParseWorker + private queue: PendingParse[] = [] + private active?: PendingParse + private timer?: NodeJS.Timeout + private nextId = 0 + private isStopped = false + private resolveStopped!: () => void + readonly stopped = new Promise((resolve) => { + this.resolveStopped = resolve + }) + private stopStartedAt?: number + private exitObserved = false + private stopTimeoutLogged = false + private readonly parser: CodeParser + private idleFailure?: Pick + + constructor( + private readonly workerFactory: () => ParseWorker = () => + new Worker(workerPath, { workerData: { wasmDirectory: __dirname } }), + private readonly timeoutMs = PARSE_TIMEOUT_MS, + workerPath = path.join(__dirname, "workers", "codeIndexParse.js"), + ) { + this.parser = new CodeParser((filePath, content, fileHash, signal) => + this.enqueue(filePath, content, fileHash, signal), + ) + } + + parseFile( + filePath: string, + options?: { content?: string; fileHash?: string; signal?: AbortSignal }, + ): Promise { + const stopError = this.getStopError(filePath, options?.signal) + if (stopError) return Promise.reject(stopError) + return this.parser.parseFile(filePath, options) + } + + private getStopError(filePath: string, signal?: AbortSignal): Error | undefined { + if (signal?.aborted) return abortError() + if (this.idleFailure) { + const { category, message } = this.idleFailure + this.idleFailure = undefined + return new ParserFatalError(category, filePath, message) + } + if (this.isStopped) return abortError() + return undefined + } + + private enqueue(filePath: string, content: string, fileHash: string, signal?: AbortSignal): Promise { + const stopError = this.getStopError(filePath, signal) + if (stopError) return Promise.reject(stopError) + return new Promise((resolve, reject) => { + const pending: PendingParse = { + request: { id: ++this.nextId, filePath, content, fileHash }, + resolve, + reject, + signal, + cancel: () => { + if (this.active === pending) this.stop() + else { + this.queue = this.queue.filter((entry) => entry !== pending) + this.settle(pending, abortError()) + } + }, + } + signal?.addEventListener("abort", pending.cancel, { once: true }) + this.queue.push(pending) + this.dispatch() + }) + } + + private settle(pending: PendingParse, error?: Error, blocks: CodeBlock[] = []): void { + pending.signal?.removeEventListener("abort", pending.cancel) + if (error) pending.reject(error) + else pending.resolve(blocks) + } + + private dispatch(): void { + if (this.isStopped || this.active || this.queue.length === 0) return + this.active = this.queue.shift()! + try { + if (!this.worker) { + this.worker = this.workerFactory() + this.worker.on("message", (reply) => { + if (reply.diagnostic) { + const { level, message, fields } = reply.diagnostic + codeIndexLog[level](message, fields) + return + } + if (this.isStopped || reply.id !== this.active?.request.id) return + if (reply.error?.fatal || (reply.error && isParserAbort(reply.error.message))) { + this.fail("parser-abort", reply.error?.message) + return + } + clearTimeout(this.timer) + const completed = this.active + this.active = undefined + const blocks = !reply.fallback + ? reply.blocks + : this.parser.fallbackContent( + completed.request.filePath, + completed.request.content, + completed.request.fileHash, + ) + this.settle(completed, reply.error ? new Error(reply.error.message) : undefined, blocks) + this.dispatch() + }) + this.worker.on("error", (error) => this.fail("parser-abort", error.message)) + this.worker.on("exit", (code) => { + this.exitObserved = true + this.fail("parser-abort", `Parse worker exited (${code})`) + codeIndexLog.info( + `parse worker exited after ${Math.round(performance.now() - this.stopStartedAt!)} ms`, + ) + this.worker = undefined + this.resolveStopped() + }) + } + this.timer = setTimeout(() => this.fail("parse-timeout"), this.timeoutMs) + this.worker.postMessage(this.active.request) + } catch (error) { + this.fail("parser-abort", String(error)) + } + } + + private fail(category: ParserFatalError["category"], message?: string): void { + if (this.isStopped) return + const failed = this.active + this.active = undefined + if (failed) this.settle(failed, new ParserFatalError(category, failed.request.filePath, message)) + else this.idleFailure = { category, message: message ?? category } + this.stop() + } + + stop(): void { + if (this.isStopped) return + this.isStopped = true + this.stopStartedAt = performance.now() + clearTimeout(this.timer) + if (this.active) this.settle(this.active, abortError()) + this.active = undefined + for (const pending of this.queue) this.settle(pending, abortError()) + this.queue = [] + if (!this.worker) { + this.exitObserved = true + this.resolveStopped() + } else if (!this.exitObserved) { + // Termination acknowledgement alone is not evidence of physical exit. + void this.worker.terminate().catch(() => undefined) + } + } + + logStopTimeout(): void { + if (!this.isStopped || this.exitObserved || this.stopTimeoutLogged) return + this.stopTimeoutLogged = true + codeIndexLog.info("parse worker exit not observed within stop limit") + } + + dispose(): void { + this.stop() + } +} diff --git a/src/services/code-index/parse-worker-protocol.ts b/src/services/code-index/parse-worker-protocol.ts new file mode 100644 index 0000000000..aa795e9f75 --- /dev/null +++ b/src/services/code-index/parse-worker-protocol.ts @@ -0,0 +1,17 @@ +import type { CodeIndexLogLevel } from "./code-index-log" +import type { CodeBlock } from "./parser-types" + +export interface ParseRequest { + id: number + filePath: string + content: string + fileHash: string +} + +export interface ParseReply { + id: number + blocks?: CodeBlock[] + fallback?: boolean + error?: { message: string; fatal: boolean } + diagnostic?: { level: CodeIndexLogLevel; message: string; fields?: unknown } +} diff --git a/src/services/code-index/parse-worker.ts b/src/services/code-index/parse-worker.ts new file mode 100644 index 0000000000..67b73c7c24 --- /dev/null +++ b/src/services/code-index/parse-worker.ts @@ -0,0 +1,38 @@ +import { codeIndexLog } from "./code-index-log" +import { parentPort, workerData } from "node:worker_threads" +import { readFile } from "node:fs/promises" +import * as path from "node:path" +import { Parser } from "web-tree-sitter" +import { CodeParser } from "./processors/parser" +import { isParserAbort } from "./parser-errors" +import type { ParseRequest, ParseReply } from "./parse-worker-protocol" + +let fault = process.env.ROO_PARSE_WORKER_FAULT +const { wasmDirectory } = workerData as { wasmDirectory: string } +const parser = new CodeParser(undefined, wasmDirectory) +parentPort!.on("message", async (request: ParseRequest) => { + let reply: ParseReply + try { + const nextFault = fault + fault = undefined + if (nextFault === "abort") throw new Error("Aborted() (injected parse worker fault)") + if (nextFault === "wasm-hang") { + const bytes = new Uint8Array(await readFile(path.join(__dirname, "fault-loop.wasm"))) + const { instance } = await WebAssembly.instantiate(bytes) + codeIndexLog.info("parse worker entering wasm fault loop") + ;(instance.exports.loop as () => void)() + } + await Parser.init({ locateFile: (filename: string) => path.join(wasmDirectory, filename) }) + const blocks = await parser.parseFile(request.filePath, { + content: request.content, + fileHash: request.fileHash, + }) + reply = { id: request.id, blocks, fallback: parser.fallbackNeeded } + } catch (error) { + reply = { + id: request.id, + error: { message: error instanceof Error ? error.message : String(error), fatal: isParserAbort(error) }, + } + } + parentPort!.postMessage(reply) +}) diff --git a/src/services/code-index/parser-errors.ts b/src/services/code-index/parser-errors.ts new file mode 100644 index 0000000000..74d454242e --- /dev/null +++ b/src/services/code-index/parser-errors.ts @@ -0,0 +1,22 @@ +export class ParserFatalError extends Error { + constructor( + public readonly category: "parser-abort" | "parse-timeout", + public readonly filePath: string, + message?: string, + ) { + super(message ?? category) + this.name = "ParserFatalError" + } +} + +export function abortError(): DOMException { + return new DOMException("Indexing aborted", "AbortError") +} + +export function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw abortError() +} + +export function isParserAbort(error: unknown): boolean { + return /Aborted\(|memory access out of bounds/i.test(error instanceof Error ? error.message : String(error)) +} diff --git a/src/services/code-index/parser-types.ts b/src/services/code-index/parser-types.ts new file mode 100644 index 0000000000..25c5062b25 --- /dev/null +++ b/src/services/code-index/parser-types.ts @@ -0,0 +1,25 @@ +export interface ICodeParser { + readonly fallbackNeeded?: boolean + fallbackContent?(filePath: string, content: string, fileHash: string): CodeBlock[] + parseFile( + filePath: string, + options?: { + minBlockLines?: number + maxBlockLines?: number + content?: string + fileHash?: string + signal?: AbortSignal + }, + ): Promise +} + +export interface CodeBlock { + file_path: string + identifier: string | null + type: string + start_line: number + end_line: number + content: string + fileHash: string + segmentHash: string +} diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 736cca0806..2bc050354e 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -1,369 +1,724 @@ -// npx vitest services/code-index/processors/__tests__/file-watcher.spec.ts - +import * as path from "path" import * as vscode from "vscode" - +import { createHash } from "crypto" +import ignore from "ignore" import { FileWatcher } from "../file-watcher" -import { FilePreparation } from "../file-preparation" - -import { clearAllMocks } from "../../../../test-utils/reset" - -// Mock TelemetryService -vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), +import type { CacheManager } from "../../cache-manager" +import type { + BatchProcessingSummary, + CodeBlock, + ICodeParser, + IEmbedder, + IVectorStore, + PointStruct, +} from "../../interfaces" +import { FileFailureRegistry } from "../../file-failure-registry" +import { GenerationAllocator } from "../../generation-allocator" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" +import { ParserFatalError } from "../../parser-errors" +import { codeIndexLog } from "../../code-index-log" +import { generateRelativeFilePath } from "../../shared/get-relative-path" +import { MAX_FILE_SIZE_BYTES } from "../../constants" +import { RooIgnoreController } from "../../../../core/ignore/RooIgnoreController" +import { makeExtensionContext, makeUri } from "../../../../test-utils/vscode" +import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" + +vi.mock("vscode", async () => { + const { makeEventEmitter, makeUri } = await import("../../../../test-utils/vscode") + return { + workspace: { fs: { readFile: vi.fn(), stat: vi.fn() }, createFileSystemWatcher: vi.fn() }, + Uri: { file: makeUri }, + RelativePattern: class { + constructor( + public base: string, + public pattern: string, + ) {} + }, + EventEmitter: class { + constructor() { + return makeEventEmitter() + } }, + } +}) +vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: class { + validateAccess = vi.fn(() => true) }, })) - -// Mock dependencies -vi.mock("../../cache-manager") -vi.mock("../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: vi.fn().mockImplementation(function () { - return { - validateAccess: vi.fn().mockReturnValue(true), - } - }), -})) -vi.mock("ignore") -vi.mock("../parser", () => ({ - codeParser: { - parseFile: vi.fn().mockImplementation(async (filePath: string) => [ - { - file_path: filePath, - content: "test content", - start_line: 1, - end_line: 1, - }, - ]), +vi.mock("../../code-index-log", () => ({ + codeIndexLog: { + redact: (message: string) => message, + createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), }, })) -const createMockEventEmitter = () => { - const listeners = new Set<(event: any) => void>() +const workspacePath = path.resolve("watcher-workspace") +const fileA = path.join(workspacePath, "a.ts") +const fileB = path.join(workspacePath, "b.ts") +const hash = (content: string) => createHash("sha256").update(content).digest("hex") +const watchers: FileWatcher[] = [] +let handlers: Record<"create" | "change" | "delete", (uri: vscode.Uri) => unknown> +const nativeDispose = vi.fn() + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} - return { - event: vi.fn((listener: (event: any) => void) => { - listeners.add(listener) - return { - dispose: () => listeners.delete(listener), - } +function blocks(filePath: string, count = 1): CodeBlock[] { + return Array.from({ length: count }, (_, blockNumber) => ({ + file_path: filePath, + content: `function block${blockNumber}() { return ${blockNumber} }`, + start_line: blockNumber + 1, + end_line: blockNumber + 1, + identifier: null, + type: "function", + fileHash: hash("current content"), + segmentHash: `${filePath}:${blockNumber}`, + })) +} + +function setup() { + const hashes = new Map() + const openFiles = new Set() + const points = new Map() + const cache = { + generationWatermark: 1024, + reserveGenerations: vi.fn().mockResolvedValue(undefined), + getHash: vi.fn((filePath: string) => hashes.get(filePath)), + isTracked: vi.fn((filePath: string) => hashes.has(filePath) || openFiles.has(filePath)), + getTrackedPaths: vi.fn(() => [...new Set([...hashes.keys(), ...openFiles])]), + invalidateFile: vi.fn(async (filePath: string) => { + hashes.delete(filePath) + openFiles.add(filePath) }), - fire: vi.fn((event: any) => { - for (const listener of listeners) { - listener(event) - } + completeFile: vi.fn((filePath: string, revision: string) => { + hashes.set(filePath, revision) + openFiles.delete(filePath) + }), + completeRemoval: vi.fn((filePath: string) => { + hashes.delete(filePath) + openFiles.delete(filePath) }), - dispose: vi.fn(() => { - listeners.clear() + } + const store = { + rateLimitScope: "qdrant:watcher-test", + initialize: vi.fn().mockResolvedValue(false), + upsertPoints: vi.fn().mockImplementation(async (batch) => { + for (const point of batch) points.set(point.id, point) }), + deletePointsByFilePath: vi + .fn() + .mockImplementation(async (filePath, cutoff) => { + for (const point of points.values()) { + const payload: Record = point.payload + if ( + payload.filePath === generateRelativeFilePath(filePath, workspacePath) && + (typeof payload.generation !== "number" || payload.generation < cutoff) + ) + points.delete(point.id) + } + }), + deletePointsByMultipleFilePaths: vi + .fn() + .mockResolvedValue(undefined), + countPointsForFile: vi + .fn() + .mockImplementation(async (filePath, generation) => { + const selected = [...points.values()] + .map((point): Record => point.payload) + .filter((payload) => payload.filePath === generateRelativeFilePath(filePath, workspacePath)) + return { + current: selected.filter((payload) => payload.generation === generation).length, + other: selected.filter((payload) => payload.generation !== generation).length, + } + }), + search: vi.fn().mockResolvedValue([]), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), + collectionExists: vi.fn().mockResolvedValue(true), + hasIndexedData: vi.fn().mockResolvedValue(true), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + } satisfies IVectorStore + const embedder = { + rateLimitScope: "openai:watcher-test", + createEmbeddings: vi + .fn() + .mockImplementation(async (texts) => ({ embeddings: texts.map(() => [1]) })), + validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), + embedderInfo: { name: "openai" }, + } satisfies IEmbedder + const parser = { + parseFile: vi.fn().mockImplementation(async (filePath) => blocks(filePath)), + } + const failures = new FileFailureRegistry() + const gate = new IndexWriteGate() + // The double supplies only the cache lookup, generation reservation, and completion methods used by watcher/writer. + const cacheManager = cache as unknown as CacheManager + const generations = new GenerationAllocator(cacheManager) + const ignored = ignore() + const access = new RooIgnoreController(workspacePath) + let currentAttempt = true + const createWatcher = () => { + const created = new FileWatcher( + workspacePath, + makeExtensionContext(), + cacheManager, + embedder, + store, + ignored, + access, + 1, + parser, + gate, + generations, + failures, + () => currentAttempt, + ) + watchers.push(created) + return created + } + return { + access, + cache, + hashes, + openFiles, + points, + store, + embedder, + parser, + failures, + gate, + ignored, + createWatcher, + watcher: createWatcher(), + supersede: () => { + currentAttempt = false + }, } } -// Mock vscode module -vi.mock("vscode", () => ({ - workspace: { - createFileSystemWatcher: vi.fn(), - workspaceFolders: [ - { - uri: { - fsPath: "/mock/workspace", - }, - }, - ], - fs: { - stat: vi.fn().mockResolvedValue({ size: 1000 }), - readFile: vi.fn().mockResolvedValue(Buffer.from("test content")), +beforeEach(() => { + clearAllMocks() + vi.useFakeTimers() + handlers = { create: () => {}, change: () => {}, delete: () => {} } + vi.mocked(vscode.workspace.fs.readFile).mockReset().mockResolvedValue(Buffer.from("current content")) + vi.mocked(vscode.workspace.fs.stat).mockReset().mockResolvedValue({ size: 100, type: 1, ctime: 0, mtime: 0 }) + vi.mocked(vscode.workspace.createFileSystemWatcher).mockImplementation(() => ({ + ignoreCreateEvents: false, + ignoreChangeEvents: false, + ignoreDeleteEvents: false, + onDidCreate: (handler) => { + handlers.create = handler + return { dispose() {} } }, - }, - RelativePattern: vi.fn().mockImplementation(function (base, pattern) { - return { base, pattern } - }), - Uri: { - file: vi.fn().mockImplementation((path) => ({ fsPath: path })), - }, - EventEmitter: vi.fn().mockImplementation(function () { - return createMockEventEmitter() - }), - ExtensionContext: vi.fn(), -})) + onDidChange: (handler) => { + handlers.change = handler + return { dispose() {} } + }, + onDidDelete: (handler) => { + handlers.delete = handler + return { dispose() {} } + }, + dispose: nativeDispose, + })) +}) +afterEach(() => { + for (const watcher of watchers.splice(0)) watcher.dispose() + restoreGlobals() + vi.useRealTimers() +}) +async function queue(paths = [fileA], type: "create" | "change" | "delete" = "change") { + for (const filePath of paths) handlers[type](makeUri(filePath)) + await vi.advanceTimersByTimeAsync(500) +} +async function drain() { + for (let turn = 0; turn < 150; turn++) await Promise.resolve() +} -describe("FileWatcher", () => { - let fileWatcher: FileWatcher - let mockWatcher: any - let mockOnDidCreate: any - let mockOnDidChange: any - let mockOnDidDelete: any - let mockContext: any - let mockCacheManager: any - let mockEmbedder: any - let mockVectorStore: any - let mockIgnoreInstance: any +describe("FileWatcher reliability", () => { + it("completes public processFile only after every batch, cleanup and exact verification", async () => { + const { watcher, parser, embedder, cache, store, hashes } = setup() + const held = deferred>>() + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + embedder.createEmbeddings.mockImplementationOnce(() => held.promise) + const pending = watcher.processFile(fileA) + await drain() + expect(watcher.isProcessing).toBe(true) + expect(cache.invalidateFile).toHaveBeenCalledOnce() + expect(hashes.has(fileA)).toBe(false) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + held.resolve({ embeddings: [[1]] }) + expect((await pending).status).toBe("success") + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(store.countPointsForFile).toHaveBeenCalledOnce() + expect(hashes.get(fileA)).toBe(hash("current content")) + expect(watcher.isProcessing).toBe(false) + }) - const waitForNextBatch = () => - new Promise((resolve) => { - const disposable = fileWatcher.onDidFinishBatchProcessing((summary) => { - disposable.dispose() - resolve(summary) - }) + it("sets processing before start listeners and stops dispatch when a listener disposes", async () => { + const { watcher, parser } = setup() + await watcher.initialize() + watcher.onDidStartBatchProcessing(() => { + expect(watcher.isProcessing).toBe(true) + watcher.dispose() }) - - const flushBatch = async () => { - await vi.advanceTimersByTimeAsync(500) - } - - beforeEach(() => { - // Reset all mocks - clearAllMocks() - vi.useFakeTimers() - - // Create mock event handlers - mockOnDidCreate = vi.fn() - mockOnDidChange = vi.fn() - mockOnDidDelete = vi.fn() - - // Create mock watcher - mockWatcher = { - onDidCreate: vi.fn().mockImplementation((handler) => { - mockOnDidCreate = handler - return { dispose: vi.fn() } - }), - onDidChange: vi.fn().mockImplementation((handler) => { - mockOnDidChange = handler - return { dispose: vi.fn() } - }), - onDidDelete: vi.fn().mockImplementation((handler) => { - mockOnDidDelete = handler - return { dispose: vi.fn() } - }), - dispose: vi.fn(), - } - - // Mock createFileSystemWatcher to return our mock watcher - vi.mocked(vscode.workspace.createFileSystemWatcher).mockReturnValue(mockWatcher) - - // Create mock dependencies - mockContext = { - subscriptions: [], - } - - mockCacheManager = { - getHash: vi.fn(), - updateHash: vi.fn(), - deleteHash: vi.fn(), - } - - mockEmbedder = { - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), - } - - mockVectorStore = { - upsertPoints: vi.fn().mockResolvedValue(undefined), - deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), - deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), - } - - mockIgnoreInstance = { - ignores: vi.fn().mockReturnValue(false), - } - - fileWatcher = new FileWatcher( - "/mock/workspace", - mockContext, - mockCacheManager, - mockEmbedder, - mockVectorStore, - mockIgnoreInstance, - ) + await queue() + expect(parser.parseFile).not.toHaveBeenCalled() + expect(nativeDispose).toHaveBeenCalledOnce() }) - afterEach(async () => { - fileWatcher?.dispose() - await vi.runOnlyPendingTimersAsync() - vi.useRealTimers() + it("initializes once and rejects pre-aborted, disposed and stale work before shared mutations", async () => { + const { watcher, failures, supersede } = setup() + await watcher.initialize() + await watcher.initialize() + expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledOnce() + supersede() + await expect(watcher.processFile(fileA)).rejects.toMatchObject({ name: "AbortError" }) + expect(failures.getRevision("a.ts")).toBeUndefined() + const other = setup().watcher + await expect(other.initialize(AbortSignal.abort())).rejects.toMatchObject({ name: "AbortError" }) + other.dispose() + await expect(other.initialize()).rejects.toMatchObject({ name: "AbortError" }) + expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledOnce() }) - it("reuses constructor-created preparation for public processFile without writing points or cache", async () => { - const prepare = vi.spyOn(FilePreparation.prototype, "prepareFile") - try { - const preparation = fileWatcher["filePreparation"] - expect(preparation).toBeInstanceOf(FilePreparation) - expect(preparation["dependencies"].fileSystem).toBe(vscode.workspace.fs) - expect(preparation["dependencies"].cacheManager).toBe(mockCacheManager) - expect(preparation["dependencies"].ignoreController).toBe(fileWatcher["ignoreController"]) - const path = "/mock/workspace/src/file.ts" - const result = await fileWatcher.processFile(path) - expect(prepare).toHaveBeenNthCalledWith(1, path) - expect(result).toBe(await prepare.mock.results[0].value) - expect(result.status).toBe("processed_for_batching") - expect(result.pointsToUpsert).toHaveLength(1) - expect(vscode.workspace.fs.stat).toHaveBeenCalledWith(vscode.Uri.file(path)) - expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(vscode.Uri.file(path)) - const secondPath = "/mock/workspace/src/second.ts" - const secondResult = await fileWatcher.processFile(secondPath) - expect(prepare).toHaveBeenCalledTimes(2) - expect(prepare).toHaveBeenNthCalledWith(2, secondPath) - expect(prepare.mock.contexts[0]).toBe(preparation) - expect(prepare.mock.contexts[1]).toBe(preparation) - expect(secondResult).toBe(await prepare.mock.results[1].value) - expect(secondResult.status).toBe("processed_for_batching") - expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() - expect(mockCacheManager.updateHash).not.toHaveBeenCalled() - expect(mockCacheManager.deleteHash).not.toHaveBeenCalled() - } finally { - prepare.mockRestore() - } + it.each(["read", "parse"] as const)( + "records %s once without automatic retry and continues unrelated files", + async (activity) => { + const { watcher, parser, failures, hashes } = setup() + if (activity === "read") + vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(new Error("read failed")) + else parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) + const finished = vi.fn() + watcher.onDidFinishBatchProcessing(finished) + await watcher.initialize() + await queue([fileA, fileB]) + await vi.advanceTimersByTimeAsync(60_000) + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity, attempt: 1, state: "exhausted" }), + ]) + expect(hashes.has(fileB)).toBe(true) + expect(finished).toHaveBeenCalledOnce() + expect(vscode.workspace.fs.readFile).toHaveBeenCalledTimes(2) + const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value + expect(batch.record).toHaveBeenCalledOnce() + expect(batch.flush).toHaveBeenCalledOnce() + }, + ) + + it("does not reparse an exhausted revision until content changes or explicit retry resets it", async () => { + const { watcher, parser, failures } = setup() + parser.parseFile.mockRejectedValue(new Error("bad parse")) + await watcher.processFile(fileA) + await watcher.processFile(fileA) + expect(parser.parseFile).toHaveBeenCalledOnce() + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("changed")) + await watcher.processFile(fileA) + expect(parser.parseFile).toHaveBeenCalledTimes(2) + failures.resetAllBudgets() + parser.parseFile.mockResolvedValue(blocks(fileA)) + expect((await watcher.processFile(fileA)).status).toBe("success") + expect(failures.list()).toEqual([]) }) - describe("file filtering", () => { - it("should ignore files in hidden directories on create events", async () => { - // Initialize the file watcher - await fileWatcher.initialize() - - const batchPromise = waitForNextBatch() - - // Simulate file creation events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.git/config", shouldProcess: false }, - { path: "/mock/workspace/.hidden/file.ts", shouldProcess: false }, - { path: "/mock/workspace/src/.next/static/file.js", shouldProcess: false }, - { path: "/mock/workspace/node_modules/package/index.js", shouldProcess: false }, - { path: "/mock/workspace/normal/file.js", shouldProcess: true }, - ] - - // Trigger file creation events - for (const { path } of testCases) { - await mockOnDidCreate({ fsPath: path }) - } - - await flushBatch() - - const batchSummary = await batchPromise - const successPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "success") - .map((result: any) => result.path) - const skippedPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "skipped") - .map((result: any) => result.path) - - // Check that files in hidden directories were not processed - expect(successPaths).toContain("/mock/workspace/src/file.ts") - expect(successPaths).toContain("/mock/workspace/normal/file.js") - expect(skippedPaths).toContain("/mock/workspace/.git/config") - expect(skippedPaths).toContain("/mock/workspace/.hidden/file.ts") - expect(skippedPaths).toContain("/mock/workspace/src/.next/static/file.js") - expect(skippedPaths).toContain("/mock/workspace/node_modules/package/index.js") + it.each(["embed", "upsert"] as const)( + "shares the %s retry allowance across batches and watcher recreation", + async (activity) => { + const { watcher, createWatcher, parser, embedder, store, failures } = setup() + parser.parseFile.mockResolvedValue(blocks(fileA, 2)) + const failure = Object.assign(new Error("temporary"), { status: 503 }) + const target = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints + target.mockRejectedValueOnce(failure).mockRejectedValueOnce(failure) + if (activity === "embed") + embedder.createEmbeddings.mockResolvedValueOnce({ embeddings: [[1]] }).mockRejectedValue(failure) + else store.upsertPoints.mockResolvedValueOnce(undefined).mockRejectedValue(failure) + const pending = watcher.processFile(fileA) + await vi.advanceTimersByTimeAsync(1500) + expect((await pending).status).toBe("local_error") + expect(target).toHaveBeenCalledTimes(4) + expect(failures.getRetryBudget("a.ts", hash("current content"), activity).retriesUsed).toBe(2) + await createWatcher().processFile(fileA) + expect(target).toHaveBeenCalledTimes(4) + }, + ) + + it.each([401, 400, 408, 409])("never retries permanent HTTP %s", async (status) => { + const { watcher, embedder, failures } = setup() + embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("permanent"), { status })) + expect((await watcher.processFile(fileA)).status).toBe("local_error") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(failures.get("a.ts")).toMatchObject({ + activity: "embed", + state: "exhausted", + classification: "permanent", }) + }) - it("should ignore files in hidden directories on change events", async () => { - // Initialize the file watcher - await fileWatcher.initialize() - - const batchPromise = waitForNextBatch() - - // Simulate file change events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.vscode/settings.json", shouldProcess: false }, - { path: "/mock/workspace/src/.cache/data.json", shouldProcess: false }, - { path: "/mock/workspace/dist/bundle.js", shouldProcess: false }, - ] + it("retries a store request without reembedding and logs every grouped failure", async () => { + const { watcher, store, embedder, hashes, failures } = setup() + store.upsertPoints.mockRejectedValueOnce(Object.assign(new Error("temporary"), { status: 503 })) + await watcher.initialize() + await queue() + await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(hashes.has(fileA)).toBe(true) + expect(failures.list()).toEqual([]) + const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value + expect(batch.record).toHaveBeenCalledOnce() + expect(batch.flush).toHaveBeenCalledOnce() + }) - // Trigger file change events - for (const { path } of testCases) { - await mockOnDidChange({ fsPath: path }) + it.each(["create", "change", "delete"] as const)( + "filters untracked hidden paths on %s and processes eligible paths", + async (type) => { + const { watcher, parser, store, hashes } = setup() + await watcher.initialize() + await queue([path.join(workspacePath, ".hidden/file.ts")], type) + expect(parser.parseFile).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + if (type !== "delete") { + await queue([fileA], type) + expect(hashes.has(fileA)).toBe(true) } + }, + ) + + it.each(["delete", "ignore", "rooignore", "size", "extension"] as const)( + "removes tracked paths with a fenced generation on %s", + async (kind) => { + const { watcher, cache, hashes, store, ignored, access, parser } = setup() + const filePath = kind === "extension" ? path.join(workspacePath, "a.unsupported") : fileA + hashes.set(filePath, "previous") + if (kind === "ignore") ignored.add("a.ts") + if (kind === "rooignore") vi.mocked(access.validateAccess).mockReturnValue(false) + if (kind === "size") + vi.mocked(vscode.workspace.fs.stat).mockResolvedValue({ + size: MAX_FILE_SIZE_BYTES + 1, + type: 1, + mtime: 0, + ctime: 0, + }) + await watcher.initialize() + await queue([filePath], kind === "delete" ? "delete" : "change") + expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) + expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(filePath, expect.any(Number)) + expect(cache.completeRemoval).toHaveBeenCalledWith(filePath) + expect(parser.parseFile).not.toHaveBeenCalled() + expect(hashes.has(filePath)).toBe(false) + }, + ) + + it("removes a registry-only deleted source and retains failed removal until explicit retry", async () => { + const { watcher, failures, store } = setup() + failures.renew("a.ts", "unread") + failures.record("a.ts", "unread", "read", "read failed", 1, "exhausted") + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + await watcher.initialize() + await queue([fileA], "delete") + await queue([fileA], "delete") + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + failures.resetAllBudgets() + await queue([fileA], "delete") + expect(store.deletePointsByFilePath).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + }) - await flushBatch() - - const batchSummary = await batchPromise - const successPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "success") - .map((result: any) => result.path) - const skippedPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "skipped") - .map((result: any) => result.path) + it.each(["success", "failure", "fatal"] as const)( + "suppresses stale parse %s continuations and events", + async (outcome) => { + const { watcher, parser, supersede, cache, failures } = setup() + const held = deferred() + parser.parseFile.mockImplementationOnce(() => held.promise) + const fatal = vi.fn() + const finished = vi.fn() + watcher.onParserFatal(fatal) + watcher.onDidFinishBatchProcessing(finished) + await watcher.initialize() + const pending = watcher.processFile(fileA) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await drain() + supersede() + if (outcome === "success") held.resolve(blocks(fileA)) + else held.reject(outcome === "fatal" ? new ParserFatalError("parser-abort", fileA) : new Error("late")) + await rejected + await queue([fileB]) + expect(parser.parseFile).toHaveBeenCalledOnce() + expect(cache.invalidateFile).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + expect(fatal).not.toHaveBeenCalled() + expect(finished).not.toHaveBeenCalled() + }, + ) + + it("propagates fatal identity immediately, stops queued dispatch and preserves prior data", async () => { + const { watcher, parser, hashes, store, failures } = setup() + const failure = new ParserFatalError("parse-timeout", fileA) + hashes.set(fileB, "completed") + parser.parseFile.mockRejectedValueOnce(failure) + const fatal = vi.fn() + watcher.onParserFatal(fatal) + await expect(watcher.processFile(fileA)).rejects.toBe(failure) + expect(fatal).toHaveBeenCalledExactlyOnceWith(failure) + expect(failures.get("a.ts")).toMatchObject({ activity: "parse", attempt: 1, state: "exhausted" }) + expect(hashes.get(fileB)).toBe("completed") + expect(store.clearCollection).not.toHaveBeenCalled() + await expect(watcher.processFile(fileB)).rejects.toMatchObject({ name: "AbortError" }) + }) - // Check that files in hidden directories were not processed - expect(successPaths).toContain("/mock/workspace/src/file.ts") - expect(skippedPaths).toContain("/mock/workspace/.vscode/settings.json") - expect(skippedPaths).toContain("/mock/workspace/src/.cache/data.json") - expect(skippedPaths).toContain("/mock/workspace/dist/bundle.js") - }) + it.each(["dispose", "abort"] as const)( + "cancels the actual provider signal on %s and drops late results", + async (action) => { + const { watcher, embedder, store, failures, cache } = setup() + const held = deferred>>() + embedder.createEmbeddings.mockImplementationOnce(() => held.promise) + const owner = new AbortController() + await watcher.initialize(owner.signal) + const pending = watcher.processFile(fileA) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await drain() + const signal = embedder.createEmbeddings.mock.calls[0][2] + if (action === "dispose") watcher.dispose() + else owner.abort() + expect(signal?.aborted).toBe(true) + held.resolve({ embeddings: [[1]] }) + await rejected + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }, + ) + + it("tracks a late store result under the session owner and never completes an aborted file", async () => { + const { watcher, store, gate, cache, failures } = setup() + const held = deferred() + store.upsertPoints.mockImplementationOnce(() => held.promise) + const owner = new AbortController() + await watcher.initialize(owner.signal) + const pending = watcher.processFile(fileA) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await drain() + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect(gate.hasPending(owner.signal)).toBe(true) + owner.abort() + held.resolve(undefined) + await rejected + await gate.settled(owner.signal) + expect(gate.hasPending(owner.signal)).toBe(false) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(store.countPointsForFile).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }) - it("should batch delete events after the debounce window", async () => { - // Initialize the file watcher - await fileWatcher.initialize() + it.each([ + { kind: "empty", parsed: [] }, + { kind: "whitespace", parsed: [{ ...blocks(fileA)[0], content: " \n\t " }] }, + ])("completes $kind revisions without embedding", async ({ parsed }) => { + const { watcher, parser, embedder, store, hashes } = setup() + parser.parseFile.mockResolvedValue(parsed) + expect((await watcher.processFile(fileA)).status).toBe("success") + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(store.countPointsForFile).toHaveBeenCalledOnce() + expect(hashes.get(fileA)).toBe(hash("current content")) + }) - const deletedFiles: string[] = [] - mockVectorStore.deletePointsByMultipleFilePaths.mockImplementation(async (filePaths: string[]) => { - deletedFiles.push(...filePaths) + it.each(["abort", "dispose", "stale"] as const)( + "does not classify a session %s during delete as per-file supersession", + async (action) => { + const { watcher, hashes, store, failures, cache, supersede } = setup() + const held = deferred() + hashes.set(fileA, "previous") + store.deletePointsByFilePath.mockImplementationOnce(() => held.promise) + const owner = new AbortController() + await watcher.initialize(owner.signal) + const pending = watcher["removeFile"](fileA) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await drain() + if (action === "dispose") watcher.dispose() + else { + owner.abort() + if (action === "stale") supersede() + } + held.resolve(undefined) + await rejected + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }, + ) + + it.each(["read", "parse", "embed", "delete", "excluded"] as const)( + "reports superseded %s work as skipped without a false batch error or completion", + async (stage) => { + const { watcher, parser, embedder, store, cache, hashes, openFiles, failures, ignored } = setup() + const held = deferred() + hashes.set(fileA, "previous") + if (stage === "read") + vi.mocked(vscode.workspace.fs.readFile).mockImplementationOnce(async () => { + await held.promise + return Buffer.from("current content") + }) + else if (stage === "parse") + parser.parseFile.mockImplementationOnce(async () => { + await held.promise + return blocks(fileA) + }) + else if (stage === "embed") + embedder.createEmbeddings.mockImplementationOnce(async () => { + await held.promise + return { embeddings: [[1]] } + }) + else { + if (stage === "excluded") ignored.add("a.ts") + store.deletePointsByFilePath.mockImplementationOnce(() => held.promise) + } + const finished = vi.fn<(summary: BatchProcessingSummary) => void>() + watcher.onDidFinishBatchProcessing(finished) + await watcher.initialize() + await queue([fileA], stage === "delete" ? "delete" : "change") + expect(finished).not.toHaveBeenCalled() + handlers.change(makeUri(fileA)) + held.resolve(undefined) + await drain() + expect(finished).toHaveBeenCalledExactlyOnceWith({ + processedFiles: [{ path: fileA, status: "skipped", reason: "File change was superseded" }], + batchError: undefined, }) - const batchPromise = waitForNextBatch() - - // Simulate file deletion events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.git/objects/abc123", shouldProcess: false }, - { path: "/mock/workspace/.DS_Store", shouldProcess: false }, - { path: "/mock/workspace/build/.cache/temp.js", shouldProcess: false }, - ] - - // Trigger file deletion events - for (const { path } of testCases) { - await mockOnDidDelete({ fsPath: path }) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + if (stage === "embed" || stage === "delete" || stage === "excluded") { + expect(openFiles.has(fileA)).toBe(true) + expect(hashes.has(fileA)).toBe(false) } + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) + expect(finished).toHaveBeenCalledTimes(2) + expect(finished.mock.calls[1][0].batchError).toBeUndefined() + expect(finished.mock.calls[1][0].processedFiles).toEqual([{ path: fileA, status: "success" }]) + }, + ) + + it("keeps parser cancellation scoped to the session when a file change is superseded", async () => { + const { watcher, parser, cache } = setup() + await watcher.initialize() + const held = deferred() + parser.parseFile.mockReturnValueOnce(held.promise) + const pending = watcher.processFile(fileA) + await drain() + const parseSignal = parser.parseFile.mock.calls[0][1]?.signal + expect(parseSignal).toBeDefined() + handlers.change(makeUri(fileA)) + expect(parseSignal?.aborted).toBe(false) + held.resolve(blocks(fileA)) + await expect(pending).resolves.toMatchObject({ status: "skipped" }) + expect(cache.completeFile).not.toHaveBeenCalled() + await expect(watcher.processFile(fileA)).resolves.toMatchObject({ status: "success" }) + watcher.dispose() + expect(parseSignal?.aborted).toBe(true) + }) - await flushBatch() - await batchPromise - - expect(deletedFiles).toEqual(testCases.map(({ path }) => path)) - }) - - it("should handle nested hidden directories correctly", async () => { - // Initialize the file watcher - await fileWatcher.initialize() - - const batchPromise = waitForNextBatch() - - // Test deeply nested hidden directories - const testCases = [ - { path: "/mock/workspace/src/components/Button.tsx", shouldProcess: true }, - { path: "/mock/workspace/src/.hidden/components/Button.tsx", shouldProcess: false }, - { path: "/mock/workspace/.hidden/src/components/Button.tsx", shouldProcess: false }, - { path: "/mock/workspace/src/components/.hidden/Button.tsx", shouldProcess: false }, - ] - - // Trigger file creation events - for (const { path } of testCases) { - await mockOnDidCreate({ fsPath: path }) - } + it("preserves a fatal parser failure after the active file change was superseded", async () => { + const { watcher, parser } = setup() + await watcher.initialize() + const reported = vi.fn() + watcher.onParserFatal(reported) + const held = deferred() + const fatal = new ParserFatalError("parse-timeout", fileA) + parser.parseFile.mockReturnValueOnce(held.promise) + const pending = watcher.processFile(fileA).catch((error: unknown) => error) + await drain() + handlers.change(makeUri(fileA)) + held.reject(fatal) + expect(await pending).toBe(fatal) + expect(reported).toHaveBeenCalledExactlyOnceWith(fatal) + }) - await flushBatch() + it("preserves a non-superseded public AbortError instead of treating it as skipped", async () => { + const { watcher, parser, failures } = setup() + const failure = new DOMException("parser cancelled", "AbortError") + parser.parseFile.mockRejectedValueOnce(failure) + await expect(watcher.processFile(fileA)).rejects.toBe(failure) + expect(failures.list()).toEqual([]) + }) - const batchSummary = await batchPromise - const successPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "success") - .map((result: any) => result.path) - const skippedPaths = batchSummary.processedFiles - .filter((result: any) => result.status === "skipped") - .map((result: any) => result.path) + it("cancels a long retry wait on revision change without renewing another file", async () => { + const { watcher, embedder, failures, hashes } = setup() + embedder.createEmbeddings.mockRejectedValueOnce( + Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 }), + ) + await watcher.initialize() + await queue() + const oldSignal = embedder.createEmbeddings.mock.calls[0][2] + failures.renew("b.ts", "other") + const otherBudget = failures.getRetryBudget("b.ts", "other", "embed") + otherBudget.retriesUsed = 2 + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("new revision")) + handlers.change(makeUri(fileA)) + expect(oldSignal?.aborted).toBe(true) + await vi.advanceTimersByTimeAsync(1000) + // The shared provider pause outlives the cancelled revision. + expect(hashes.get(fileA)).toBeUndefined() + await vi.advanceTimersByTimeAsync(28_000) + expect(hashes.get(fileA)).toBeUndefined() + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(2000) + expect(hashes.get(fileA)).toBe(hash("new revision")) + expect(failures.getRetryBudget("b.ts", "other", "embed")).toBe(otherBudget) + await vi.advanceTimersByTimeAsync(60_000) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) + }) - // Check that files in hidden directories were not processed - expect(successPaths).toContain("/mock/workspace/src/components/Button.tsx") - expect(skippedPaths).toContain("/mock/workspace/src/.hidden/components/Button.tsx") - expect(skippedPaths).toContain("/mock/workspace/.hidden/src/components/Button.tsx") - expect(skippedPaths).toContain("/mock/workspace/src/components/.hidden/Button.tsx") - }) + it("cancels an active revision on delete before generation-scoped removal", async () => { + const { watcher, embedder, hashes, store, failures } = setup() + embedder.createEmbeddings.mockRejectedValueOnce(Object.assign(new Error("temporary"), { status: 503 })) + await watcher.initialize() + await queue() + const oldSignal = embedder.createEmbeddings.mock.calls[0][2] + handlers.delete(makeUri(fileA)) + expect(oldSignal?.aborted).toBe(true) + await vi.advanceTimersByTimeAsync(1000) + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(hashes.has(fileA)).toBe(false) + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync(60_000) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() }) - describe("dispose", () => { - it("should dispose of the watcher when disposed", async () => { - await fileWatcher.initialize() - fileWatcher.dispose() + it("keeps verification mismatch and uncertain unchanged content incomplete", async () => { + const { watcher, hashes, gate, store, parser, cache } = setup() + hashes.set(fileA, hash("current content")) + expect((await watcher.processFile(fileA)).status).toBe("skipped") + gate.recover([fileA]) + store.countPointsForFile.mockResolvedValue({ current: 0, other: 1 }) + expect((await watcher.processFile(fileA)).status).toBe("local_error") + expect(parser.parseFile).toHaveBeenCalledOnce() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(hashes.has(fileA)).toBe(false) + }) - expect(mockWatcher.dispose).toHaveBeenCalled() - }) + it("bounds dispatch while allowing an unrelated file to finish independently", async () => { + const { watcher, parser, hashes } = setup() + const held = deferred() + parser.parseFile.mockImplementation(async (filePath) => (filePath === fileA ? blocks(fileA) : held.promise)) + await watcher.initialize() + const paths = [ + fileA, + ...Array.from({ length: 11 }, (_, fileNumber) => path.join(workspacePath, `${fileNumber}.ts`)), + ] + await queue(paths) + expect(hashes.has(fileA)).toBe(true) + expect(parser.parseFile).toHaveBeenCalledTimes(11) + watcher.dispose() + held.resolve([]) + await drain() + expect(parser.parseFile).toHaveBeenCalledTimes(11) }) }) diff --git a/src/services/code-index/processors/__tests__/parser-constants.spec.ts b/src/services/code-index/processors/__tests__/parser-constants.spec.ts new file mode 100644 index 0000000000..07cdf42e7b --- /dev/null +++ b/src/services/code-index/processors/__tests__/parser-constants.spec.ts @@ -0,0 +1,38 @@ +import { build } from "esbuild" +import * as path from "path" +import { readFile } from "fs/promises" +import * as parserConstants from "../parser-constants" + +it("preserves the four foundation parser constants exactly", async () => { + const source = await readFile(path.resolve("services/code-index/constants/index.ts"), "utf8") + const expected = { + MAX_BLOCK_CHARS: 1000, + MIN_BLOCK_CHARS: 50, + MIN_CHUNK_REMAINDER_CHARS: 200, + MAX_CHARS_TOLERANCE_FACTOR: 1.15, + } + expect(parserConstants).toEqual(expected) + for (const [name, value] of Object.entries(expected)) { + expect(source).toMatch(new RegExp(`export const ${name} = ${String(value).replace(".", "\\.")}(?:\\s|$)`)) + } +}) + +it("bundles the actual parser worker without host or types/provider barrel inputs", async () => { + const result = await build({ + entryPoints: [path.resolve("workers/codeIndexParse.ts")], + bundle: true, + platform: "node", + format: "cjs", + write: false, + metafile: true, + }) + const inputs = Object.keys(result.metafile.inputs).map((input) => input.replaceAll("\\", "/")) + expect(inputs).toContain("services/code-index/processors/parser-constants.ts") + expect( + inputs.filter((input) => + /(?:code-index\/constants\/|packages\/types\/|\/providers\/|telemetry|RooIgnoreController)/.test(input), + ), + ).toEqual([]) + const imports = Object.values(result.metafile.outputs).flatMap((output) => output.imports) + expect(imports.filter((entry) => entry.path === "vscode" || entry.path === "@roo-code/types")).toEqual([]) +}) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index e17128d0c5..656b78cfde 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -8,13 +8,8 @@ import { Node } from "web-tree-sitter" import { clearAllMocks } from "../../../../test-utils/reset" -// Mock TelemetryService -vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), - }, - }, +vi.mock("../../code-index-log", () => ({ + codeIndexLog: { error: vi.fn(), warn: vi.fn() }, })) // Override Jest-based fs/promises mock with vitest-compatible version @@ -41,6 +36,7 @@ const mockLanguageParser = { js: { parser: { parse: vi.fn((content: string) => ({ + delete: vi.fn(), rootNode: { text: content, startPosition: { row: 0 }, diff --git a/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts b/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts new file mode 100644 index 0000000000..c7584022ce --- /dev/null +++ b/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts @@ -0,0 +1,368 @@ +import { createHash } from "crypto" +import { readFile } from "fs/promises" +import type { Node, QueryCapture, Tree } from "web-tree-sitter" +import { CodeParser } from "../parser" +import { codeIndexLog } from "../../code-index-log" +import { MAX_BLOCK_CHARS } from "../../constants" +import type { CodeBlock } from "../../interfaces" +import { ParserFatalError } from "../../parser-errors" +import { scannerExtensions } from "../../shared/supported-extensions" +import { type LanguageParser, loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser" +import { parseMarkdown } from "../../../tree-sitter/markdownParser" + +vi.mock("fs/promises", () => ({ readFile: vi.fn() })) +vi.mock("../../../tree-sitter/languageParser", () => ({ loadRequiredLanguageParsers: vi.fn() })) +vi.mock("../../../tree-sitter/markdownParser", () => ({ parseMarkdown: vi.fn() })) +vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) + +type ParseDelegate = NonNullable[0]> + +function createNode(content: string, row = 0, children: Node[] = []): Node { + // The parser only reads text, positions, type, children and the name field from these nodes. + return { + text: content, + startPosition: { row, column: 0 }, + endPosition: { row: row + content.split("\n").length - 1, column: 0 }, + type: "function_declaration", + children, + childForFieldName: () => null, + } as unknown as Node +} + +function createLanguage() { + const release = vi.fn<() => void>() + const rootNode = createNode("source") + // Only rootNode and delete are consumed by CodeParser. + const tree = { rootNode, delete: release } as unknown as Tree + const parse = vi.fn().mockReturnValue(tree) + const captures = vi.fn().mockReturnValue([]) + // These structural doubles supply the only parser/query methods CodeParser invokes. + const language = { js: { parser: { parse }, query: { captures } } } as unknown as LanguageParser + return { language, parse, captures, release } +} + +function expectUnicodePreserved(blocks: CodeBlock[], content: string): void { + expect(blocks.length).toBeGreaterThan(1) + expect(blocks.map((block) => block.content).join("")).toBe(content) + for (const block of blocks) { + expect(Buffer.from(block.content, "utf8").toString("utf8")).toBe(block.content) + expect(block.content.length).toBeLessThanOrEqual(MAX_BLOCK_CHARS) + } + expect(new Set(blocks.map((block) => block.segmentHash)).size).toBe(blocks.length) +} + +beforeEach(() => { + vi.resetAllMocks() + vi.mocked(parseMarkdown).mockReturnValue([]) +}) + +describe("CodeParser worker contract", () => { + it("keeps worker-safe extension support identical to the scanner", async () => { + const delegate = vi.fn().mockResolvedValue([]) + const parser = new CodeParser(delegate) + const supported = scannerExtensions.filter((extension) => parser["isSupportedLanguage"](extension)) + expect(supported).toEqual(scannerExtensions) + expect(parser["isSupportedLanguage"](".unsupported")).toBe(false) + await expect(parser.parseFile("file.unsupported")).resolves.toEqual([]) + expect(delegate).not.toHaveBeenCalled() + expect(readFile).not.toHaveBeenCalled() + }) + + it.each(["file.js", "file.TS", "file.dart", "file.rdl", "file.tla"])( + "delegates structural parsing for %s with content, hash and signal", + async (filePath) => { + const blocks: CodeBlock[] = [] + const delegate = vi.fn().mockResolvedValue(blocks) + const signal = new AbortController().signal + const result = await new CodeParser(delegate).parseFile(filePath, { + content: "source", + fileHash: "hash", + signal, + }) + expect(result).toBe(blocks) + expect(delegate).toHaveBeenCalledExactlyOnceWith(filePath, "source", "hash", signal) + expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() + expect(readFile).not.toHaveBeenCalled() + }, + ) + + it.each(["file.md", "file.markdown", "file.txt", "file.TXT", "file.vb", "file.scala", "file.swift"])( + "keeps %s on the local Markdown/fallback path", + async (filePath) => { + const delegate = vi.fn() + const content = "Local parsing preserves Zoo language support and searchable file content." + const blocks = await new CodeParser(delegate).parseFile(filePath, { content, fileHash: "hash" }) + expect(blocks).toHaveLength(1) + expect(blocks[0]).toMatchObject({ content, fileHash: "hash", start_line: 1, end_line: 1 }) + expect(delegate).not.toHaveBeenCalled() + expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() + }, + ) + + it.each(["", "const value = 42"])("hashes supplied content without reading a file: %j", async (content) => { + const delegate = vi.fn().mockResolvedValue([]) + await new CodeParser(delegate).parseFile("file.js", { content }) + expect(delegate).toHaveBeenCalledWith( + "file.js", + content, + createHash("sha256").update(content).digest("hex"), + undefined, + ) + expect(readFile).not.toHaveBeenCalled() + }) + + it("reads and hashes content before delegation", async () => { + vi.mocked(readFile).mockResolvedValue("file contents") + const delegate = vi.fn().mockResolvedValue([]) + await new CodeParser(delegate).parseFile("file.js") + expect(readFile).toHaveBeenCalledWith("file.js", "utf8") + expect(delegate).toHaveBeenCalledWith( + "file.js", + "file contents", + createHash("sha256").update("file contents").digest("hex"), + undefined, + ) + }) + + it("logs ordinary read failure without delegating", async () => { + const error = new Error("read failure") + vi.mocked(readFile).mockRejectedValue(error) + const delegate = vi.fn() + await expect(new CodeParser(delegate).parseFile("file.js")).resolves.toEqual([]) + expect(codeIndexLog.error).toHaveBeenCalledWith("Error reading file file.js:", error) + expect(delegate).not.toHaveBeenCalled() + }) + + it.each(["file.js", "file.md", "file.txt", "file.unsupported"])( + "rejects pre-cancelled work before reading or parsing %s", + async (filePath) => { + const controller = new AbortController() + controller.abort() + const delegate = vi.fn() + await expect( + new CodeParser(delegate).parseFile(filePath, { signal: controller.signal }), + ).rejects.toMatchObject({ + name: "AbortError", + }) + expect(delegate).not.toHaveBeenCalled() + expect(readFile).not.toHaveBeenCalled() + expect(parseMarkdown).not.toHaveBeenCalled() + }, + ) + + it.each([false, true])("cancels after an in-flight read settles (read failure: %s)", async (rejectRead) => { + const controller = new AbortController() + vi.mocked(readFile).mockImplementation(async () => { + controller.abort() + if (rejectRead) throw new Error("late read failure") + return "source" + }) + const delegate = vi.fn() + await expect( + new CodeParser(delegate).parseFile("file.js", { signal: controller.signal }), + ).rejects.toMatchObject({ + name: "AbortError", + }) + expect(delegate).not.toHaveBeenCalled() + expect(codeIndexLog.error).not.toHaveBeenCalled() + }) + + it.each([ + new ParserFatalError("parse-timeout", "file.js"), + new ParserFatalError("parser-abort", "file.js"), + new Error("ordinary worker failure"), + ])("propagates delegate failure unchanged: %s", async (error) => { + const delegate = vi.fn().mockRejectedValue(error) + await expect(new CodeParser(delegate).parseFile("file.js", { content: "source" })).rejects.toBe(error) + }) + + it("passes the worker WASM directory and requests host fallback only when captures are absent", async () => { + const { language, captures, release } = createLanguage() + vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) + const parser = new CodeParser(undefined, "extension-dist") + const content = "Uncaptured structural content still needs to be indexed through host fallback." + await expect(parser.parseFile("file.js", { content })).resolves.toEqual([]) + expect(loadRequiredLanguageParsers).toHaveBeenCalledExactlyOnceWith(["file.js"], "extension-dist") + expect(parser.fallbackNeeded).toBe(true) + expect(release).toHaveBeenCalledOnce() + expect(parser.fallbackContent("file.js", content, "hash")[0]).toMatchObject({ content, fileHash: "hash" }) + + captures.mockReturnValue([{ node: createNode(content), name: "definition.function", patternIndex: 0 }]) + const blocks = await parser.parseFile("file.js", { content }) + expect(blocks).toHaveLength(1) + expect(parser.fallbackNeeded).toBe(false) + expect(release).toHaveBeenCalledTimes(2) + expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() + }) + + it.each(["file.txt", "file.md", "file.unsupported"])("resets stale fallback state for %s", async (filePath) => { + const { language } = createLanguage() + vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) + const parser = new CodeParser(undefined, "extension-dist") + await parser.parseFile("file.js", { content: "source" }) + expect(parser.fallbackNeeded).toBe(true) + await parser.parseFile(filePath, { content: "Enough local content to create a full indexable fallback block." }) + expect(parser.fallbackNeeded).toBe(false) + }) + + it.each(["Aborted()", "memory access out of bounds", "ordinary load failure"])( + "handles a fresh and shared parser-load failure: %s", + async (message) => { + const error = new Error(message) + let rejectLoad!: (reason: Error) => void + const pending = new Promise((_resolve, reject) => { + rejectLoad = reject + }) + vi.mocked(loadRequiredLanguageParsers).mockReturnValue(pending) + const parser = new CodeParser() + const first = parser.parseFile("first.js", { content: "source" }).catch((failure: unknown) => failure) + const second = parser.parseFile("second.js", { content: "source" }).catch((failure: unknown) => failure) + rejectLoad(error) + const results = await Promise.all([first, second]) + if (message === "ordinary load failure") expect(results).toEqual([[], []]) + else for (const result of results) expect(result).toBe(error) + expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() + expect(parser["pendingLoads"].size).toBe(0) + }, + ) + + it("shares a successful parser load between concurrent files", async () => { + const { language, parse, release } = createLanguage() + let resolveLoad!: (loaded: LanguageParser) => void + vi.mocked(loadRequiredLanguageParsers).mockReturnValue( + new Promise((resolve) => { + resolveLoad = resolve + }), + ) + const parser = new CodeParser() + const first = parser.parseFile("first.js", { content: "source" }) + const second = parser.parseFile("second.js", { content: "source" }) + resolveLoad(language) + await Promise.all([first, second]) + expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() + expect(parse).toHaveBeenCalledTimes(2) + expect(release).toHaveBeenCalledTimes(2) + }) + + it.each(["success", "query failure", "fallback", "null tree"])("releases tree on %s", async (outcome) => { + const { language, parse, captures, release } = createLanguage() + vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) + const content = "A structural function with sufficient content for indexing and lifecycle checks." + const error = new Error("memory access out of bounds") + if (outcome === "query failure") + captures.mockImplementation(() => { + throw error + }) + if (outcome === "success") + captures.mockReturnValue([{ node: createNode(content), name: "definition", patternIndex: 0 }]) + if (outcome === "null tree") parse.mockReturnValue(null) + const result = new CodeParser().parseFile("file.js", { content }) + if (outcome === "query failure") await expect(result).rejects.toBe(error) + else expect((await result)[0].content).toBe(content) + expect(release).toHaveBeenCalledTimes(outcome === "null tree" ? 0 : 1) + }) + + it("walks oversized children in capture order and deduplicates captured nodes", async () => { + const { language, captures, release } = createLanguage() + const child = createNode("A child declaration with enough meaningful content to create a code block.", 4) + const leaf = createNode("z".repeat(2001), 8) + const parent = createNode("p".repeat(4000), 0, [child, leaf]) + const capture = (node: Node): QueryCapture => ({ node, name: "definition", patternIndex: 0 }) + captures.mockReturnValue([capture(parent), capture(child)]) + vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) + const blocks = await new CodeParser().parseFile("file.js", { content: "source" }) + expect(blocks.map((block) => block.content)).toEqual([child.text, "z".repeat(1000), "z".repeat(1000), "z"]) + expect(blocks.map((block) => block.start_line)).toEqual([5, 9, 9, 9]) + expect(release).toHaveBeenCalledOnce() + }) +}) + +describe("CodeParser Unicode segmentation", () => { + it.each(["lines", "supplementary long line"])("keeps fallback scaling proportional for %s", (shape) => { + const parser = new CodeParser() + const source = + shape === "lines" + ? "const example = value => value.toUpperCase();\n".repeat(750) + : ("a".repeat(999) + "\u{1f600}").repeat(32) + const inputs = [1, 2, 4, 8].map((scale) => source.repeat(scale)) + const measure = (content: string) => { + const samples: number[] = [] + for (let sample = 0; sample < 5; sample++) { + const started = performance.now() + for (let repeat = 0; repeat < 40; repeat++) parser.fallbackContent("file.txt", content, "hash") + samples.push(performance.now() - started) + } + return Math.min(...samples) + } + for (let warmup = 0; warmup < 40; warmup++) parser.fallbackContent("file.txt", source, "hash") + const timings = inputs.map(measure) + expect(timings[2] / timings[0]).toBeLessThan(9) + expect(timings[3] / timings[0]).toBeLessThan(18) + }) + + it.each([MAX_BLOCK_CHARS - 2, MAX_BLOCK_CHARS - 1, MAX_BLOCK_CHARS])( + "keeps a supplementary code point whole at offset %s and hashes actual offsets", + (pairOffset) => { + const content = "a".repeat(pairOffset) + "\u{1f600}" + "b".repeat(MAX_BLOCK_CHARS + 17) + const blocks = new CodeParser()["_chunkTextByLines"]([content], "file.js", "hash", "code", new Set(), 42) + expectUnicodePreserved(blocks, content) + expect(blocks[0].content.length).toBe( + pairOffset === MAX_BLOCK_CHARS - 1 ? MAX_BLOCK_CHARS - 1 : MAX_BLOCK_CHARS, + ) + let offset = 0 + for (const block of blocks) { + expect(block).toMatchObject({ start_line: 42, end_line: 42, type: "code_segment" }) + expect(block.segmentHash).toBe( + createHash("sha256") + .update(`file.js-42-42-${offset}-${block.content.length}-${block.content.slice(0, 100)}`) + .digest("hex"), + ) + offset += block.content.length + } + }, + ) + + it("uses adjusted cuts for consecutive pairs and preserves a short tail", () => { + const content = "a".repeat(999) + ("\u{1f600}" + "a".repeat(997)).repeat(3) + "\u{1f680}" + const blocks = new CodeParser().fallbackContent("file.txt", content, "hash") + expectUnicodePreserved(blocks, content) + expect(blocks.map((block) => block.content.length)).toEqual([999, 999, 999, 999, 2]) + }) + + it.each(["file.md", "file.markdown", "file.txt", "file.scala", "file.vb", "file.js"])( + "preserves Unicode through parseFile for %s", + async (filePath) => { + vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(createLanguage().language) + const content = "a".repeat(999) + "\u{1f600}" + "漢é".repeat(1000) + const blocks = await new CodeParser().parseFile(filePath, { content }) + expectUnicodePreserved(blocks, content) + }, + ) + + it.each([ + "ascii ", + "café Ελληνικά Кириллица ", + "漢字かなカナ한국어 ", + "e\u0301 a\u0308 \u200d \ufe0f ", + "\ud7ff\ue000\ufffd\uffff ", + ])("retains BMP segment contents and hashes: %j", (seed) => { + const content = seed.repeat(500) + const expected: CodeBlock[] = [] + for (let offset = 0; offset < content.length; offset += MAX_BLOCK_CHARS) { + const segment = content.substring(offset, offset + MAX_BLOCK_CHARS) + expected.push({ + file_path: "file.txt", + identifier: null, + type: "fallback_chunk_segment", + start_line: 1, + end_line: 1, + content: segment, + fileHash: "hash", + segmentHash: createHash("sha256") + .update(`file.txt-1-1-${offset}-${segment.length}-${segment.slice(0, 100)}`) + .digest("hex"), + }) + } + expect(new CodeParser().fallbackContent("file.txt", content, "hash")).toEqual(expected) + }) +}) diff --git a/src/services/code-index/processors/__tests__/processor-factories.spec.ts b/src/services/code-index/processors/__tests__/processor-factories.spec.ts index 1b222fb997..57718968ac 100644 --- a/src/services/code-index/processors/__tests__/processor-factories.spec.ts +++ b/src/services/code-index/processors/__tests__/processor-factories.spec.ts @@ -7,15 +7,14 @@ import { CacheManager } from "../../cache-manager" import { BATCH_SEGMENT_THRESHOLD } from "../../constants" import { OpenAiEmbedder } from "../../embedders/openai" import { QdrantVectorStore } from "../../vector-store/qdrant-client" -import { CodeIndexServiceFactory } from "../../service-factory" -import { CodeIndexConfigManager } from "../../config-manager" -import { EmbedderFactory } from "../../embedders/embedder-factory" -import { VectorStoreFactory } from "../../vector-store/vector-store-factory" -import type { CodeIndexConfig } from "../../interfaces/config" +import { ParseWorkerClient } from "../../parse-worker-client" +import { IndexWriteGate } from "../../index-write-gate" +import { GenerationAllocator } from "../../generation-allocator" +import { FileFailureRegistry } from "../../file-failure-registry" +import { ProviderPacerRegistry } from "../../provider-pacer" import { DirectoryScannerFactory } from "../directory-scanner-factory" import { FileWatcherFactory } from "../file-watcher-factory" import { getEmbeddingBatchSize } from "../get-embedding-batch-size" -import { codeParser } from "../parser" import { DirectoryScanner } from "../scanner" import { FileWatcher } from "../file-watcher" @@ -25,8 +24,7 @@ vi.mock("vscode", () => ({ vi.mock("../../cache-manager") vi.mock("../../embedders/openai") vi.mock("../../vector-store/qdrant-client") -vi.mock("../../config-manager") -vi.mock("../parser", () => ({ codeParser: { parseFile: vi.fn() } })) +vi.mock("../../parse-worker-client") vi.mock("../scanner") vi.mock("../file-watcher") @@ -36,14 +34,20 @@ describe("processor factories", () => { function dependencies() { const context = makeExtensionContext() + const cacheManager = new CacheManager(context, "/workspace") return { context, workspacePath: "/workspace", - cacheManager: new CacheManager(context, "/workspace"), + cacheManager, embedder: new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), vectorStore: new QdrantVectorStore("/workspace", "http://localhost:6333", 512), - parser: codeParser, + parser: new ParseWorkerClient(), ignoreInstance: ignore(), + writeGate: new IndexWriteGate(), + generations: new GenerationAllocator(cacheManager), + failures: new FileFailureRegistry(), + isCurrentAttempt: vi.fn(() => true), + providerPacers: new ProviderPacerRegistry(), } } @@ -67,6 +71,12 @@ describe("processor factories", () => { options.cacheManager, options.ignoreInstance, batchSize, + options.writeGate, + options.generations, + options.failures, + options.workspacePath, + options.isCurrentAttempt, + options.providerPacers, ) expect(FileWatcher).toHaveBeenCalledWith( options.workspacePath, @@ -77,6 +87,12 @@ describe("processor factories", () => { options.ignoreInstance, undefined, batchSize, + options.parser, + options.writeGate, + options.generations, + options.failures, + options.isCurrentAttempt, + options.providerPacers, ) expect(vscode.workspace.getConfiguration).toHaveBeenCalledWith(Package.name) expect(configuration.get).toHaveBeenCalledWith("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) @@ -139,49 +155,62 @@ describe("processor factories", () => { expect(warn).not.toHaveBeenCalled() }) - it("preserves service factory wiring, including the explicitly supplied watcher cache", () => { + it("keeps shared retry budgets and current-attempt callbacks across processor recreation", () => { const options = dependencies() - const watcherCache = new CacheManager(options.context, "/workspace") - const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager - const config: CodeIndexConfig = { isConfigured: true, embedderProvider: "openai-compatible" } - Object.defineProperty(configManager, "isFeatureConfigured", { value: true }) - vi.mocked(configManager.getConfig).mockReturnValue(config) - const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create").mockReturnValue(options.embedder) - const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create").mockReturnValue(options.vectorStore) - const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) - - const services = factory.createServices(options.context, watcherCache, options.ignoreInstance) - - expect(configManager.getConfig).toHaveBeenCalledTimes(1) - expect(createEmbedder).toHaveBeenCalledExactlyOnceWith(config) - expect(createVectorStore).toHaveBeenCalledExactlyOnceWith(config, options.workspacePath) - expect(services).toEqual({ - embedder: options.embedder, - vectorStore: options.vectorStore, - parser: codeParser, - scanner: vi.mocked(DirectoryScanner).mock.instances[0], - fileWatcher: vi.mocked(FileWatcher).mock.instances[0], - }) - expect(vi.mocked(DirectoryScanner).mock.calls[0][3]).toBe(options.cacheManager) - expect(vi.mocked(FileWatcher).mock.calls[0][2]).toBe(watcherCache) - expect(vi.mocked(FileWatcher).mock.calls[0][0]).toBe(options.workspacePath) + options.failures.renew("file.ts", "revision") + const budget = options.failures.getRetryBudget("file.ts", "revision", "embed") + budget.retriesUsed = 2 + const reset = vi.spyOn(options.failures, "resetAllBudgets") + const scanners = new DirectoryScannerFactory() + const watchers = new FileWatcherFactory() + + for (const current of [true, false]) { + options.isCurrentAttempt.mockReturnValue(current) + scanners.create(options) + watchers.create(options) + const scannerArguments = vi.mocked(DirectoryScanner).mock.lastCall! + const watcherArguments = vi.mocked(FileWatcher).mock.lastCall! + expect(scannerArguments[2]).toBe(options.parser) + expect(watcherArguments[8]).toBe(options.parser) + expect(scannerArguments[6]).toBe(options.writeGate) + expect(watcherArguments[9]).toBe(options.writeGate) + expect(scannerArguments[7]).toBe(options.generations) + expect(watcherArguments[10]).toBe(options.generations) + expect(scannerArguments[8]).toBe(options.failures) + expect(watcherArguments[11]).toBe(options.failures) + expect(scannerArguments[10]).toBe(options.isCurrentAttempt) + expect(watcherArguments[12]).toBe(options.isCurrentAttempt) + expect(scannerArguments[11]).toBe(options.providerPacers) + expect(watcherArguments[13]).toBe(options.providerPacers) + expect(scannerArguments[10]?.()).toBe(current) + expect(watcherArguments[12]?.()).toBe(current) + } + + expect(options.failures.getRetryBudget("file.ts", "revision", "embed")).toBe(budget) + expect(budget.retriesUsed).toBe(2) + expect(reset).not.toHaveBeenCalled() }) - it("does not create services when indexing is not configured", () => { - const options = dependencies() - const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager - Object.defineProperty(configManager, "isFeatureConfigured", { value: false }) - const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create") - const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create") - const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) - - expect(() => factory.createServices(options.context, options.cacheManager, options.ignoreInstance)).toThrow( - "serviceFactory.codeIndexingNotConfigured", - ) - expect(configManager.getConfig).not.toHaveBeenCalled() - expect(createEmbedder).not.toHaveBeenCalled() - expect(createVectorStore).not.toHaveBeenCalled() - expect(DirectoryScanner).not.toHaveBeenCalled() - expect(FileWatcher).not.toHaveBeenCalled() + it("preserves optional constructor defaults for standalone factory callers", () => { + const { embedder, vectorStore, parser, cacheManager, ignoreInstance, context, workspacePath } = dependencies() + new DirectoryScannerFactory().create({ embedder, vectorStore, parser, cacheManager, ignoreInstance }) + new FileWatcherFactory().create({ workspacePath, context, embedder, vectorStore, cacheManager, ignoreInstance }) + + expect(vi.mocked(DirectoryScanner).mock.lastCall?.slice(6)).toEqual([ + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ]) + expect(vi.mocked(FileWatcher).mock.lastCall?.slice(8)).toEqual([ + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ]) }) }) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 72fefd676d..bfb2f3ac86 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -1,493 +1,565 @@ -// npx vitest services/code-index/processors/__tests__/scanner.spec.ts - -import { DirectoryScanner } from "../scanner" +import * as path from "path" +import * as vscode from "vscode" +import { Stats } from "fs" import { stat } from "fs/promises" - -// Mock TelemetryService -vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ - TelemetryService: { - instance: { - captureEvent: vi.fn(), - }, +import { createHash } from "crypto" +import ignore from "ignore" +import { DirectoryScanner } from "../scanner" +import type { CacheManager } from "../../cache-manager" +import type { CodeBlock, ICodeParser, IEmbedder, IVectorStore, PointStruct } from "../../interfaces" +import { FileFailureRegistry } from "../../file-failure-registry" +import { GenerationAllocator } from "../../generation-allocator" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" +import { ParserFatalError } from "../../parser-errors" +import { codeIndexLog } from "../../code-index-log" +import { listFiles } from "../../../glob/list-files" +import { generateRelativeFilePath } from "../../shared/get-relative-path" +import { PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, MAX_FILE_SIZE_BYTES } from "../../constants" + +import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" + +vi.mock("fs/promises", () => ({ stat: vi.fn() })) +vi.mock("vscode", () => ({ + workspace: { fs: { readFile: vi.fn() } }, + Uri: { file: (filePath: string) => ({ fsPath: filePath }) }, +})) +vi.mock("../../../../utils/path", () => ({ getWorkspacePathForContext: (directory: string) => directory })) +vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: class { + async initialize() {} + filterPaths(paths: string[]) { + return paths + } }, })) - -vi.mock("fs/promises", () => ({ - default: { - readFile: vi.fn(), - writeFile: vi.fn(), - mkdir: vi.fn(), - access: vi.fn(), - rename: vi.fn(), - constants: {}, +vi.mock("../../../glob/list-files", () => ({ listFiles: vi.fn() })) +vi.mock("../../code-index-log", () => ({ + codeIndexLog: { + redact: (message: string) => message, + createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), + info: vi.fn(), }, - stat: vi.fn(), })) -// Create a simple mock for vscode since we can't access the real one -vi.mock("vscode", () => ({ - workspace: { - workspaceFolders: [ - { - uri: { - fsPath: "/mock/workspace", - }, - }, - ], - getWorkspaceFolder: vi.fn().mockReturnValue({ - uri: { - fsPath: "/mock/workspace", - }, +const workspacePath = path.resolve("scanner-workspace") +const fileA = path.join(workspacePath, "a.ts") +const fileB = path.join(workspacePath, "b.ts") +const hash = (content: string) => createHash("sha256").update(content).digest("hex") + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function blocks(filePath: string, count = 1): CodeBlock[] { + return Array.from({ length: count }, (_, blockNumber) => ({ + file_path: filePath, + content: `function block${blockNumber}() { return ${blockNumber} }`, + start_line: blockNumber + 1, + end_line: blockNumber + 1, + identifier: null, + type: "function", + fileHash: hash("current content"), + segmentHash: `${filePath}:${blockNumber}`, + })) +} + +function setup() { + const hashes = new Map() + const openFiles = new Set() + const points = new Map() + const cache = { + generationWatermark: 1024, + reserveGenerations: vi.fn().mockResolvedValue(undefined), + getHash: vi.fn((filePath: string) => hashes.get(filePath)), + getTrackedPaths: vi.fn(() => [...new Set([...hashes.keys(), ...openFiles])]), + invalidateFile: vi.fn(async (filePath: string) => { + hashes.delete(filePath) + openFiles.add(filePath) }), - fs: { - readFile: vi.fn().mockResolvedValue(Buffer.from("test content")), - }, - }, - Uri: { - file: vi.fn().mockImplementation((path) => { - return path + completeFile: vi.fn((filePath: string, revision: string) => { + hashes.set(filePath, revision) + openFiles.delete(filePath) }), - }, - window: { - activeTextEditor: { - document: { - uri: { - fsPath: "/mock/workspace", - }, - }, + completeRemoval: vi.fn((filePath: string) => { + hashes.delete(filePath) + openFiles.delete(filePath) + }), + } + const store = { + rateLimitScope: "qdrant:scanner-test", + initialize: vi.fn().mockResolvedValue(false), + upsertPoints: vi.fn().mockImplementation(async (batch) => { + for (const point of batch) points.set(point.id, point) + }), + deletePointsByFilePath: vi + .fn() + .mockImplementation(async (filePath, cutoff) => { + for (const point of points.values()) { + const payload: Record = point.payload + if ( + payload.filePath === generateRelativeFilePath(filePath, workspacePath) && + (typeof payload.generation !== "number" || payload.generation < cutoff) + ) + points.delete(point.id) + } + }), + deletePointsByMultipleFilePaths: vi + .fn() + .mockResolvedValue(undefined), + countPointsForFile: vi + .fn() + .mockImplementation(async (filePath, generation) => { + const selected = [...points.values()] + .map((point): Record => point.payload) + .filter((payload) => payload.filePath === generateRelativeFilePath(filePath, workspacePath)) + return { + current: selected.filter((payload) => payload.generation === generation).length, + other: selected.filter((payload) => payload.generation !== generation).length, + } + }), + search: vi.fn().mockResolvedValue([]), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), + collectionExists: vi.fn().mockResolvedValue(true), + hasIndexedData: vi.fn().mockResolvedValue(true), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + } satisfies IVectorStore + const embedder = { + rateLimitScope: "openai:scanner-test", + createEmbeddings: vi + .fn() + .mockImplementation(async (texts) => ({ embeddings: texts.map(() => [1]) })), + validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), + embedderInfo: { name: "openai" }, + } satisfies IEmbedder + const parser = { + parseFile: vi.fn().mockImplementation(async (filePath) => blocks(filePath)), + } + const failures = new FileFailureRegistry() + const gate = new IndexWriteGate() + // The double supplies only the cache lookup, generation reservation, and completion methods used by scanner/writer. + const cacheManager = cache as unknown as CacheManager + const generations = new GenerationAllocator(cacheManager) + const ignored = ignore() + let currentAttempt = true + const createScanner = () => + new DirectoryScanner( + embedder, + store, + parser, + cacheManager, + ignored, + 1, + gate, + generations, + failures, + workspacePath, + () => currentAttempt, + ) + return { + cache, + hashes, + openFiles, + points, + store, + embedder, + parser, + failures, + gate, + ignored, + createScanner, + scanner: createScanner(), + supersede: () => { + currentAttempt = false }, - }, -})) - -vi.mock("../../../../core/ignore/RooIgnoreController") -vi.mock("ignore") - -// Override the Jest-based mock with a vitest-compatible version -vi.mock("../../../glob/list-files", () => ({ - listFiles: vi.fn(), -})) + } +} + +beforeEach(() => { + vi.useFakeTimers() + clearAllMocks() + const stats = new Stats() + stats.size = 100 + vi.mocked(stat).mockReset().mockResolvedValue(stats) + vi.mocked(listFiles) + .mockReset() + .mockResolvedValue([[fileA], false]) + vi.mocked(vscode.workspace.fs.readFile).mockReset().mockResolvedValue(Buffer.from("current content")) +}) +afterEach(() => { + restoreGlobals() + vi.useRealTimers() +}) describe("DirectoryScanner", () => { - let scanner: DirectoryScanner - let mockEmbedder: any - let mockVectorStore: any - let mockCodeParser: any - let mockCacheManager: any - let mockIgnoreInstance: any - let mockStats: any - - beforeEach(async () => { - mockEmbedder = { - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), - embedderInfo: { name: "mock-embedder", dimensions: 384 }, - } - mockVectorStore = { - upsertPoints: vi.fn().mockResolvedValue(undefined), - deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), - deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), - initialize: vi.fn().mockResolvedValue(true), - search: vi.fn().mockResolvedValue([]), - clearCollection: vi.fn().mockResolvedValue(undefined), - deleteCollection: vi.fn().mockResolvedValue(undefined), - collectionExists: vi.fn().mockResolvedValue(true), - } - mockCodeParser = { - parseFile: vi.fn().mockResolvedValue([]), - } - mockCacheManager = { - getHash: vi.fn().mockReturnValue(undefined), - getAllHashes: vi.fn().mockReturnValue({}), - updateHash: vi.fn().mockResolvedValue(undefined), - deleteHash: vi.fn().mockResolvedValue(undefined), - initialize: vi.fn().mockResolvedValue(undefined), - clearCacheFile: vi.fn().mockResolvedValue(undefined), - } - mockIgnoreInstance = { - ignores: vi.fn().mockReturnValue(false), - } - - scanner = new DirectoryScanner( - mockEmbedder, - mockVectorStore, - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, + it("preserves callback positions and completes only after all batches and verification", async () => { + const { scanner, parser, cache, store, points } = setup() + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const held = deferred() + store.upsertPoints.mockImplementationOnce(async (batch) => { + await held.promise + for (const point of batch) points.set(point.id, point) + }) + const onError = vi.fn() + const onIndexed = vi.fn() + const onParsed = vi.fn() + const pending = scanner.scanDirectory(workspacePath, onError, onIndexed, onParsed) + await vi.advanceTimersByTimeAsync(0) + expect(onParsed).toHaveBeenCalledExactlyOnceWith(3) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(onIndexed).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + held.resolve(undefined) + expect(await pending).toEqual({ stats: { processed: 1, skipped: 0 }, totalBlockCount: 3, incompleteFiles: [] }) + expect(onIndexed).toHaveBeenCalledExactlyOnceWith(3) + expect(onError).not.toHaveBeenCalled() + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(new Set([...points.keys()]).size).toBe(3) + expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileA, hash("current content")) + expect(cache.invalidateFile.mock.invocationCallOrder[0]).toBeLessThan( + store.upsertPoints.mock.invocationCallOrder[0], ) - - // Mock default implementations - create proper Stats object - mockStats = { - size: 1024, - isFile: () => true, - isDirectory: () => false, - isBlockDevice: () => false, - isCharacterDevice: () => false, - isSymbolicLink: () => false, - isFIFO: () => false, - isSocket: () => false, - dev: 0, - ino: 0, - mode: 0, - nlink: 0, - uid: 0, - gid: 0, - rdev: 0, - blksize: 0, - blocks: 0, - atimeMs: 0, - mtimeMs: 0, - ctimeMs: 0, - birthtimeMs: 0, - atime: new Date(), - mtime: new Date(), - ctime: new Date(), - birthtime: new Date(), - atimeNs: BigInt(0), - mtimeNs: BigInt(0), - ctimeNs: BigInt(0), - birthtimeNs: BigInt(0), - } - vi.mocked(stat).mockResolvedValue(mockStats) - - // Get and mock the listFiles function - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) + expect(store.countPointsForFile.mock.invocationCallOrder[0]).toBeLessThan( + cache.completeFile.mock.invocationCallOrder[0], + ) + expect(scanner.hasIncompleteFiles).toBe(false) }) - describe("scanDirectory", () => { - it("should skip files larger than MAX_FILE_SIZE_BYTES", async () => { - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) - - // Create large file mock stats - const largeFileStats = { - ...mockStats, - size: 2 * 1024 * 1024, // 2MB > 1MB limit - } - vi.mocked(stat).mockResolvedValueOnce(largeFileStats) - - const result = await scanner.scanDirectory("/test") - expect(result.stats.skipped).toBe(1) - expect(mockCodeParser.parseFile).not.toHaveBeenCalled() - }) - - it("should parse changed files and return empty codeBlocks array", async () => { - // Create scanner without embedder to test the non-embedding path - const scannerNoEmbeddings = new DirectoryScanner( - null as any, // No embedder - null as any, // No vector store - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - ) - - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) - const mockBlocks: any[] = [ - { - file_path: "test/file1.js", - content: "test content", - start_line: 1, - end_line: 5, - identifier: "test", - type: "function", - fileHash: "hash", - segmentHash: "segment-hash", - }, - ] - ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) - - const result = await scannerNoEmbeddings.scanDirectory("/test") - expect(result.stats.processed).toBe(1) - }) - - it("should process embeddings for new/changed files", async () => { - const mockBlocks: any[] = [ - { - file_path: "test/file1.js", - content: "test content", - start_line: 1, - end_line: 5, - identifier: "test", - type: "function", - fileHash: "hash", - segmentHash: "segment-hash", - }, - ] - ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) - - await scanner.scanDirectory("/test") - expect(mockEmbedder.createEmbeddings).toHaveBeenCalled() - expect(mockVectorStore.upsertPoints).toHaveBeenCalled() - }) - - it("should clean up a threshold-triggered batch", async () => { - const thresholdScanner = new DirectoryScanner( - mockEmbedder, - mockVectorStore, - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - 1, - ) - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) - mockCodeParser.parseFile.mockResolvedValue([ - { - file_path: "test/file1.js", - content: "test content", - start_line: 1, - end_line: 5, - identifier: "test", - type: "function", - fileHash: "hash", - segmentHash: "threshold-segment", - }, + it.each(["read", "parse"] as const)( + "records a single %s failure while unrelated files finish", + async (activity) => { + const { scanner, parser, failures, cache } = setup() + vi.mocked(listFiles).mockResolvedValue([[fileA, fileB], false]) + if (activity === "read") vi.mocked(stat).mockRejectedValueOnce(new Error("read failed")) + else parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) + const onError = vi.fn() + const result = await scanner.scanDirectory(workspacePath, onError) + expect(result.incompleteFiles).toEqual([fileA]) + expect(onError).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity, attempt: 1, state: "exhausted" }), ]) + expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileB, hash("current content")) + expect(parser.parseFile).toHaveBeenCalledTimes(activity === "read" ? 1 : 2) + }, + ) + + it("keeps unchanged parse failures exhausted across scanner recreation and renews changed content", async () => { + const { scanner, parser, failures, createScanner } = setup() + const recorded = vi.fn[0]>() + failures.onDidRecord(recorded) + const onError = vi.fn<(error: Error) => void>() + parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) + await scanner.scanDirectory(workspacePath, onError) + const repeated = createScanner() + const result = await repeated.scanDirectory(workspacePath, onError) + expect(result.incompleteFiles).toEqual([fileA]) + expect(repeated.hasIncompleteFiles).toBe(true) + expect(recorded).toHaveBeenCalledOnce() + expect(onError).toHaveBeenCalledTimes(2) + expect(parser.parseFile).toHaveBeenCalledOnce() + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("changed content")) + await createScanner().scanDirectory(workspacePath) + expect(parser.parseFile).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + }) - await thresholdScanner.scanDirectory("/test") - - expect(mockEmbedder.createEmbeddings).toHaveBeenCalledTimes(1) - expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1) - }) - - it("should delete points for removed files", async () => { - ;(mockCacheManager.getAllHashes as any).mockReturnValue({ "old/file.js": "old-hash" }) - - await scanner.scanDirectory("/test") - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("old/file.js") - expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("old/file.js") - }) - - it("should filter out files in hidden directories", async () => { - const { listFiles } = await import("../../../glob/list-files") - // Mock listFiles to return files including some in hidden directories - vi.mocked(listFiles).mockResolvedValue([ - [ - "test/file1.js", - "test/.hidden/file2.js", - ".git/config", - "src/.next/static/file3.js", - "normal/file4.js", - ], - false, - ]) - - // Mock parseFile to track which files are actually processed - const processedFiles: string[] = [] - ;(mockCodeParser.parseFile as any).mockImplementation((filePath: string) => { - processedFiles.push(filePath) - return [] - }) - - await scanner.scanDirectory("/test") - - // Verify that only non-hidden files were processed - expect(processedFiles).toEqual(["test/file1.js", "normal/file4.js"]) - expect(processedFiles).not.toContain("test/.hidden/file2.js") - expect(processedFiles).not.toContain(".git/config") - expect(processedFiles).not.toContain("src/.next/static/file3.js") - - // Verify the stats - expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(2) - }) - - it("should process markdown files alongside code files", async () => { - // Create scanner without embedder to test the non-embedding path - const scannerNoEmbeddings = new DirectoryScanner( - null as any, // No embedder - null as any, // No vector store - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - ) - - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/README.md", "test/app.js", "docs/guide.markdown"], false]) - - const mockMarkdownBlocks: any[] = [ - { - file_path: "test/README.md", - content: "# Introduction\nThis is a comprehensive guide...", - start_line: 1, - end_line: 10, - identifier: "Introduction", - type: "markdown_header_h1", - fileHash: "md-hash", - segmentHash: "md-segment-hash", - }, - ] - - const mockJsBlocks: any[] = [ - { - file_path: "test/app.js", - content: "function main() { return 'hello'; }", - start_line: 1, - end_line: 3, - identifier: "main", - type: "function", - fileHash: "js-hash", - segmentHash: "js-segment-hash", - }, - ] - - const mockMarkdownBlocks2: any[] = [ - { - file_path: "docs/guide.markdown", - content: "## Getting Started\nFollow these steps...", - start_line: 1, - end_line: 8, - identifier: "Getting Started", - type: "markdown_header_h2", - fileHash: "markdown-hash", - segmentHash: "markdown-segment-hash", - }, - ] - - // Mock parseFile to return different blocks based on file extension - ;(mockCodeParser.parseFile as any).mockImplementation((filePath: string) => { - if (filePath.endsWith(".md")) { - return mockMarkdownBlocks - } else if (filePath.endsWith(".markdown")) { - return mockMarkdownBlocks2 - } else if (filePath.endsWith(".js")) { - return mockJsBlocks - } - return [] - }) - - const result = await scannerNoEmbeddings.scanDirectory("/test") - - // Verify all files were processed - expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(3) - expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/README.md", expect.any(Object)) - expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/app.js", expect.any(Object)) - expect(mockCodeParser.parseFile).toHaveBeenCalledWith("docs/guide.markdown", expect.any(Object)) - - // Verify processing still works without codeBlocks accumulation - expect(result.stats.processed).toBe(3) - }) - - it("should generate unique point IDs for each block from the same file", async () => { - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/large-doc.md"], false]) - - // Mock multiple blocks from the same file with different segmentHash values - const mockBlocks: any[] = [ - { - file_path: "test/large-doc.md", - content: "# Introduction\nThis is the intro section...", - start_line: 1, - end_line: 10, - identifier: "Introduction", - type: "markdown_header_h1", - fileHash: "same-file-hash", - segmentHash: "unique-segment-hash-1", - }, - { - file_path: "test/large-doc.md", - content: "## Getting Started\nHere's how to begin...", - start_line: 11, - end_line: 20, - identifier: "Getting Started", - type: "markdown_header_h2", - fileHash: "same-file-hash", - segmentHash: "unique-segment-hash-2", - }, - { - file_path: "test/large-doc.md", - content: "## Advanced Topics\nFor advanced users...", - start_line: 21, - end_line: 30, - identifier: "Advanced Topics", - type: "markdown_header_h2", - fileHash: "same-file-hash", - segmentHash: "unique-segment-hash-3", - }, - ] - - ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) - - await scanner.scanDirectory("/test") - - // Verify that upsertPoints was called with unique IDs for each block - expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1) - const upsertCall = mockVectorStore.upsertPoints.mock.calls[0] - const points = upsertCall[0] + it.each(["embed", "upsert"] as const)( + "shares the %s retry allowance across batches and service recreation", + async (activity) => { + const { scanner, parser, embedder, store, failures, createScanner, cache } = setup() + parser.parseFile.mockResolvedValue(blocks(fileA, 2)) + const failure = Object.assign(new Error("unavailable"), { status: 503 }) + if (activity === "embed") + embedder.createEmbeddings + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ embeddings: [[1]] }) + .mockRejectedValue(failure) + else + store.upsertPoints + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce(undefined) + .mockRejectedValue(failure) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(1500) + expect((await pending).incompleteFiles).toEqual([fileA]) + const request = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints + expect(request).toHaveBeenCalledTimes(4) + expect(failures.getRetryBudget("a.ts", hash("current content"), activity).retriesUsed).toBe(2) + expect(failures.get("a.ts")).toMatchObject({ activity, attempt: 3, state: "exhausted" }) + await createScanner().scanDirectory(workspacePath) + expect(request).toHaveBeenCalledTimes(4) + expect(cache.completeFile).not.toHaveBeenCalled() + }, + ) + + it.each([400, 401, 403, 408, 409])("does not retry permanent HTTP %s failures", async (status) => { + const { scanner, embedder, failures } = setup() + embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("permanent"), { status })) + await scanner.scanDirectory(workspacePath) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(failures.get("a.ts")).toMatchObject({ activity: "embed", attempt: 1, state: "exhausted" }) + }) - // Extract the IDs from the points - const pointIds = points.map((point: any) => point.id) + it("keeps parser cancellation scoped to the session when a revision is superseded", async () => { + const { scanner, parser, failures, cache } = setup() + const controller = new AbortController() + const held = deferred() + parser.parseFile.mockReturnValueOnce(held.promise) + const pending = scanner.scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + await vi.advanceTimersByTimeAsync(0) + const parseSignal = parser.parseFile.mock.calls[0][1]?.signal + expect(parseSignal).toBe(controller.signal) + failures.renew("a.ts", "newer revision") + expect(parseSignal?.aborted).toBe(false) + held.resolve(blocks(fileA)) + await pending + expect(cache.completeFile).not.toHaveBeenCalled() + controller.abort() + expect(parseSignal?.aborted).toBe(true) + }) - // Verify all IDs are unique - expect(pointIds).toHaveLength(3) - expect(new Set(pointIds).size).toBe(3) // All IDs should be unique + it("preserves a fatal parser failure after the active revision was superseded", async () => { + const { scanner, parser, failures } = setup() + const held = deferred() + const fatal = new ParserFatalError("parse-timeout", fileA) + parser.parseFile.mockReturnValueOnce(held.promise) + const pending = scanner.scanDirectory(workspacePath).catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + failures.renew("a.ts", "newer revision") + held.reject(fatal) + expect(await pending).toBe(fatal) + }) - // Verify that each point has the correct payload - expect(points[0].payload.segmentHash).toBe("unique-segment-hash-1") - expect(points[1].payload.segmentHash).toBe("unique-segment-hash-2") - expect(points[2].payload.segmentHash).toBe("unique-segment-hash-3") - }) + it.each([60_000, 61_000])("paces repeated %s ms Retry-After without consuming retries", async (retryAfterMs) => { + const { scanner, embedder, failures, cache } = setup() + const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs }) + embedder.createEmbeddings.mockRejectedValueOnce(limited).mockRejectedValueOnce(limited) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(0) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync(retryAfterMs - 1) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync(retryAfterMs - 1) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1) + expect((await pending).incompleteFiles).toEqual([]) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(3) + expect(failures.getRetryBudget("a.ts", hash("current content"), "embed").retriesUsed).toBe(0) + expect(cache.completeFile).toHaveBeenCalledOnce() + }) - it("should stop processing files when signal is aborted", async () => { - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js", "test/file3.js"], false]) + it("retries saving without reembedding and reports grouped diagnostics", async () => { + const { scanner, embedder, store, failures } = setup() + store.upsertPoints.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) + expect((await pending).incompleteFiles).toEqual([]) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)?.value + expect(batch?.record).toHaveBeenCalledOnce() + expect(batch?.flush).toHaveBeenCalledOnce() + expect(codeIndexLog.info).toHaveBeenCalledWith( + "Scan summary", + expect.objectContaining({ failed: 0, blocksIndexed: 1 }), + ) + }) - // Create an already-aborted signal + it.each(["parse", "embed", "retry"] as const)( + "cancels during %s without recording cancellation or late completion", + async (phase) => { + const { scanner, parser, embedder, store, cache, failures } = setup() const controller = new AbortController() + const heldParse = deferred() + const heldEmbed = deferred<{ embeddings: number[][] }>() + if (phase === "parse") parser.parseFile.mockReturnValue(heldParse.promise) + else if (phase === "embed") embedder.createEmbeddings.mockReturnValue(heldEmbed.promise) + else + embedder.createEmbeddings.mockRejectedValue( + Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 }), + ) + const onError = vi.fn() + const pending = scanner + .scanDirectory(workspacePath, onError, undefined, undefined, controller.signal) + .catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) controller.abort() + heldParse.resolve(blocks(fileA)) + heldEmbed.resolve({ embeddings: [[1]] }) + expect(await pending).toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(60_000) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(phase === "parse" ? 0 : 1) + }, + ) + + it("rejects already cancelled scans before discovery or removal", async () => { + const { scanner, store } = setup() + const controller = new AbortController() + controller.abort() + await expect( + scanner.scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }) + expect(listFiles).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + }) - const result = await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) - - // No files should have been processed since signal was already aborted - expect(mockCodeParser.parseFile).not.toHaveBeenCalled() - expect(result.stats.processed).toBe(0) + it("propagates fatal parser identity immediately and stops queued dispatch", async () => { + const { scanner, parser, store } = setup() + const concurrency = Math.min(PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY) + const paths = Array.from({ length: concurrency + 2 }, (_, fileNumber) => + path.join(workspacePath, `${fileNumber}.ts`), + ) + vi.mocked(listFiles).mockResolvedValue([paths, false]) + const fatal = new ParserFatalError("parse-timeout", paths[0]) + const held = deferred() + parser.parseFile.mockImplementation(async (filePath) => { + if (filePath === paths[0]) throw fatal + return held.promise }) + const onError = vi.fn() + await expect(scanner.scanDirectory(workspacePath, onError)).rejects.toBe(fatal) + held.resolve([]) + await vi.advanceTimersByTimeAsync(0) + expect(parser.parseFile.mock.calls.length).toBeLessThanOrEqual(concurrency) + expect(onError).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + }) - it("should stop processing batches when signal is aborted mid-scan", async () => { - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) - - const controller = new AbortController() - - const mockBlocks: any[] = [ - { - file_path: "test/file1.js", - content: "function hello() {}", - start_line: 1, - end_line: 3, - identifier: "hello", - type: "function", - fileHash: "hash1", - segmentHash: "seg-hash-1", - }, - ] - - // Abort after first file is parsed - ;(mockCodeParser.parseFile as any).mockImplementation(async () => { - controller.abort() - return mockBlocks - }) - - // AbortError should propagate up (the orchestrator handles it in its catch block) - await expect( - scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), - ).rejects.toThrow("Indexing aborted") - }) + it.each(["success", "failure", "fatal"] as const)( + "fences late %s parsing from a superseded attempt", + async (outcome) => { + const { scanner, parser, supersede, cache, failures } = setup() + const held = deferred() + parser.parseFile.mockReturnValue(held.promise) + const onError = vi.fn() + const onParsed = vi.fn() + const pending = scanner + .scanDirectory(workspacePath, onError, undefined, onParsed) + .catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(0) + supersede() + if (outcome === "success") held.resolve(blocks(fileA)) + else + held.reject( + outcome === "fatal" ? new ParserFatalError("parser-abort", fileA) : new Error("late failure"), + ) + expect(await pending).toMatchObject({ name: "AbortError" }) + expect(failures.list()).toEqual([]) + expect(onError).not.toHaveBeenCalled() + expect(onParsed).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }, + ) + + it.each(["deleted", "excluded", "oversized"] as const)("fences removal of tracked %s files", async (selection) => { + const { scanner, hashes, cache, store, ignored } = setup() + hashes.set(fileA, "old hash") + if (selection === "deleted") vi.mocked(listFiles).mockResolvedValue([[], false]) + if (selection === "excluded") ignored.add("a.ts") + if (selection === "oversized") { + const stats = new Stats() + stats.size = MAX_FILE_SIZE_BYTES + 1 + vi.mocked(stat).mockResolvedValue(stats) + } + expect((await scanner.scanDirectory(workspacePath)).incompleteFiles).toEqual([]) + expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(fileA, expect.any(Number)) + expect(cache.invalidateFile).toHaveBeenCalledWith(fileA) + expect(cache.completeRemoval).toHaveBeenCalledWith(fileA) + }) - it("should not process deleted files when signal is aborted", async () => { - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([[], false]) + it("cleans registry-only deleted paths and requires explicit retry after failed removal", async () => { + const { scanner, failures, store } = setup() + failures.renew("a.ts", "unread") + failures.record("a.ts", "unread", "read", "read failed", 1, "exhausted") + vi.mocked(listFiles).mockResolvedValue([[], false]) + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + await scanner.scanDirectory(workspacePath) + await scanner.scanDirectory(workspacePath) + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + failures.resetAllBudgets() + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await pending + expect(failures.list()).toEqual([]) + }) - // Set up cached files that would normally be detected as deleted - ;(mockCacheManager.getAllHashes as any).mockReturnValue({ "old/file.js": "old-hash" }) + it.each(["empty", "whitespace"] as const)( + "completes %s files only after cleanup and verification", + async (kind) => { + const { scanner, parser, hashes, store, cache, embedder } = setup() + hashes.set(fileA, "old hash") + parser.parseFile.mockResolvedValue(kind === "empty" ? [] : [{ ...blocks(fileA)[0], content: " \n\t" }]) + const onIndexed = vi.fn() + expect((await scanner.scanDirectory(workspacePath, undefined, onIndexed)).incompleteFiles).toEqual([]) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(fileA, expect.any(Number)) + expect(store.countPointsForFile).toHaveBeenCalledOnce() + expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileA, hash("current content")) + expect(onIndexed).toHaveBeenCalledExactlyOnceWith(0) + }, + ) + + it("checks cancellation after the parsed callback before invalidating or embedding", async () => { + const { scanner, store, cache, embedder } = setup() + const controller = new AbortController() + await expect( + scanner.scanDirectory(workspacePath, undefined, undefined, () => controller.abort(), controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }) + expect(cache.invalidateFile).not.toHaveBeenCalled() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + }) - // Create an already-aborted signal - const controller = new AbortController() - controller.abort() + it("does not mark verification mismatches complete", async () => { + const { scanner, store, cache, failures } = setup() + store.countPointsForFile.mockResolvedValue({ current: 0, other: 1 }) + expect((await scanner.scanDirectory(workspacePath)).incompleteFiles).toEqual([fileA]) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(failures.get("a.ts")).toMatchObject({ activity: "verify", state: "exhausted" }) + }) - await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) + it("skips verified unchanged content but rechecks uncertain content", async () => { + const { scanner, hashes, parser, gate } = setup() + hashes.set(fileA, hash("current content")) + expect((await scanner.scanDirectory(workspacePath)).stats.skipped).toBe(1) + expect(parser.parseFile).not.toHaveBeenCalled() + gate.recover([fileA]) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect((await pending).incompleteFiles).toEqual([]) + expect(parser.parseFile).toHaveBeenCalledOnce() + }) - // Deleted file cleanup should not have run - expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() - }) + it("retains Zoo extension and hidden-directory filtering", async () => { + const { scanner, parser } = setup() + const selected = ["README.md", "guide.markdown", "app.js"].map((name) => path.join(workspacePath, name)) + vi.mocked(listFiles).mockResolvedValue([ + [ + ...selected, + path.join(workspacePath, ".hidden", "a.ts"), + path.join(workspacePath, "image.png"), + `${workspacePath}/directory/`, + ], + false, + ]) + await scanner.scanDirectory(workspacePath) + expect(parser.parseFile.mock.calls.map(([filePath]) => filePath)).toEqual(selected) }) }) diff --git a/src/services/code-index/processors/directory-scanner-factory.ts b/src/services/code-index/processors/directory-scanner-factory.ts index 099aa6d261..f3f3f3e20b 100644 --- a/src/services/code-index/processors/directory-scanner-factory.ts +++ b/src/services/code-index/processors/directory-scanner-factory.ts @@ -9,6 +9,12 @@ export class DirectoryScannerFactory implements IDirectoryScannerFactory { parser, cacheManager, ignoreInstance, + writeGate, + generations, + failures, + workspacePath, + isCurrentAttempt, + providerPacers, }: DirectoryScannerFactoryOptions): DirectoryScanner { return new DirectoryScanner( embedder, @@ -17,6 +23,12 @@ export class DirectoryScannerFactory implements IDirectoryScannerFactory { cacheManager, ignoreInstance, getEmbeddingBatchSize(), + writeGate, + generations, + failures, + workspacePath, + isCurrentAttempt, + providerPacers, ) } } diff --git a/src/services/code-index/processors/file-watcher-factory.ts b/src/services/code-index/processors/file-watcher-factory.ts index 2f4cc227b6..bedce2ff7e 100644 --- a/src/services/code-index/processors/file-watcher-factory.ts +++ b/src/services/code-index/processors/file-watcher-factory.ts @@ -12,6 +12,12 @@ export class FileWatcherFactory implements IFileWatcherFactory { vectorStore, ignoreInstance, rooIgnoreController, + parser, + writeGate, + generations, + failures, + isCurrentAttempt, + providerPacers, }: FileWatcherFactoryOptions): IFileWatcher { return new FileWatcher( workspacePath, @@ -22,6 +28,12 @@ export class FileWatcherFactory implements IFileWatcherFactory { ignoreInstance, rooIgnoreController, getEmbeddingBatchSize(), + parser, + writeGate, + generations, + failures, + isCurrentAttempt, + providerPacers, ) } } diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 0a0514a891..f5e71aecd5 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -1,38 +1,52 @@ import * as vscode from "vscode" -import { BATCH_SEGMENT_THRESHOLD, MAX_BATCH_RETRIES, INITIAL_RETRY_DELAY_MS } from "../constants" +import * as path from "path" +import { createHash } from "crypto" +import type { Ignore } from "ignore" +import { MAX_FILE_SIZE_BYTES, BATCH_SEGMENT_THRESHOLD } from "../constants" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" -import { Ignore } from "ignore" import { scannerExtensions } from "../shared/supported-extensions" -import { +import type { IFileWatcher, FileProcessingResult, IEmbedder, IVectorStore, - PointStruct, + ICodeParser, BatchProcessingSummary, } from "../interfaces" -import { codeParser } from "./parser" -import { FilePreparation } from "./file-preparation" -import { CacheManager } from "../cache-manager" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" -import { sanitizeErrorMessage } from "../shared/validation-helpers" +import { ParseWorkerClient } from "../parse-worker-client" +import { IndexWriteGate } from "../index-write-gate" +import { GenerationAllocator } from "../generation-allocator" +import { FileIndexWriter } from "../file-completion" +import { abortError, ParserFatalError, throwIfAborted } from "../parser-errors" +import type { CacheManager } from "../cache-manager" +import { generateRelativeFilePath } from "../shared/get-relative-path" +import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" import { Package } from "../../../shared/package" +import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" +import { codeIndexLog } from "../code-index-log" +import { ProviderPacerRegistry } from "../provider-pacer" + +type FileEvent = { uri: vscode.Uri; type: "create" | "change" | "delete" } -/** - * Implementation of the file watcher interface - */ export class FileWatcher implements IFileWatcher { - private readonly filePreparation: FilePreparation - private ignoreInstance?: Ignore private fileWatcher?: vscode.FileSystemWatcher - private ignoreController: RooIgnoreController - private accumulatedEvents: Map = new Map() + private readonly ignoreController: RooIgnoreController + private readonly accumulatedEvents = new Map() private batchProcessDebounceTimer?: NodeJS.Timeout private readonly BATCH_DEBOUNCE_DELAY_MS = 500 private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10 private readonly batchSegmentThreshold: number - + private readonly writer?: FileIndexWriter + private readonly pathControllers = new Map() + private readonly localAbortController = new AbortController() + private signal = this.localAbortController.signal + private attemptOwner = this.localAbortController.signal + private processingBatch = false + private activeFiles = 0 + private disposed = false + private fatalReported = false + private readonly parserFatal = new vscode.EventEmitter() + public readonly onParserFatal = this.parserFatal.event private readonly _onDidStartBatchProcessing = new vscode.EventEmitter() private readonly _onBatchProgressUpdate = new vscode.EventEmitter<{ processedInBatch: number @@ -40,55 +54,38 @@ export class FileWatcher implements IFileWatcher { currentFile?: string }>() private readonly _onDidFinishBatchProcessing = new vscode.EventEmitter() - - /** - * Event emitted when a batch of files begins processing - */ public readonly onDidStartBatchProcessing = this._onDidStartBatchProcessing.event - - /** - * Event emitted to report progress during batch processing - */ public readonly onBatchProgressUpdate = this._onBatchProgressUpdate.event - - /** - * Event emitted when a batch of files has finished processing - */ public readonly onDidFinishBatchProcessing = this._onDidFinishBatchProcessing.event - /** - * Creates a new file watcher - * @param workspacePath Path to the workspace - * @param context VS Code extension context - * @param embedder Optional embedder - * @param vectorStore Optional vector store - * @param cacheManager Cache manager - */ constructor( - private workspacePath: string, + private readonly workspacePath: string, _context: vscode.ExtensionContext, private readonly cacheManager: CacheManager, - private embedder?: IEmbedder, - private vectorStore?: IVectorStore, - ignoreInstance?: Ignore, + private readonly embedder?: IEmbedder, + vectorStore?: IVectorStore, + private readonly ignoreInstance?: Ignore, ignoreController?: RooIgnoreController, batchSegmentThreshold?: number, + private readonly parser: ICodeParser = new ParseWorkerClient(), + private readonly writeGate = new IndexWriteGate(), + generations = new GenerationAllocator(cacheManager), + private readonly failures = new FileFailureRegistry(), + private readonly isCurrentAttempt: () => boolean = () => true, + private readonly providerPacers = new ProviderPacerRegistry(), ) { - this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) - if (ignoreInstance) { - this.ignoreInstance = ignoreInstance + this.ignoreController = ignoreController ?? new RooIgnoreController(workspacePath) + if (vectorStore) { + this.writer = new FileIndexWriter( + cacheManager, + vectorStore, + writeGate, + generations, + workspacePath, + failures, + this.providerPacers, + ) } - this.filePreparation = new FilePreparation({ - workspacePath: this.workspacePath, - ignoreController: this.ignoreController, - ignoreInstance: this.ignoreInstance, - fileSystem: vscode.workspace.fs, - cacheManager: this.cacheManager, - parser: codeParser, - embedder: this.embedder, - }) - // Get the configurable batch size from VSCode settings, fallback to default - // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -97,417 +94,257 @@ export class FileWatcher implements IFileWatcher { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { - // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } } - /** - * Initializes the file watcher - */ - async initialize(): Promise { - // Create file watcher + public get isProcessing(): boolean { + return this.processingBatch || this.activeFiles > 0 + } + + private isLive(): boolean { + return !this.disposed && !this.signal.aborted && this.isCurrentAttempt() + } + + private checkActive(signal = this.signal): void { + throwIfAborted(signal) + if (!this.isLive()) throw abortError() + } + + async initialize(signal?: AbortSignal): Promise { + this.checkActive() + if (this.fileWatcher) return + this.attemptOwner = signal ?? this.localAbortController.signal + this.signal = signal + ? AbortSignal.any([signal, this.localAbortController.signal]) + : this.localAbortController.signal + this.checkActive() + this.writeGate.own(this.signal, this.attemptOwner) const filePattern = new vscode.RelativePattern( this.workspacePath, - `**/*{${scannerExtensions.map((e) => e.substring(1)).join(",")}}`, + `**/*{${scannerExtensions.map((extension) => extension.substring(1)).join(",")}}`, ) this.fileWatcher = vscode.workspace.createFileSystemWatcher(filePattern) - - // Register event handlers - this.fileWatcher.onDidCreate(this.handleFileCreated.bind(this)) - this.fileWatcher.onDidChange(this.handleFileChanged.bind(this)) - this.fileWatcher.onDidDelete(this.handleFileDeleted.bind(this)) + this.fileWatcher.onDidCreate((uri) => this.handleFileEvent(uri, "create")) + this.fileWatcher.onDidChange((uri) => this.handleFileEvent(uri, "change")) + this.fileWatcher.onDidDelete((uri) => this.handleFileEvent(uri, "delete")) } - /** - * Disposes the file watcher - */ dispose(): void { + if (this.disposed) return + this.disposed = true + this.localAbortController.abort() + this.parserFatal.dispose() this.fileWatcher?.dispose() - if (this.batchProcessDebounceTimer) { - clearTimeout(this.batchProcessDebounceTimer) - } + clearTimeout(this.batchProcessDebounceTimer) this._onDidStartBatchProcessing.dispose() this._onBatchProgressUpdate.dispose() this._onDidFinishBatchProcessing.dispose() this.accumulatedEvents.clear() } - /** - * Handles file creation events - * @param uri URI of the created file - */ - private async handleFileCreated(uri: vscode.Uri): Promise { - this.accumulatedEvents.set(uri.fsPath, { uri, type: "create" }) - this.scheduleBatchProcessing() - } - - /** - * Handles file change events - * @param uri URI of the changed file - */ - private async handleFileChanged(uri: vscode.Uri): Promise { - this.accumulatedEvents.set(uri.fsPath, { uri, type: "change" }) + private handleFileEvent(uri: vscode.Uri, type: FileEvent["type"]): void { + if (!this.isLive()) return + this.pathControllers.get(uri.fsPath)?.abort() + this.accumulatedEvents.set(uri.fsPath, { uri, type }) this.scheduleBatchProcessing() } - /** - * Handles file deletion events - * @param uri URI of the deleted file - */ - private async handleFileDeleted(uri: vscode.Uri): Promise { - this.accumulatedEvents.set(uri.fsPath, { uri, type: "delete" }) - this.scheduleBatchProcessing() - } - - /** - * Schedules batch processing with debounce - */ private scheduleBatchProcessing(): void { - if (this.batchProcessDebounceTimer) { - clearTimeout(this.batchProcessDebounceTimer) - } - this.batchProcessDebounceTimer = setTimeout(() => this.triggerBatchProcessing(), this.BATCH_DEBOUNCE_DELAY_MS) + if (!this.isLive()) return + clearTimeout(this.batchProcessDebounceTimer) + this.batchProcessDebounceTimer = setTimeout( + () => void this.triggerBatchProcessing(), + this.BATCH_DEBOUNCE_DELAY_MS, + ) } - /** - * Triggers processing of accumulated events - */ private async triggerBatchProcessing(): Promise { - if (this.accumulatedEvents.size === 0) { - return - } - + if (!this.isLive() || this.processingBatch || this.accumulatedEvents.size === 0) return const eventsToProcess = new Map(this.accumulatedEvents) this.accumulatedEvents.clear() - - const filePathsInBatch = Array.from(eventsToProcess.keys()) - this._onDidStartBatchProcessing.fire(filePathsInBatch) - - await this.processBatch(eventsToProcess) - } - - /** - * Processes a batch of accumulated events - * @param eventsToProcess Map of events to process - */ - private async _handleBatchDeletions( - batchResults: FileProcessingResult[], - processedCountInBatch: number, - totalFilesInBatch: number, - pathsToExplicitlyDelete: string[], - filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }>, - ): Promise<{ overallBatchError?: Error; clearedPaths: Set; processedCount: number }> { - let overallBatchError: Error | undefined - const allPathsToClearFromDB = new Set(pathsToExplicitlyDelete) - - for (const fileDetail of filesToUpsertDetails) { - if (fileDetail.originalType === "change") { - allPathsToClearFromDB.add(fileDetail.path) - } - } - - if (allPathsToClearFromDB.size > 0 && this.vectorStore) { - try { - await this.vectorStore.deletePointsByMultipleFilePaths(Array.from(allPathsToClearFromDB)) - - for (const path of pathsToExplicitlyDelete) { - this.cacheManager.deleteHash(path) - batchResults.push({ path, status: "success" }) - processedCountInBatch++ - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: path, - }) - } - } catch (error: any) { - const errorStatus = error?.status || error?.response?.status || error?.statusCode - const errorMessage = error instanceof Error ? error.message : String(error) - - // Log telemetry for deletion error - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(errorMessage), - location: "deletePointsByMultipleFilePaths", - errorType: "deletion_error", - errorStatus: errorStatus, - }) - - // Mark all paths as error - overallBatchError = error as Error - for (const path of pathsToExplicitlyDelete) { - batchResults.push({ path, status: "error", error: error as Error }) - processedCountInBatch++ - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: path, - }) - } + this.processingBatch = true + try { + this._onDidStartBatchProcessing.fire([...eventsToProcess.keys()]) + await this.processBatch(eventsToProcess) + } catch (error) { + if (!this.isLive()) return + if (error instanceof ParserFatalError) this.reportFatal(error) + else { + this._onDidFinishBatchProcessing.fire({ processedFiles: [], batchError: this.toError(error) }) } + } finally { + this.processingBatch = false + if (this.accumulatedEvents.size > 0) this.scheduleBatchProcessing() } - - return { overallBatchError, clearedPaths: allPathsToClearFromDB, processedCount: processedCountInBatch } } - private async _processFilesAndPrepareUpserts( - filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }>, - batchResults: FileProcessingResult[], - processedCountInBatch: number, - totalFilesInBatch: number, - pathsToExplicitlyDelete: string[], - ): Promise<{ - pointsForBatchUpsert: PointStruct[] - successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }> - processedCount: number - }> { - const pointsForBatchUpsert: PointStruct[] = [] - const successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }> = [] - const filesToProcessConcurrently = [...filesToUpsertDetails] - - for (let i = 0; i < filesToProcessConcurrently.length; i += this.FILE_PROCESSING_CONCURRENCY_LIMIT) { - const chunkToProcess = filesToProcessConcurrently.slice(i, i + this.FILE_PROCESSING_CONCURRENCY_LIMIT) - - const chunkProcessingPromises = chunkToProcess.map(async (fileDetail) => { - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: fileDetail.path, - }) - try { - const result = await this.processFile(fileDetail.path) - return { path: fileDetail.path, result: result, error: undefined } - } catch (e) { - const error = e as Error - console.error(`[FileWatcher] Unhandled exception processing file ${fileDetail.path}:`, e) - return { path: fileDetail.path, result: undefined, error: error } - } - }) - - const settledChunkResults = await Promise.allSettled(chunkProcessingPromises) - - for (const settledResult of settledChunkResults) { - let resultPath: string | undefined - - if (settledResult.status === "fulfilled") { - const { path, result, error: directError } = settledResult.value - resultPath = path - - if (directError) { - batchResults.push({ path, status: "error", error: directError }) - } else if (result) { - if (result.status === "skipped" || result.status === "local_error") { - batchResults.push(result) - } else if (result.status === "processed_for_batching" && result.pointsToUpsert) { - pointsForBatchUpsert.push(...result.pointsToUpsert) - if (result.path && result.newHash) { - successfullyProcessedForUpsert.push({ path: result.path, newHash: result.newHash }) - } else if (result.path && !result.newHash) { - successfullyProcessedForUpsert.push({ path: result.path }) - } - } else { - batchResults.push({ - path, - status: "error", - error: new Error( - `Unexpected result status from processFile: ${result.status} for file ${path}`, - ), - }) - } - } else { - batchResults.push({ - path, - status: "error", - error: new Error(`Fulfilled promise with no result or error for file ${path}`), + private async processBatch(eventsToProcess: Map): Promise { + this.checkActive() + const events = [...eventsToProcess.values()] + const failureBatch = codeIndexLog.createFailureBatch({ source: "watcher" }) + const subscription = this.failures.onDidRecord((failure) => { + if (this.isLive()) failureBatch.record({ ...failure, willRetry: failure.state === "retrying" }) + }) + try { + const results: FileProcessingResult[] = [] + let nextEvent = 0 + let completed = 0 + this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: events.length }) + const processNext = async () => { + while (nextEvent < events.length) { + this.checkActive() + const event = events[nextEvent++] + const result = + event.type === "delete" + ? await this.removeFile(event.uri.fsPath) + : await this.processFile(event.uri.fsPath) + this.checkActive() + results.push(result) + if (result.status === "success" || result.status === "skipped") { + completed++ + this._onBatchProgressUpdate.fire({ + processedInBatch: completed, + totalInBatch: events.length, + currentFile: result.path, }) } - } else { - const error = settledResult.reason as Error - const rejectedPath = (settledResult.reason as any)?.path || "unknown" - console.error("[FileWatcher] A file processing promise was rejected:", settledResult.reason) - batchResults.push({ - path: rejectedPath, - status: "error", - error: error, - }) - } - - if (!pathsToExplicitlyDelete.includes(resultPath || "")) { - processedCountInBatch++ } - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: resultPath, - }) } - } - - return { - pointsForBatchUpsert, - successfullyProcessedForUpsert, - processedCount: processedCountInBatch, + // Complete each file independently rather than waiting for a cross-file flush. + await Promise.all( + Array.from({ length: Math.min(events.length, this.FILE_PROCESSING_CONCURRENCY_LIMIT) }, processNext), + ) + this.checkActive() + const errors = results.filter((result) => result.status === "local_error") + this._onDidFinishBatchProcessing.fire({ + processedFiles: results, + batchError: errors.length ? new Error(`${errors.length} file changes could not be indexed`) : undefined, + }) + } finally { + subscription.dispose() + failureBatch.flush() } } - private async _executeBatchUpsertOperations( - pointsForBatchUpsert: PointStruct[], - successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }>, - batchResults: FileProcessingResult[], - overallBatchError?: Error, - ): Promise { - if (pointsForBatchUpsert.length > 0 && this.vectorStore && !overallBatchError) { - try { - for (let i = 0; i < pointsForBatchUpsert.length; i += this.batchSegmentThreshold) { - const batch = pointsForBatchUpsert.slice(i, i + this.batchSegmentThreshold) - let retryCount = 0 - let upsertError: Error | undefined - - while (retryCount < MAX_BATCH_RETRIES) { - try { - await this.vectorStore.upsertPoints(batch) - break - } catch (error) { - upsertError = error as Error - retryCount++ - if (retryCount === MAX_BATCH_RETRIES) { - // Log telemetry for upsert failure - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(upsertError.message), - location: "upsertPoints", - errorType: "upsert_retry_exhausted", - retryCount: MAX_BATCH_RETRIES, - }) - throw new Error( - `Failed to upsert batch after ${MAX_BATCH_RETRIES} retries: ${upsertError.message}`, - ) - } - await new Promise((resolve) => - setTimeout(resolve, INITIAL_RETRY_DELAY_MS * Math.pow(2, retryCount - 1)), - ) - } - } - } - - for (const { path, newHash } of successfullyProcessedForUpsert) { - if (newHash) { - this.cacheManager.updateHash(path, newHash) - } - batchResults.push({ path, status: "success" }) - } - } catch (error) { - const err = error as Error - overallBatchError = overallBatchError || err - // Log telemetry for batch upsert error - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(err.message), - location: "executeBatchUpsertOperations", - errorType: "batch_upsert_error", - affectedFiles: successfullyProcessedForUpsert.length, - }) - for (const { path } of successfullyProcessedForUpsert) { - batchResults.push({ path, status: "error", error: err }) - } - } - } else if (overallBatchError && pointsForBatchUpsert.length > 0) { - for (const { path } of successfullyProcessedForUpsert) { - batchResults.push({ path, status: "error", error: overallBatchError }) - } - } - - return overallBatchError + private reportFatal(error: ParserFatalError): void { + if (this.fatalReported || !this.isLive()) return + this.fatalReported = true + this.parserFatal.fire(error) + this.dispose() } - private async processBatch( - eventsToProcess: Map, - ): Promise { - const batchResults: FileProcessingResult[] = [] - let processedCountInBatch = 0 - const totalFilesInBatch = eventsToProcess.size - let overallBatchError: Error | undefined - - // Initial progress update - this._onBatchProgressUpdate.fire({ - processedInBatch: 0, - totalInBatch: totalFilesInBatch, - currentFile: undefined, - }) - - // Categorize events - const pathsToExplicitlyDelete: string[] = [] - const filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }> = [] + private toError(error: unknown): Error { + return new Error(codeIndexLog.redact(error instanceof Error ? error.message : String(error)), { cause: error }) + } - for (const event of eventsToProcess.values()) { - if (event.type === "delete") { - pathsToExplicitlyDelete.push(event.uri.fsPath) - } else { - filesToUpsertDetails.push({ - path: event.uri.fsPath, - uri: event.uri, - originalType: event.type, - }) - } + private async removeFile(filePath: string, signal = this.signal): Promise { + this.checkActive(signal) + const controller = signal === this.signal ? new AbortController() : undefined + if (controller) { + this.pathControllers.get(filePath)?.abort() + this.pathControllers.set(filePath, controller) + signal = AbortSignal.any([signal, controller.signal]) } - - // Phase 1: Handle deletions - const { overallBatchError: deletionError, processedCount: deletionCount } = await this._handleBatchDeletions( - batchResults, - processedCountInBatch, - totalFilesInBatch, - pathsToExplicitlyDelete, - filesToUpsertDetails, - ) - overallBatchError = deletionError - processedCountInBatch = deletionCount - - // Phase 2: Process files and prepare upserts - const { - pointsForBatchUpsert, - successfullyProcessedForUpsert, - processedCount: upsertCount, - } = await this._processFilesAndPrepareUpserts( - filesToUpsertDetails, - batchResults, - processedCountInBatch, - totalFilesInBatch, - pathsToExplicitlyDelete, - ) - processedCountInBatch = upsertCount - - // Phase 3: Execute batch upsert - overallBatchError = await this._executeBatchUpsertOperations( - pointsForBatchUpsert, - successfullyProcessedForUpsert, - batchResults, - overallBatchError, - ) - - // Finalize - this._onDidFinishBatchProcessing.fire({ - processedFiles: batchResults, - batchError: overallBatchError, - }) - this._onBatchProgressUpdate.fire({ - processedInBatch: totalFilesInBatch, - totalInBatch: totalFilesInBatch, - }) - - if (this.accumulatedEvents.size === 0) { - this._onBatchProgressUpdate.fire({ - processedInBatch: 0, - totalInBatch: 0, - currentFile: undefined, - }) + try { + this.checkActive(signal) + if ( + !this.cacheManager.isTracked(filePath) && + !this.failures.has(generateRelativeFilePath(filePath, this.workspacePath)) + ) { + return { path: filePath, status: "skipped", reason: "File is not tracked" } + } + if (!this.writer) throw new Error("Vector store is not configured") + await this.writer.remove(filePath, signal, this.attemptOwner) + this.checkActive(signal) + return { path: filePath, status: "success" } + } catch (error) { + this.checkActive() + if (signal.aborted) return { path: filePath, status: "skipped", reason: "File change was superseded" } + if (error instanceof Error && error.name === "AbortError") throw error + return { path: filePath, status: "local_error", error: this.toError(error) } + } finally { + if (controller && this.pathControllers.get(filePath) === controller) this.pathControllers.delete(filePath) } } - /** - * Processes a file - * @param filePath Path to the file to process - * @returns Promise resolving to processing result - */ async processFile(filePath: string): Promise { - return this.filePreparation.prepareFile(filePath) + this.checkActive() + this.activeFiles++ + this.pathControllers.get(filePath)?.abort() + const controller = new AbortController() + this.pathControllers.set(filePath, controller) + let signal = AbortSignal.any([this.signal, controller.signal]) + const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) + let revision = this.failures.getRevision(relativeFilePath) ?? UNKNOWN_REVISION + let activity: FileFailureActivity = "read" + try { + this.failures.renew(relativeFilePath, revision) + if ( + !scannerExtensions.includes(path.extname(filePath).toLowerCase()) || + isPathInIgnoredDirectory(relativeFilePath) || + !this.ignoreController.validateAccess(filePath) || + this.ignoreInstance?.ignores(relativeFilePath) + ) { + return await this.removeFile(filePath, signal) + } + const fileStat = await vscode.workspace.fs.stat(vscode.Uri.file(filePath)) + this.checkActive(signal) + if (fileStat.size > MAX_FILE_SIZE_BYTES) return await this.removeFile(filePath, signal) + const fileContent = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) + this.checkActive(signal) + const content = Buffer.from(fileContent).toString("utf8") + const newHash = createHash("sha256").update(content).digest("hex") + revision = newHash + signal = AbortSignal.any([signal, this.failures.renew(relativeFilePath, newHash)]) + this.writeGate.own(signal, this.attemptOwner) + if ( + this.cacheManager.getHash(filePath) === newHash && + !this.writeGate.needsReconciliation(filePath) && + !this.failures.has(relativeFilePath) + ) { + return { path: filePath, status: "skipped", reason: "File has not changed" } + } + activity = "embed" + if (!this.writer || !this.embedder) throw new Error("Indexing services are not configured") + this.failures.assertRetryable(relativeFilePath, revision) + activity = "parse" + // Cancelling an active parse terminates the shared worker, not just this file's request. + const blocks = await this.parser.parseFile(filePath, { content, fileHash: newHash, signal: this.signal }) + this.checkActive(signal) + activity = "embed" + await this.writer.replace( + filePath, + newHash, + blocks, + this.embedder, + signal, + this.batchSegmentThreshold, + undefined, + this.attemptOwner, + ) + this.checkActive(signal) + return { path: filePath, status: "success" } + } catch (error) { + this.checkActive() + if ( + (activity === "read" || activity === "parse") && + !signal.aborted && + !(error instanceof Error && error.name === "AbortError") + ) { + this.failures.record(relativeFilePath, revision, activity, this.toError(error).message, 1, "exhausted") + } + if (error instanceof ParserFatalError) { + this.reportFatal(error) + throw error + } + if (signal.aborted) return { path: filePath, status: "skipped", reason: "File change was superseded" } + if (error instanceof Error && error.name === "AbortError") throw error + return { path: filePath, status: "local_error", error: this.toError(error) } + } finally { + if (this.pathControllers.get(filePath) === controller) this.pathControllers.delete(filePath) + this.activeFiles-- + } } } diff --git a/src/services/code-index/processors/parser-constants.ts b/src/services/code-index/processors/parser-constants.ts new file mode 100644 index 0000000000..ec82fee326 --- /dev/null +++ b/src/services/code-index/processors/parser-constants.ts @@ -0,0 +1,5 @@ +// Keep worker parsing independent of the host/types barrel in ../constants. +export const MAX_BLOCK_CHARS = 1000 +export const MIN_BLOCK_CHARS = 50 +export const MIN_CHUNK_REMAINDER_CHARS = 200 +export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 8445ea1971..84eeb301c8 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -1,15 +1,68 @@ +import { codeIndexLog } from "../code-index-log" import { readFile } from "fs/promises" import { createHash } from "crypto" import * as path from "path" -import { Node } from "web-tree-sitter" -import { LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser" +import type { Node } from "web-tree-sitter" +import { type LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser" import { parseMarkdown } from "../../tree-sitter/markdownParser" -import { ICodeParser, CodeBlock } from "../interfaces" -import { scannerExtensions, shouldUseFallbackChunking } from "../shared/supported-extensions" -import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_TOLERANCE_FACTOR } from "../constants" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" -import { sanitizeErrorMessage } from "../shared/validation-helpers" +import type { ICodeParser, CodeBlock } from "../parser-types" +import { isFallbackExtension as shouldUseFallbackChunking } from "../../shared/fallback-extensions" +import { + MAX_BLOCK_CHARS, + MIN_BLOCK_CHARS, + MIN_CHUNK_REMAINDER_CHARS, + MAX_CHARS_TOLERANCE_FACTOR, +} from "./parser-constants" +import { isParserAbort, throwIfAborted } from "../parser-errors" + +// The scanner's extension barrel imports host-only code; keep workers independent of it. +const parserExtensions = new Set( + [ + "tla", + "js", + "jsx", + "ts", + "vue", + "tsx", + "py", + "rs", + "go", + "c", + "h", + "cpp", + "hpp", + "cs", + "rb", + "java", + "php", + "swift", + "sol", + "kt", + "kts", + "ex", + "exs", + "el", + "html", + "htm", + "md", + "markdown", + "txt", + "json", + "css", + "rdl", + "ml", + "mli", + "lua", + "scala", + "toml", + "zig", + "elm", + "ejs", + "erb", + "vb", + "dart", + ].map((extension) => `.${extension}`), +) /** * Implementation of the code parser interface @@ -17,8 +70,17 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers" export class CodeParser implements ICodeParser { private loadedParsers: LanguageParser = {} private pendingLoads: Map> = new Map() - // Markdown files are now supported using the custom markdown parser - // which extracts headers and sections for semantic indexing + public fallbackNeeded = false + + constructor( + private readonly parseInWorker?: ( + filePath: string, + content: string, + fileHash: string, + signal?: AbortSignal, + ) => Promise, + private readonly wasmDirectory?: string, + ) {} /** * Parses a code file into code blocks @@ -31,8 +93,11 @@ export class CodeParser implements ICodeParser { options?: { content?: string fileHash?: string + signal?: AbortSignal }, ): Promise { + this.fallbackNeeded = false + throwIfAborted(options?.signal) // Get file extension const ext = path.extname(filePath).toLowerCase() @@ -53,17 +118,16 @@ export class CodeParser implements ICodeParser { content = await readFile(filePath, "utf8") fileHash = this.createFileHash(content) } catch (error) { - console.error(`Error reading file ${filePath}:`, error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "parseFile", - }) + throwIfAborted(options?.signal) + codeIndexLog.error(`Error reading file ${filePath}:`, error) return [] } } - // Parse the file + throwIfAborted(options?.signal) + if (this.parseInWorker && ext !== ".md" && ext !== ".markdown" && !shouldUseFallbackChunking(ext)) { + return this.parseInWorker(filePath, content, fileHash, options?.signal) + } return this.parseContent(filePath, content, fileHash) } @@ -73,7 +137,7 @@ export class CodeParser implements ICodeParser { * @returns Boolean indicating if the language is supported */ private isSupportedLanguage(extension: string): boolean { - return scannerExtensions.includes(extension) + return parserExtensions.has(extension) } /** @@ -85,13 +149,10 @@ export class CodeParser implements ICodeParser { return createHash("sha256").update(content).digest("hex") } - /** - * Parses file content into code blocks - * @param filePath Path to the file - * @param content File content - * @param fileHash File hash - * @returns Array of code blocks - */ + public fallbackContent(filePath: string, content: string, fileHash: string): CodeBlock[] { + return this._performFallbackChunking(filePath, content, fileHash, new Set()) + } + private async parseContent(filePath: string, content: string, fileHash: string): Promise { const ext = path.extname(filePath).slice(1).toLowerCase() const seenSegmentHashes = new Set() @@ -113,16 +174,12 @@ export class CodeParser implements ICodeParser { try { await pendingLoad } catch (error) { - console.error(`Error in pending parser load for ${filePath}:`, error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "parseContent:loadParser", - }) + if (isParserAbort(error)) throw error + codeIndexLog.error(`Error in pending parser load for ${filePath}:`, error) return [] } } else { - const loadPromise = loadRequiredLanguageParsers([filePath]) + const loadPromise = loadRequiredLanguageParsers([filePath], this.wasmDirectory) this.pendingLoads.set(ext, loadPromise) try { const newParsers = await loadPromise @@ -130,12 +187,8 @@ export class CodeParser implements ICodeParser { this.loadedParsers = { ...this.loadedParsers, ...newParsers } } } catch (error) { - console.error(`Error loading language parser for ${filePath}:`, error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "parseContent:loadParser", - }) + if (isParserAbort(error)) throw error + codeIndexLog.error(`Error loading language parser for ${filePath}:`, error) return [] } finally { this.pendingLoads.delete(ext) @@ -145,59 +198,39 @@ export class CodeParser implements ICodeParser { const language = this.loadedParsers[ext] if (!language) { - console.warn(`No parser available for file extension: ${ext}`) + codeIndexLog.warn(`No parser available for file extension: ${ext}`) return [] } const tree = language.parser.parse(content) - // We don't need to get the query string from languageQueries since it's already loaded - // in the language object - const captures = tree ? language.query.captures(tree.rootNode) : [] - - // Check if captures are empty - if (captures.length === 0) { - if (content.length >= MIN_BLOCK_CHARS) { - // Perform fallback chunking if content is large enough - const blocks = this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) - return blocks - } else { - // Return empty if content is too small for fallback - return [] + try { + const captures = tree ? language.query.captures(tree.rootNode) : [] + if (captures.length === 0) { + if (this.wasmDirectory) { + this.fallbackNeeded = true + return [] + } + return this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) } - } - - const results: CodeBlock[] = [] - - // Process captures if not empty - const queue: Node[] = Array.from(captures).map((capture) => capture.node) - while (queue.length > 0) { - const currentNode = queue.shift()! - // const lineSpan = currentNode.endPosition.row - currentNode.startPosition.row + 1 // Removed as per lint error + const results: CodeBlock[] = [] + const queue: Node[] = Array.from(captures).map((capture) => capture.node) - // Check if the node meets the minimum character requirement - if (currentNode.text.length >= MIN_BLOCK_CHARS) { - // If it also exceeds the maximum character limit, try to break it down + for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) { + const currentNode = queue[queueIndex] + if (currentNode.text.length < MIN_BLOCK_CHARS) continue if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) { - if (currentNode.children.filter((child) => child !== null).length > 0) { - // If it has children, process them instead - queue.push(...currentNode.children.filter((child) => child !== null)) + const children = currentNode.children.filter((child) => child !== null) + if (children.length > 0) { + queue.push(...children) } else { - // If it's a leaf node, chunk it - const chunkedBlocks = this._chunkLeafNodeByLines( - currentNode, - filePath, - fileHash, - seenSegmentHashes, - ) - results.push(...chunkedBlocks) + results.push(...this._chunkLeafNodeByLines(currentNode, filePath, fileHash, seenSegmentHashes)) } } else { - // Node meets min chars and is within max chars, create a block const identifier = currentNode.childForFieldName("name")?.text || - currentNode.children.find((c) => c?.type === "identifier")?.text || + currentNode.children.find((child) => child?.type === "identifier")?.text || null const type = currentNode.type const start_line = currentNode.startPosition.row + 1 @@ -223,10 +256,11 @@ export class CodeParser implements ICodeParser { } } } - // Nodes smaller than minBlockChars are ignored - } - return results + return results + } finally { + tree?.delete() + } } /** @@ -245,6 +279,8 @@ export class CodeParser implements ICodeParser { let currentChunkLength = 0 let chunkStartLineIndex = 0 // 0-based index within the `lines` array const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR + const lineOffsets = [0] + for (const line of lines) lineOffsets.push(lineOffsets[lineOffsets.length - 1] + line.length + 1) const finalizeChunk = (endLineIndex: number) => { if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) { @@ -310,14 +346,22 @@ export class CodeParser implements ICodeParser { finalizeChunk(i - 1) } - // Split the oversized line into segments - let remainingLineContent = line let currentSegmentStartChar = 0 - while (remainingLineContent.length > 0) { - const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS) - remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS) + while (currentSegmentStartChar < line.length) { + let segmentEnd = Math.min(currentSegmentStartChar + MAX_BLOCK_CHARS, line.length) + const precedingUnit = line.charCodeAt(segmentEnd - 1) + const followingUnit = line.charCodeAt(segmentEnd) + if ( + precedingUnit >= 0xd800 && + precedingUnit <= 0xdbff && + followingUnit >= 0xdc00 && + followingUnit <= 0xdfff + ) { + segmentEnd-- + } + const segment = line.slice(currentSegmentStartChar, segmentEnd) createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar) - currentSegmentStartChar += MAX_BLOCK_CHARS + currentSegmentStartChar = segmentEnd } // Update chunkStartLineIndex to continue processing from the next line chunkStartLineIndex = i + 1 @@ -328,10 +372,7 @@ export class CodeParser implements ICodeParser { if (currentChunkLength > 0 && currentChunkLength + lineLength > effectiveMaxChars) { // Re-balancing Logic let splitIndex = i - 1 - let remainderLength = 0 - for (let j = i; j < lines.length; j++) { - remainderLength += lines[j].length + (j < lines.length - 1 ? 1 : 0) - } + const remainderLength = lineOffsets[lines.length] - lineOffsets[i] - 1 if ( currentChunkLength >= MIN_BLOCK_CHARS && @@ -339,10 +380,8 @@ export class CodeParser implements ICodeParser { currentChunkLines.length > 1 ) { for (let k = i - 2; k >= chunkStartLineIndex; k--) { - const potentialChunkLines = lines.slice(chunkStartLineIndex, k + 1) - const potentialChunkLength = potentialChunkLines.join("\n").length + 1 - const potentialNextChunkLines = lines.slice(k + 1) - const potentialNextChunkLength = potentialNextChunkLines.join("\n").length + 1 + const potentialChunkLength = lineOffsets[k + 1] - lineOffsets[chunkStartLineIndex] + const potentialNextChunkLength = lineOffsets[lines.length] - lineOffsets[k + 1] if ( potentialChunkLength >= MIN_BLOCK_CHARS && diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 4504e02e9b..bac59bc96c 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -1,38 +1,35 @@ import { listFiles } from "../../glob/list-files" -import { Ignore } from "ignore" +import type { Ignore } from "ignore" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { stat } from "fs/promises" import * as path from "path" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { generateRelativeFilePath } from "../shared/get-relative-path" import { getWorkspacePathForContext } from "../../../utils/path" import { scannerExtensions } from "../shared/supported-extensions" import * as vscode from "vscode" -import { CodeBlock, ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" +import type { ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" import { createHash } from "crypto" -import { v5 as uuidv5 } from "uuid" -import pLimit from "p-limit" -import { Mutex } from "async-mutex" -import { CacheManager } from "../cache-manager" -import { t } from "../../../i18n" +import type { CacheManager } from "../cache-manager" import { - QDRANT_CODE_BLOCK_NAMESPACE, MAX_FILE_SIZE_BYTES, MAX_LIST_FILES_LIMIT_CODE_INDEX, BATCH_SEGMENT_THRESHOLD, - MAX_BATCH_RETRIES, - INITIAL_RETRY_DELAY_MS, PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, - MAX_PENDING_BATCHES, } from "../constants" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" -import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" +import { IndexWriteGate } from "../index-write-gate" +import { abortError, ParserFatalError, throwIfAborted } from "../parser-errors" +import { GenerationAllocator } from "../generation-allocator" +import { FileIndexWriter } from "../file-completion" +import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" +import { codeIndexLog } from "../code-index-log" +import { ProviderPacerRegistry } from "../provider-pacer" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number + private incompleteFiles = new Set() constructor( private readonly embedder: IEmbedder, @@ -41,9 +38,13 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, + private readonly writeGate = new IndexWriteGate(), + private readonly generations = new GenerationAllocator(cacheManager), + private readonly failures = new FileFailureRegistry(), + private readonly workspacePath?: string, + private readonly isCurrentAttempt: () => boolean = () => true, + private readonly providerPacers = new ProviderPacerRegistry(), ) { - // Get the configurable batch size from VSCode settings, fallback to default - // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -52,467 +53,232 @@ export class DirectoryScanner implements IDirectoryScanner { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { - // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } } - /** - * Recursively scans a directory for code blocks in supported files. - * @param directoryPath The directory to scan - * @param rooIgnoreController Optional RooIgnoreController instance for filtering - * @param context VS Code ExtensionContext for cache storage - * @param onError Optional error handler callback - * @returns Promise<{codeBlocks: CodeBlock[], stats: {processed: number, skipped: number}}> Array of parsed code blocks and processing stats - */ + public get hasIncompleteFiles(): boolean { + return this.incompleteFiles.size > 0 || this.failures.list().length > 0 + } + public async scanDirectory( directory: string, onError?: (error: Error) => void, onBlocksIndexed?: (indexedCount: number) => void, onFileParsed?: (fileBlockCount: number) => void, - signal?: AbortSignal, - ): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> { - const directoryPath = directory - // Capture workspace context at scan start - const scanWorkspace = getWorkspacePathForContext(directoryPath) - - // Get all files recursively (handles .gitignore automatically) - const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) - - // Filter out directories (marked with trailing '/') - const filePaths = allPaths.filter((p) => !p.endsWith("/")) - - // Initialize RooIgnoreController if not provided - const ignoreController = new RooIgnoreController(directoryPath) - - await ignoreController.initialize() - - // Filter paths using .rooignore - const allowedPaths = ignoreController.filterPaths(filePaths) - - // Filter by supported extensions, ignore patterns, and excluded directories - const supportedPaths = allowedPaths.filter((filePath) => { - const ext = path.extname(filePath).toLowerCase() - const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace) - - // Check if file is in an ignored directory using the shared helper - // Use relative path to avoid matching parent directories outside the workspace - if (isPathInIgnoredDirectory(relativeFilePath)) { - return false - } - - return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath) - }) - - // Initialize tracking variables - const processedFiles = new Set() + signal = new AbortController().signal, + ): Promise<{ + stats: { processed: number; skipped: number } + totalBlockCount: number + incompleteFiles?: string[] + }> { + const startedAt = Date.now() let processedCount = 0 let skippedCount = 0 - - // Initialize parallel processing tools - const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing - const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing - const mutex = new Mutex() - - // Shared batch accumulators (protected by mutex) - let currentBatchBlocks: CodeBlock[] = [] - let currentBatchTexts: string[] = [] - let currentBatchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[] = [] - const activeBatchPromises = new Set>() - let pendingBatchCount = 0 - - // Initialize block counter let totalBlockCount = 0 + let indexedBlockCount = 0 + let partialReason: string | undefined + let fatalError: ParserFatalError | undefined + const checkActive = () => { + throwIfAborted(signal) + if (!this.isCurrentAttempt()) throw abortError() + if (fatalError) throw fatalError + } + checkActive() + const failureBatch = codeIndexLog.createFailureBatch({ source: "scan" }) + const subscription = this.failures.onDidRecord((failure) => { + if (!signal.aborted && this.isCurrentAttempt()) { + failureBatch.record({ ...failure, willRetry: failure.state === "retrying" }) + } + }) + try { + const incompleteFiles = new Set() + this.incompleteFiles = incompleteFiles + const scanWorkspace = this.workspacePath ?? getWorkspacePathForContext(directory) + const [allPaths] = await listFiles(directory, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) + checkActive() + const ignoreController = new RooIgnoreController(directory) + await ignoreController.initialize() + checkActive() + const allowedPaths = ignoreController.filterPaths(allPaths.filter((filePath) => !filePath.endsWith("/"))) + const selectedPaths = new Set( + allowedPaths.filter((filePath) => { + const relativePath = generateRelativeFilePath(filePath, scanWorkspace) + return ( + !isPathInIgnoredDirectory(relativePath) && + scannerExtensions.includes(path.extname(filePath).toLowerCase()) && + !this.ignoreInstance.ignores(relativePath) + ) + }), + ) + // Discovery precedes dispatch so stopping during a read or parse still reports partial work. + for (const filePath of selectedPaths) incompleteFiles.add(filePath) + const trackedPaths = new Set([ + ...this.cacheManager.getTrackedPaths(), + ...this.failures.list().map((failure) => path.resolve(scanWorkspace, failure.path)), + ]) + for (const filePath of trackedPaths) { + if (!selectedPaths.has(filePath)) incompleteFiles.add(filePath) + } - // Process all files in parallel with concurrency control - const parsePromises = supportedPaths.map((filePath) => - parseLimiter(async () => { - // Check abort signal before processing each file - if (signal?.aborted) return - + const writer = new FileIndexWriter( + this.cacheManager, + this.qdrantClient, + this.writeGate, + this.generations, + scanWorkspace, + this.failures, + this.providerPacers, + ) + const reportError = (error: unknown, filePath: string) => { + checkActive() + if (error instanceof ParserFatalError) { + fatalError = error + throw error + } + if (error instanceof Error && error.name === "AbortError") throw error + const message = codeIndexLog.redact(error instanceof Error ? error.message : String(error)) + const relativePath = generateRelativeFilePath(filePath, scanWorkspace) + onError?.(new Error(`${message} (File: ${relativePath})`, { cause: error })) + } + const processFile = async (filePath: string) => { + checkActive() + const relativePath = generateRelativeFilePath(filePath, scanWorkspace) + let revision = this.failures.getRevision(relativePath) ?? UNKNOWN_REVISION + this.failures.renew(relativePath, revision) + let activity: FileFailureActivity = "read" + let fileSignal = signal try { - // Check file size const stats = await stat(filePath) + checkActive() if (stats.size > MAX_FILE_SIZE_BYTES) { - skippedCount++ // Skip large files + selectedPaths.delete(filePath) + if (!trackedPaths.has(filePath)) incompleteFiles.delete(filePath) + skippedCount++ return } - - // Read file content - const content = await vscode.workspace.fs - .readFile(vscode.Uri.file(filePath)) - .then((buffer) => Buffer.from(buffer).toString("utf-8")) - - // Calculate current hash - const currentFileHash = createHash("sha256").update(content).digest("hex") - processedFiles.add(filePath) - - // Check against cache - const cachedFileHash = this.cacheManager.getHash(filePath) - const isNewFile = !cachedFileHash - if (cachedFileHash === currentFileHash) { - // File is unchanged + const buffer = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) + checkActive() + const content = Buffer.from(buffer).toString("utf-8") + const hash = createHash("sha256").update(content).digest("hex") + revision = hash + fileSignal = AbortSignal.any([signal, this.failures.renew(relativePath, hash)]) + this.writeGate.own(fileSignal, signal) + if ( + this.cacheManager.getHash(filePath) === hash && + !this.writeGate.needsReconciliation(filePath) && + !this.failures.has(relativePath) + ) { + incompleteFiles.delete(filePath) skippedCount++ return } - - // File is new or changed - parse it using the injected parser function - const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash }) - const fileBlockCount = blocks.length - onFileParsed?.(fileBlockCount) + activity = "embed" + this.failures.assertRetryable(relativePath, hash) + activity = "parse" + // Cancelling an active parse terminates the shared worker, so only the session may cancel it. + const blocks = await this.codeParser.parseFile(filePath, { + content, + fileHash: hash, + signal, + }) + throwIfAborted(fileSignal) + checkActive() processedCount++ - - // Process embeddings if configured - if (this.embedder && this.qdrantClient && blocks.length > 0) { - // Add to batch accumulators - let addedBlocksFromFile = false - for (const block of blocks) { - const trimmedContent = block.content.trim() - if (trimmedContent) { - const release = await mutex.acquire() - try { - currentBatchBlocks.push(block) - currentBatchTexts.push(trimmedContent) - addedBlocksFromFile = true - - // Check if batch threshold is met - // Check abort signal before dispatching batch - if (signal?.aborted) { - throw new DOMException("Indexing aborted", "AbortError") - } - - if (currentBatchBlocks.length >= this.batchSegmentThreshold) { - // Wait if we've reached the maximum pending batches - while (pendingBatchCount >= MAX_PENDING_BATCHES) { - if (signal?.aborted) { - throw new DOMException("Indexing aborted", "AbortError") - } - await Promise.race(activeBatchPromises) - } - - // Copy current batch data and clear accumulators - const batchBlocks = [...currentBatchBlocks] - const batchTexts = [...currentBatchTexts] - const batchFileInfos = [...currentBatchFileInfos] - currentBatchBlocks = [] - currentBatchTexts = [] - currentBatchFileInfos = [] - - // Increment pending batch count - pendingBatchCount++ - - // Queue batch processing - const batchPromise = batchLimiter(() => - this.processBatch( - batchBlocks, - batchTexts, - batchFileInfos, - scanWorkspace, - onError, - onBlocksIndexed, - ), - ) - activeBatchPromises.add(batchPromise) - - // Clean up completed promises to prevent memory accumulation - const cleanupBatch = () => { - activeBatchPromises.delete(batchPromise) - pendingBatchCount-- - } - void batchPromise.then(cleanupBatch, cleanupBatch) - } - } finally { - release() - } - } - } - - // Add file info once per file (outside the block loop) - if (addedBlocksFromFile) { - const release = await mutex.acquire() - try { - totalBlockCount += fileBlockCount - currentBatchFileInfos.push({ - filePath, - fileHash: currentFileHash, - isNew: isNewFile, - }) - } finally { - release() - } - } - } else { - // Only update hash if not being processed in a batch - await this.cacheManager.updateHash(filePath, currentFileHash) - } + const selectedBlockCount = blocks.filter((block) => block.content.trim()).length + onFileParsed?.(selectedBlockCount) + checkActive() + if (!this.embedder || !this.qdrantClient) return + totalBlockCount += selectedBlockCount + activity = "embed" + await writer.replace( + filePath, + hash, + blocks, + this.embedder, + fileSignal, + this.batchSegmentThreshold, + (count) => { + checkActive() + incompleteFiles.delete(filePath) + indexedBlockCount += count + onBlocksIndexed?.(count) + }, + signal, + ) } catch (error) { - // Re-throw AbortError — it's not a file processing error, just a user-initiated stop - if (error instanceof DOMException && error.name === "AbortError") { - throw error - } - console.error(`Error processing file ${filePath} in workspace ${scanWorkspace}:`, error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "scanDirectory:processFile", - }) - if (onError) { - onError( - error instanceof Error - ? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`) - : new Error( - t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) + - ` (Workspace: ${scanWorkspace})`, - ), + checkActive() + if ( + (activity === "read" || activity === "parse") && + !fileSignal.aborted && + !(error instanceof Error && error.name === "AbortError") + ) { + this.failures.record( + relativePath, + revision, + activity, + codeIndexLog.redact(error instanceof Error ? error.message : String(error)), + 1, + "exhausted", ) } + if (error instanceof ParserFatalError) reportError(error, filePath) + if (fileSignal.aborted) return + reportError(error, filePath) } - }), - ) - - // Wait for all parsing to complete - await Promise.all(parsePromises) - - // Check abort signal before processing remaining batch - if (signal?.aborted) { - return { - stats: { - processed: processedCount, - skipped: skippedCount, - }, - totalBlockCount, } - } - - // Process any remaining items in batch - if (currentBatchBlocks.length > 0) { - const release = await mutex.acquire() - try { - // Copy current batch data and clear accumulators - const batchBlocks = [...currentBatchBlocks] - const batchTexts = [...currentBatchTexts] - const batchFileInfos = [...currentBatchFileInfos] - currentBatchBlocks = [] - currentBatchTexts = [] - currentBatchFileInfos = [] - - // Increment pending batch count for final batch - pendingBatchCount++ - - // Queue final batch processing - const batchPromise = batchLimiter(() => - this.processBatch(batchBlocks, batchTexts, batchFileInfos, scanWorkspace, onError, onBlocksIndexed), - ) - activeBatchPromises.add(batchPromise) - - // Clean up completed promises to prevent memory accumulation - const cleanupBatch = () => { - activeBatchPromises.delete(batchPromise) - pendingBatchCount-- + const processPaths = async (paths: string[], process: (filePath: string) => Promise) => { + let nextPath = 0 + const worker = async () => { + while (nextPath < paths.length) { + checkActive() + const filePath = paths[nextPath++] + await process(filePath) + } } - void batchPromise.then(cleanupBatch, cleanupBatch) - } finally { - release() + const concurrency = Math.min(PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, paths.length) + await Promise.all(Array.from({ length: concurrency }, worker)) } - } - - // Wait for all batch processing to complete - await Promise.all(activeBatchPromises) - - // Check abort signal before handling deleted files - if (signal?.aborted) { + await processPaths([...selectedPaths], processFile) + checkActive() + + const removedPaths = [...new Set([...trackedPaths, ...this.cacheManager.getTrackedPaths()])].filter( + (filePath) => !selectedPaths.has(filePath), + ) + for (const filePath of removedPaths) incompleteFiles.add(filePath) + await processPaths(removedPaths, async (filePath) => { + try { + checkActive() + if (!this.qdrantClient) return + await writer.remove(filePath, signal) + checkActive() + incompleteFiles.delete(filePath) + } catch (error) { + reportError(error, filePath) + } + }) + checkActive() return { - stats: { - processed: processedCount, - skipped: skippedCount, - }, + stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount, + incompleteFiles: [...incompleteFiles], } - } - - // Handle deleted files - const oldHashes = this.cacheManager.getAllHashes() - for (const cachedFilePath of Object.keys(oldHashes)) { - if (!processedFiles.has(cachedFilePath)) { - // File was deleted or is no longer supported/indexed - if (this.qdrantClient) { - try { - await this.qdrantClient.deletePointsByFilePath(cachedFilePath) - await this.cacheManager.deleteHash(cachedFilePath) - } catch (error: any) { - const errorStatus = error?.status || error?.response?.status || error?.statusCode - const errorMessage = error instanceof Error ? error.message : String(error) - - console.error( - `[DirectoryScanner] Failed to delete points for ${cachedFilePath} in workspace ${scanWorkspace}:`, - error, - ) - - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(errorMessage), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "scanDirectory:deleteRemovedFiles", - errorStatus: errorStatus, - }) - - if (onError) { - // Report error to error handler - onError( - error instanceof Error - ? new Error( - `${error.message} (Workspace: ${scanWorkspace}, File: ${cachedFilePath})`, - ) - : new Error( - t("embeddings:scanner.unknownErrorDeletingPoints", { - filePath: cachedFilePath, - }) + ` (Workspace: ${scanWorkspace})`, - ), - ) - } - // Log error and continue processing instead of re-throwing - console.error(`Failed to delete points for removed file: ${cachedFilePath}`, error) - } - } - } - } - - return { - stats: { + } catch (error) { + partialReason = error instanceof Error ? error.message : String(error) + throw error + } finally { + subscription.dispose() + failureBatch.flush() + codeIndexLog.info("Scan summary", { processed: processedCount, skipped: skippedCount, - }, - totalBlockCount, - } - } - - private async processBatch( - batchBlocks: CodeBlock[], - batchTexts: string[], - batchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[], - scanWorkspace: string, - onError?: (error: Error) => void, - onBlocksIndexed?: (indexedCount: number) => void, - ): Promise { - if (batchBlocks.length === 0) return - - let attempts = 0 - let success = false - let lastError: Error | null = null - - while (attempts < MAX_BATCH_RETRIES && !success) { - attempts++ - try { - // --- Deletion Step --- - const uniqueFilePaths = [ - ...new Set( - batchFileInfos - .filter((info) => !info.isNew) // Only modified files (not new) - .map((info) => info.filePath), - ), - ] - if (uniqueFilePaths.length > 0) { - try { - await this.qdrantClient.deletePointsByMultipleFilePaths(uniqueFilePaths) - } catch (deleteError: any) { - const errorStatus = - deleteError?.status || deleteError?.response?.status || deleteError?.statusCode - const errorMessage = deleteError instanceof Error ? deleteError.message : String(deleteError) - - console.error( - `[DirectoryScanner] Failed to delete points for ${uniqueFilePaths.length} files before upsert in workspace ${scanWorkspace}:`, - deleteError, - ) - - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(errorMessage), - stack: - deleteError instanceof Error - ? sanitizeErrorMessage(deleteError.stack || "") - : undefined, - location: "processBatch:deletePointsByMultipleFilePaths", - fileCount: uniqueFilePaths.length, - errorStatus: errorStatus, - }) - - // Re-throw with workspace context - throw new Error( - `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`, - { cause: deleteError }, - ) - } - } - // --- End Deletion Step --- - - // Create embeddings for batch - const { embeddings } = await this.embedder.createEmbeddings(batchTexts) - - // Prepare points for Qdrant - const points = batchBlocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, scanWorkspace) - - // Use segmentHash for unique ID generation to handle multiple segments from same line - const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, scanWorkspace), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - segmentHash: block.segmentHash, - }, - } - }) - - // Upsert points to Qdrant - await this.qdrantClient.upsertPoints(points) - onBlocksIndexed?.(batchBlocks.length) - - // Update hashes for successfully processed files in this batch - for (const fileInfo of batchFileInfos) { - await this.cacheManager.updateHash(fileInfo.filePath, fileInfo.fileHash) - } - success = true - } catch (error) { - lastError = error as Error - console.error( - `[DirectoryScanner] Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}:`, - error, - ) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), - stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, - location: "processBatch:retry", - attemptNumber: attempts, - batchSize: batchBlocks.length, - }) - - if (attempts < MAX_BATCH_RETRIES) { - const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1) - await new Promise((resolve) => setTimeout(resolve, delay)) - } - } - } - - if (!success && lastError) { - console.error(`[DirectoryScanner] Failed to process batch after ${MAX_BATCH_RETRIES} attempts`) - if (onError) { - // Preserve the original error message from embedders which now have detailed i18n messages - const errorMessage = lastError.message || "Unknown error" - - // For other errors, provide context - onError( - new Error( - t("embeddings:scanner.failedToProcessBatchWithError", { - maxRetries: MAX_BATCH_RETRIES, - errorMessage, - }), - ), - ) - } + failed: this.failures.list().length, + blocksFound: totalBlockCount, + blocksIndexed: indexedBlockCount, + elapsedMs: Date.now() - startedAt, + partialReason: partialReason ?? (this.hasIncompleteFiles ? "Unresolved files" : undefined), + }) } } } diff --git a/src/services/code-index/provider-pacer.ts b/src/services/code-index/provider-pacer.ts new file mode 100644 index 0000000000..d754951ad2 --- /dev/null +++ b/src/services/code-index/provider-pacer.ts @@ -0,0 +1,339 @@ +import { createHash } from "node:crypto" +import type { Disposable, Event } from "vscode" +import { + BATCH_PROCESSING_CONCURRENCY, + PACER_INCREASE_AFTER, + PACER_MAX_BACKOFF_MS, + PACER_RESUME_SPACING_MS, +} from "./constants" +import { codeIndexLog } from "./code-index-log" +import { classifyIndexingError, EmbeddingError } from "./shared/indexing-errors" + +const MAX_TIMEOUT_MS = 2 ** 31 - 1 + +export interface ProviderPacerState { + pausedUntil?: number + waitingPaths: string[] + limit: number + episodeSeq: number + issueSeq: number + consecutiveRateLimits: number +} + +export interface ProviderWait { + service: string + resumeAt?: number + waitingFiles: string[] +} + +type Waiter = { + path?: string + signal?: AbortSignal + resolve: (permit: Permit) => void + reject: (error: unknown) => void + onAbort: () => void +} + +type Permit = { probeEpisode?: number } + +function serviceLabel(scope: string): string { + const prefix = scope.split(":", 1)[0] + return /^[a-z][a-z0-9-]{0,63}$/i.test(prefix) ? prefix : "provider" +} + +export function makeRateLimitScope(provider: string, baseUrl?: string, apiKey?: string, extra?: string): string { + let normalizedUrl = "" + if (baseUrl) { + const parsedUrl = new URL(baseUrl) + parsedUrl.username = "" + parsedUrl.password = "" + parsedUrl.hash = "" + normalizedUrl = parsedUrl.toString().replace(/\/+$/, "") + } + const keyHash = apiKey ? createHash("sha256").update(apiKey).digest("hex") : "" + // Hash the tuple as well: configured URLs can contain query-string credentials. + const allowanceHash = createHash("sha256") + .update(JSON.stringify([provider, normalizedUrl, keyHash, extra ?? ""])) + .digest("hex") + return `${serviceLabel(provider)}:${allowanceHash}` +} + +export class ProviderPacer { + readonly service: string + private nextIssueSeq = 0 + private episodeSeq = 0 + private consecutiveRateLimits = 0 + private successes = 0 + private limit = BATCH_PROCESSING_CONCURRENCY + private pausedUntil = 0 + private active = 0 + private needsProbe = false + private probeEpisode?: number + private probeGeneration = 0 + private stagedRemaining = 0 + private nextReleaseAt = 0 + private timer?: ReturnType + private readonly waiters = new Set() + private readonly changes = new Set<() => void>() + + constructor(scope: string) { + this.service = serviceLabel(scope) + } + + readonly onDidChange: Event = (listener, thisArgument, disposables) => { + const callback = () => listener.call(thisArgument, undefined) + this.changes.add(callback) + const subscription: Disposable = { + dispose: () => { + this.changes.delete(callback) + }, + } + disposables?.push(subscription) + return subscription + } + + state(): ProviderPacerState { + return { + pausedUntil: this.pausedUntil > Date.now() ? this.pausedUntil : undefined, + waitingPaths: [ + ...new Set([...this.waiters].flatMap((waiter) => (waiter.path === undefined ? [] : [waiter.path]))), + ], + limit: this.limit, + episodeSeq: this.episodeSeq, + issueSeq: this.nextIssueSeq, + consecutiveRateLimits: this.consecutiveRateLimits, + } + } + + get hasWaitingRequests(): boolean { + return this.waiters.size > 0 + } + + markIssued(): number { + return this.nextIssueSeq++ + } + + reportRateLimit(error: unknown, issueSeq: number): void { + const now = Date.now() + const paused = now < this.pausedUntil + const stale = issueSeq < this.episodeSeq + const requestedDelay = new EmbeddingError("Provider rate limited", error).retryAfterMs + const providerDelay = + typeof requestedDelay === "number" && Number.isFinite(requestedDelay) && requestedDelay > 0 + ? requestedDelay + : undefined + if (stale && providerDelay === undefined) return + const newEpisode = !paused && !stale + if (newEpisode) { + this.episodeSeq = this.nextIssueSeq + this.consecutiveRateLimits++ + this.limit = Math.max(1, Math.floor(this.limit / 2)) + this.successes = 0 + } + const base = Math.min( + PACER_MAX_BACKOFF_MS, + 1000 * 2 ** Math.min(30, Math.max(0, this.consecutiveRateLimits - 1)), + ) + const delay = providerDelay ?? base / 2 + (Math.random() * base) / 2 + const extendedUntil = Math.max(this.pausedUntil, now + delay) + if (!newEpisode && extendedUntil === this.pausedUntil) return + this.pausedUntil = extendedUntil + this.needsProbe = true + this.probeGeneration++ + this.probeEpisode = undefined + this.stagedRemaining = 0 + this.nextReleaseAt = 0 + codeIndexLog.info("Provider pacing wait", { + service: this.service, + delayMs: this.pausedUntil - now, + source: providerDelay === undefined ? "backoff" : "provider", + waitingCount: this.state().waitingPaths.length, + }) + this.changed() + this.pump() + } + + async run(path: string | undefined, signal: AbortSignal | undefined, request: () => Promise): Promise { + while (true) { + const permit = await this.acquire(path, signal) + let rateLimited = false + const onAbort = () => this.finishProbe(permit) + signal?.addEventListener("abort", onAbort, { once: true }) + try { + signal?.throwIfAborted() + // Another completion may pause the allowance before this granted permit is consumed. + if (Date.now() < this.pausedUntil) continue + const issueSeq = this.markIssued() + try { + const result = await request() + if (!signal?.aborted && issueSeq >= this.episodeSeq) { + this.consecutiveRateLimits = 0 + if (++this.successes >= PACER_INCREASE_AFTER) { + this.successes = 0 + this.limit = Math.min(BATCH_PROCESSING_CONCURRENCY, this.limit + 1) + this.changed() + } + } + return result + } catch (error) { + if (issueSeq >= this.episodeSeq) this.successes = 0 + if (classifyIndexingError(error, signal) !== "rate-limit") throw error + rateLimited = true + this.reportRateLimit(error, issueSeq) + } + } finally { + signal?.removeEventListener("abort", onAbort) + this.active-- + if (!rateLimited) this.finishProbe(permit) + this.pump() + } + } + } + + private acquire(path: string | undefined, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason) + return + } + const waiter: Waiter = { + path, + signal, + resolve, + reject, + onAbort: () => { + this.waiters.delete(waiter) + signal?.removeEventListener("abort", waiter.onAbort) + reject(signal?.reason) + this.changed() + this.pump() + }, + } + this.waiters.add(waiter) + signal?.addEventListener("abort", waiter.onAbort, { once: true }) + this.changed() + this.pump() + }) + } + + private finishProbe(permit: Permit): void { + if (permit.probeEpisode === undefined || permit.probeEpisode !== this.probeEpisode) return + this.probeEpisode = undefined + this.stagedRemaining = this.limit - 1 + this.nextReleaseAt = Date.now() + this.resumeGap() + this.pump() + } + + private resumeGap(): number { + return PACER_RESUME_SPACING_MS * (0.5 + Math.random()) + } + + private pump(): void { + if (this.timer !== undefined) clearTimeout(this.timer) + this.timer = undefined + const now = Date.now() + if (now < this.pausedUntil) { + this.schedule(this.pausedUntil - now) + return + } + if (this.pausedUntil !== 0) { + this.pausedUntil = 0 + this.changed() + } + if (this.probeEpisode !== undefined || this.waiters.size === 0 || this.active >= this.limit) return + if (now < this.nextReleaseAt) { + this.schedule(this.nextReleaseAt - now) + return + } + for (const waiter of this.waiters) { + if (this.active >= this.limit) break + this.waiters.delete(waiter) + waiter.signal?.removeEventListener("abort", waiter.onAbort) + const permit: Permit = {} + if (this.needsProbe) { + this.needsProbe = false + this.probeEpisode = this.probeGeneration + permit.probeEpisode = this.probeEpisode + } + this.active++ + waiter.resolve(permit) + this.changed() + if (permit.probeEpisode !== undefined) break + if (this.stagedRemaining > 0) { + this.stagedRemaining-- + this.nextReleaseAt = now + this.resumeGap() + if (this.waiters.size > 0) this.schedule(this.nextReleaseAt - now) + break + } + } + } + + private schedule(delay: number): void { + // Node clamps oversized timeouts to 1 ms; retain the full deadline and wait in chunks. + this.timer = setTimeout( + () => { + this.timer = undefined + this.pump() + }, + Math.min(MAX_TIMEOUT_MS, Math.ceil(delay)), + ) + } + + private changed(): void { + for (const listener of this.changes) listener() + } +} + +export class ProviderPacerRegistry { + private readonly pacers = new Map() + private readonly changes = new Set<() => void>() + + readonly onDidChange: Event = (listener, thisArgument, disposables) => { + const callback = () => listener.call(thisArgument, undefined) + this.changes.add(callback) + const subscription: Disposable = { + dispose: () => { + this.changes.delete(callback) + }, + } + disposables?.push(subscription) + return subscription + } + + get(scope: string): ProviderPacer { + let pacer = this.pacers.get(scope) + if (!pacer) { + pacer = new ProviderPacer(scope) + pacer.onDidChange(() => { + for (const listener of this.changes) listener() + }) + this.pacers.set(scope, pacer) + } + return pacer + } + + providerWaits(): ProviderWait[] { + return [...this.pacers.values()].flatMap((pacer) => { + const state = pacer.state() + return state.pausedUntil === undefined + ? [] + : [ + { + service: pacer.service, + resumeAt: state.pausedUntil, + waitingFiles: state.waitingPaths, + }, + ] + }) + } + + hasWaitingRequests(): boolean { + return [...this.pacers.values()].some( + (pacer) => pacer.hasWaitingRequests && pacer.state().pausedUntil !== undefined, + ) + } + + waitingPaths(): string[] { + return [...new Set([...this.pacers.values()].flatMap((pacer) => pacer.state().waitingPaths))] + } +} diff --git a/src/services/code-index/search-service.ts b/src/services/code-index/search-service.ts index a56f5cc674..6479fd5197 100644 --- a/src/services/code-index/search-service.ts +++ b/src/services/code-index/search-service.ts @@ -1,32 +1,49 @@ import * as path from "path" -import { VectorStoreSearchResult } from "./interfaces" -import { IEmbedder } from "./interfaces/embedder" -import { IVectorStore } from "./interfaces/vector-store" -import { CodeIndexConfigManager } from "./config-manager" -import { CodeIndexStateManager } from "./state-manager" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" +import type { VectorStoreSearchResult } from "./interfaces/vector-store" +import type { IEmbedder } from "./interfaces/embedder" +import type { IVectorStore } from "./interfaces/vector-store" +import type { CodeIndexConfigManager } from "./config-manager" +import type { CodeIndexStateManager } from "./state-manager" +import { codeIndexLog } from "./code-index-log" +import type { FileFailureRegistry } from "./file-failure-registry" +import { retryWithPolicy } from "./shared/retry-policy" +import { ProviderPacerRegistry } from "./provider-pacer" +import { classifyIndexingError, SearchProviderRateLimitError } from "./shared/indexing-errors" +import { t } from "../../i18n" + +export interface CodeIndexSearchResponse { + results: VectorStoreSearchResult[] + incomplete: boolean + failedPaths: string[] + waitingPaths: string[] +} /** * Service responsible for searching the code index. */ export class CodeIndexSearchService { constructor( - private readonly configManager: CodeIndexConfigManager, - private readonly stateManager: CodeIndexStateManager, - private readonly embedder: IEmbedder, - private readonly vectorStore: IVectorStore, + private readonly configManager: Pick< + CodeIndexConfigManager, + "isFeatureEnabled" | "isFeatureConfigured" | "currentSearchMinScore" | "currentSearchMaxResults" + >, + private readonly stateManager: Pick, + private readonly embedder: Pick, + private readonly vectorStore: Pick, + private readonly failures: FileFailureRegistry, + private readonly providerPacers: ProviderPacerRegistry, + private readonly getCurrentStatus: () => Pick< + ReturnType, + "systemStatus" + > = () => this.stateManager.getCurrentStatus(), ) {} - /** - * Searches the code index for relevant content. - * @param query The search query - * @param limit Maximum number of results to return - * @param directoryPrefix Optional directory path to filter results by - * @returns Array of search results - * @throws Error if the service is not properly configured or ready - */ - public async searchIndex(query: string, directoryPrefix?: string): Promise { + public async searchIndex( + query: string, + directoryPrefix?: string, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted() if (!this.configManager.isFeatureEnabled || !this.configManager.isFeatureConfigured) { throw new Error("Code index feature is disabled or not configured.") } @@ -34,41 +51,59 @@ export class CodeIndexSearchService { const minScore = this.configManager.currentSearchMinScore const maxResults = this.configManager.currentSearchMaxResults - const currentState = this.stateManager.getCurrentStatus().systemStatus - if (currentState !== "Indexed" && currentState !== "Indexing") { - // Allow search during Indexing too - throw new Error(`Code index is not ready for search. Current state: ${currentState}`) - } - try { - // Generate embedding for query - const embeddingResponse = await this.embedder.createEmbeddings([query]) + let currentState = this.getCurrentStatus().systemStatus + let partialStandby = false + if ( + currentState === "Standby" && + (this.failures.list().length > 0 || this.providerPacers.waitingPaths().length > 0) + ) { + partialStandby = await this.vectorStore.collectionExists() + signal?.throwIfAborted() + currentState = this.getCurrentStatus().systemStatus + partialStandby &&= + currentState === "Standby" && + (this.failures.list().length > 0 || this.providerPacers.waitingPaths().length > 0) + } + if (currentState !== "Indexed" && currentState !== "Indexing" && !partialStandby) { + throw new Error(`Code index is not ready for search. Current state: ${currentState}`) + } + + const pacer = this.providerPacers.get(this.embedder.rateLimitScope) + const embeddingResponse = await retryWithPolicy( + async () => { + const issueSeq = pacer.markIssued() + try { + return await this.embedder.createEmbeddings([query], undefined, signal) + } catch (error) { + signal?.throwIfAborted() + if (classifyIndexingError(error) !== "rate-limit") throw error + pacer.reportRateLimit(error, issueSeq) + throw new SearchProviderRateLimitError(t("embeddings:searchProviderRateLimited"), error) + } + }, + { signal, budget: { retriesUsed: 0 } }, + ) + signal?.throwIfAborted() const vector = embeddingResponse?.embeddings[0] if (!vector) { throw new Error("Failed to generate embedding for query.") } - // Handle directory prefix let normalizedPrefix: string | undefined = undefined if (directoryPrefix) { normalizedPrefix = path.normalize(directoryPrefix) } - // Perform search const results = await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) - return results + signal?.throwIfAborted() + const failedPaths = this.failures.list().map((failure) => failure.path) + const waitingPaths = this.providerPacers.waitingPaths() + return { results, incomplete: failedPaths.length > 0 || waitingPaths.length > 0, failedPaths, waitingPaths } } catch (error) { - console.error("[CodeIndexSearchService] Error during search:", error) - this.stateManager.setSystemState("Error", `Search failed: ${(error as Error).message}`) - - // Capture telemetry for the error - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: (error as Error).message, - stack: (error as Error).stack, - location: "searchIndex", - }) - - throw error // Re-throw the error after setting state + signal?.throwIfAborted() + codeIndexLog.error("Search failed", { error }) + throw error } } } diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index d2bbe43f2a..0ee62730b9 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -8,10 +8,15 @@ import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import { EmbedderFactory } from "./embedders/embedder-factory" import { EmbedderValidationManager } from "./embedders/embedder-validation-manager" import { VectorStoreFactory } from "./vector-store/vector-store-factory" -import { codeParser, DirectoryScanner } from "./processors" +import type { DirectoryScanner } from "./processors/scanner" +import { ParseWorkerClient } from "./parse-worker-client" +import { IndexWriteGate } from "./index-write-gate" +import { GenerationAllocator } from "./generation-allocator" +import { FileFailureRegistry } from "./file-failure-registry" +import { ProviderPacerRegistry } from "./provider-pacer" import { DirectoryScannerFactory } from "./processors/directory-scanner-factory" import { FileWatcherFactory } from "./processors/file-watcher-factory" -import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" +import type { IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" @@ -32,6 +37,12 @@ export class CodeIndexServiceFactory { private readonly configManager: CodeIndexConfigManager, private readonly workspacePath: string, private readonly cacheManager: CacheManager, + private readonly writeGate = new IndexWriteGate(), + private readonly generations = new GenerationAllocator(cacheManager), + private readonly failures = new FileFailureRegistry(), + private readonly parser = new ParseWorkerClient(), + private readonly isCurrentAttempt: () => boolean = () => true, + private readonly providerPacers = new ProviderPacerRegistry(), ) {} /** @@ -57,7 +68,7 @@ export class CodeIndexServiceFactory { ): { embedder: IEmbedder vectorStore: IVectorStore - parser: ICodeParser + parser: ParseWorkerClient scanner: DirectoryScanner fileWatcher: IFileWatcher } { @@ -68,13 +79,19 @@ export class CodeIndexServiceFactory { const config = this.configManager.getConfig() const embedder = this.embedderFactory.create(config) const vectorStore = this.vectorStoreFactory.create(config, this.workspacePath) - const parser = codeParser + const parser = this.parser const scanner = this.directoryScannerFactory.create({ embedder, vectorStore, parser, cacheManager: this.cacheManager, ignoreInstance, + writeGate: this.writeGate, + generations: this.generations, + failures: this.failures, + workspacePath: this.workspacePath, + isCurrentAttempt: this.isCurrentAttempt, + providerPacers: this.providerPacers, }) const fileWatcher = this.fileWatcherFactory.create({ workspacePath: this.workspacePath, @@ -84,6 +101,12 @@ export class CodeIndexServiceFactory { cacheManager, ignoreInstance, rooIgnoreController, + parser, + writeGate: this.writeGate, + generations: this.generations, + failures: this.failures, + isCurrentAttempt: this.isCurrentAttempt, + providerPacers: this.providerPacers, }) return { diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts new file mode 100644 index 0000000000..eddab2f58c --- /dev/null +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -0,0 +1,139 @@ +import { APIConnectionError, APIConnectionTimeoutError, APIUserAbortError } from "openai" +import { QdrantClientTimeoutError } from "@qdrant/js-client-rest" +import { + classifyIndexingError, + getIndexingErrorStatusCode, + EmbeddingError, + VectorStoreError, + parseRetryAfterMs, +} from "../indexing-errors" +import { formatEmbeddingError } from "../validation-helpers" +import { IndexingRetryError } from "../indexing-errors" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) + +describe("indexing error classification", () => { + it.each([ + [{ status: 429 }, "rate-limit"], + [{ status: 500 }, "server"], + [{ statusCode: 503 }, "server"], + [{ response: { status: 502 } }, "server"], + [{ $metadata: { httpStatusCode: 503 } }, "server"], + [{ status: 408 }, "permanent"], + [{ status: 409 }, "permanent"], + [{ status: 401 }, "permanent"], + [{ status: "500", statusCode: 429 }, "rate-limit"], + [{ status: 400, statusCode: 503 }, "permanent"], + [{ status: 400, code: "ECONNRESET" }, "permanent"], + [new Error("HTTP 503: ECONNREFUSED"), "permanent"], + [new TypeError("validation"), "permanent"], + [new TypeError("fetch failed", { cause: new Error("network") }), "connection"], + [new APIConnectionError({ cause: new Error("network") }), "connection"], + [new APIConnectionTimeoutError(), "connection"], + [new QdrantClientTimeoutError("timeout"), "connection"], + [{ name: "TimeoutError" }, "connection"], + [{ name: "AbortError", status: 503 }, "permanent"], + [new APIUserAbortError(), "permanent"], + [{ name: "ThrottlingException" }, "permanent"], + [undefined, "permanent"], + ] as const)("classifies %o as %s", (error, classification) => { + expect(classifyIndexingError(error)).toBe(classification) + }) + + it.each([ + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "ENOTFOUND", + "EAI_AGAIN", + "EPIPE", + "UND_ERR_SOCKET", + "ERR_HTTP2_STREAM_CANCEL", + ])("walks causes for %s", (code) => { + expect(classifyIndexingError(new Error("outer", { cause: { code } }))).toBe("connection") + }) + + it("handles cycles and preserves status precedence", () => { + const error = { status: 400, cause: {} } + error.cause = error + expect(classifyIndexingError(error)).toBe("permanent") + expect(getIndexingErrorStatusCode({ cause: { statusCode: 503 } })).toBe(503) + }) + + it("lets caller cancellation override structured transient errors", () => { + const error = new EmbeddingError("server", { status: 503 }) + expect(classifyIndexingError(error, AbortSignal.abort())).toBe("permanent") + }) + + it.each([EmbeddingError, VectorStoreError])("preserves fields and the raw cause in %s", (ErrorClass) => { + const cause = { $metadata: { httpStatusCode: 503 } } + const error = new ErrorClass("unchanged", cause) + expect(error.message).toBe("unchanged") + expect(error.cause).toBe(cause) + expect(error.statusCode).toBe(503) + expect(error.classification).toBe("server") + expect(classifyIndexingError(new Error("outer", { cause: error }))).toBe("server") + }) + + it("classifies raw errors before translated formatting", () => { + const cause = new APIConnectionTimeoutError() + const error = formatEmbeddingError(cause) + expect(error).toBeInstanceOf(EmbeddingError) + expect(error.message).toBe("embeddings:failedWithError") + expect(error.cause).toBe(cause) + expect(error.classification).toBe("connection") + }) +}) + +describe("parseRetryAfterMs", () => { + const now = Date.parse("2026-11-10T12:00:00Z") + it.each([ + [{ "Retry-After": "2" }, 2000], + [new Headers({ "retry-after-ms": "2500", "retry-after": "60" }), 2500], + [{ "retry-after-ms": "invalid", "retry-after": "60" }, 60_000], + [{ "retry-after-ms": "-1", "retry-after": "61" }, 61_000], + [{ "retry-after": new Date(now + 61_000).toUTCString() }, 61_000], + [{ "retry-after": new Date(now - 1000).toUTCString() }, undefined], + [{ "retry-after": "-1" }, undefined], + [{ "retry-after": "" }, undefined], + [{ "retry-after": "invalid" }, undefined], + [{ "retry-after": "Infinity" }, undefined], + [7, 7000], + [0, 0], + [-1, undefined], + [Infinity, undefined], + [undefined, undefined], + ] as const)("parses %o", (source, expected) => { + expect(parseRetryAfterMs(source, now)).toBe(expected) + }) + + it.each([EmbeddingError, VectorStoreError])("preserves delay and explicit override in %s", (ErrorClass) => { + const cause = { status: 429, headers: new Headers({ "retry-after": "2" }) } + expect(new ErrorClass("limited", cause)).toMatchObject({ retryAfterMs: 2000, cause }) + expect(new ErrorClass("limited", cause, { retryAfterMs: 7000 })).toMatchObject({ retryAfterMs: 7000 }) + }) +}) + +it.each(["2", "invalid", undefined])( + "T33 discovers header delay once across formatting and retry wrappers: %s", + (value) => { + const get = vi.fn((name: string) => (name === "retry-after" ? value : undefined)) + const original = { status: 429, headers: { get } } + const formatted = formatEmbeddingError(original) + const retried = new IndexingRetryError(formatted, 3, "rate-limit", "embed") + expect(retried.retryAfterMs).toBe(value === "2" ? 2000 : undefined) + expect(formatted.retryAfterMs).toBe(retried.retryAfterMs) + expect(get.mock.calls.filter(([name]) => name === "retry-after")).toHaveLength(1) + get.mockClear() + const constructed = new EmbeddingError("limited", original) + expect(constructed.retryAfterMs).toBe(formatted.retryAfterMs) + expect(get.mock.calls.filter(([name]) => name === "retry-after")).toHaveLength(1) + }, +) + +it("T33 explicit delay wins without reading headers", () => { + const get = vi.fn() + const error = new VectorStoreError("limited", { status: 429, headers: { get } }, { retryAfterMs: 7000 }) + expect(error.retryAfterMs).toBe(7000) + expect(get).not.toHaveBeenCalled() +}) diff --git a/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts b/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts new file mode 100644 index 0000000000..c32994861d --- /dev/null +++ b/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts @@ -0,0 +1,30 @@ +import { formatProviderWaitMessage } from "../provider-wait-message" +import { t } from "../../../../i18n" + +vi.mock("../../../../i18n", () => ({ t: vi.fn((key: string) => key) })) + +describe("formatProviderWaitMessage", () => { + beforeEach(() => vi.clearAllMocks()) + + it.each([undefined, 123_000])("formats a provider wait with resume time %s", (resumeAt) => { + const key = + resumeAt === undefined + ? "embeddings:orchestrator.providerWaiting" + : "embeddings:orchestrator.providerWaitingUntil" + expect(formatProviderWaitMessage([{ service: "embedding", resumeAt, waitingFiles: ["waiting.ts"] }])).toBe(key) + expect(t).toHaveBeenCalledExactlyOnceWith(key, { + service: "embedding", + time: resumeAt === undefined ? undefined : new Date(resumeAt).toISOString(), + }) + }) + + it("joins separate provider messages and accepts no waits", () => { + expect(formatProviderWaitMessage([])).toBe("") + expect( + formatProviderWaitMessage([ + { service: "embedding", waitingFiles: [] }, + { service: "qdrant", resumeAt: 123_000, waitingFiles: ["waiting.ts"] }, + ]), + ).toBe("embeddings:orchestrator.providerWaiting\nembeddings:orchestrator.providerWaitingUntil") + }) +}) diff --git a/src/services/code-index/shared/__tests__/retry-policy.spec.ts b/src/services/code-index/shared/__tests__/retry-policy.spec.ts new file mode 100644 index 0000000000..f85a5283dc --- /dev/null +++ b/src/services/code-index/shared/__tests__/retry-policy.spec.ts @@ -0,0 +1,206 @@ +import "../../__tests__/mock-code-index-log" +import { ProviderPacer } from "../../provider-pacer" +import { retryWithPolicy, type RetryFailure } from "../retry-policy" +import { EmbeddingError, IndexingRetryError } from "../indexing-errors" +import { APIUserAbortError } from "openai" + +beforeEach(() => vi.useFakeTimers()) +afterEach(() => vi.useRealTimers()) + +it("paces repeated limits even after both retries were consumed, then preserves exhaustion", async () => { + const budget = { retriesUsed: 0 } + const pacer = new ProviderPacer("openai:test") + const onFailure = vi.fn() + const unavailable = new EmbeddingError("server", { status: 503 }) + const limited = new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 120_000 }) + const request = vi + .fn() + .mockRejectedValueOnce(unavailable) + .mockRejectedValueOnce(unavailable) + .mockResolvedValueOnce("first batch") + const run = () => retryWithPolicy(() => pacer.run("test.ts", undefined, request), { budget, onFailure }) + const first = run() + await vi.advanceTimersByTimeAsync(1500) + await expect(first).resolves.toBe("first batch") + expect(budget.retriesUsed).toBe(2) + expect(onFailure).toHaveBeenCalledTimes(2) + request + .mockRejectedValueOnce(limited) + .mockRejectedValueOnce(limited) + .mockRejectedValueOnce(limited) + .mockRejectedValueOnce(limited) + .mockResolvedValueOnce("second batch") + const second = run() + await vi.advanceTimersByTimeAsync(0) + for (const index of [0, 1, 2, 3]) { + expect(request).toHaveBeenCalledTimes(4 + index) + expect(budget.retriesUsed).toBe(2) + expect(onFailure).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(120_000) + } + await expect(second).resolves.toBe("second batch") + request.mockRejectedValueOnce(unavailable) + const exhausted = run().catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(375) + expect(await exhausted).toMatchObject({ classification: "server", attempts: 3 }) + expect(request).toHaveBeenCalledTimes(9) + expect(onFailure).toHaveBeenCalledTimes(3) +}) + +it.each([undefined, 100, 2000, 60_000, 120_000, NaN, -1])( + "rethrows rate limits unchanged before callbacks or budget use: %s", + async (retryAfterMs) => { + const original = Object.freeze(new EmbeddingError("limited", { status: 429 }, { retryAfterMs })) + const request = vi.fn().mockRejectedValue(original) + const onFailure = vi.fn() + const budget = { retriesUsed: 2 } + await expect(retryWithPolicy(request, { budget, onFailure })).rejects.toBe(original) + expect(budget.retriesUsed).toBe(2) + expect(onFailure).not.toHaveBeenCalled() + expect(request).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }, +) + +it.each([503, { code: "ECONNRESET" }])("retains bounded connection/server waits: %s", async (failure) => { + const cause = typeof failure === "number" ? { status: failure } : failure + const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", cause)) + const failures: RetryFailure[] = [] + const pending = retryWithPolicy(request, { onFailure: (failure) => failures.push(failure) }).catch((error) => error) + for (const [index, delay] of [500, 1000].entries()) { + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(index + 1) + expect(failures[index]).toMatchObject({ delayMs: delay, delaySource: "default", willRetry: true }) + await vi.advanceTimersByTimeAsync(delay - 1) + expect(request).toHaveBeenCalledTimes(index + 1) + await vi.advanceTimersByTimeAsync(1) + } + expect(await pending).toMatchObject({ attempts: 3 }) + expect(request).toHaveBeenCalledTimes(3) +}) + +it("aborts a long wait without another request", async () => { + const controller = new AbortController() + const request = vi.fn().mockRejectedValue(new EmbeddingError("unavailable", { status: 503 })) + const pending = retryWithPolicy(request, { signal: controller.signal }).catch((error) => error) + await vi.advanceTimersByTimeAsync(0) + controller.abort() + expect(await pending).toBe(controller.signal.reason) + await vi.advanceTimersByTimeAsync(120_000) + expect(request).toHaveBeenCalledOnce() +}) + +it.each([401, 408, 409])("does not retry permanent HTTP %s", async (status) => { + const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", { status })) + await expect(retryWithPolicy(request)).rejects.toMatchObject({ + attempts: 1, + }) + expect(request).toHaveBeenCalledOnce() +}) + +it("returns a successful value and never issues an aborted first request", async () => { + const request = vi.fn().mockResolvedValue("result") + await expect(retryWithPolicy(request)).resolves.toBe("result") + await expect(retryWithPolicy(request, { signal: AbortSignal.abort() })).rejects.toMatchObject({ + name: "AbortError", + }) + expect(request).toHaveBeenCalledOnce() +}) + +it("shares two retries across successful batches without charging their first calls", async () => { + const budget = { retriesUsed: 0 } + const request = vi.fn().mockResolvedValue("result") + await retryWithPolicy(request, { budget }) + await retryWithPolicy(request, { budget }) + expect(budget.retriesUsed).toBe(0) + request.mockRejectedValueOnce({ status: 503 }).mockRejectedValueOnce({ status: 503 }) + const recovered = retryWithPolicy(request, { budget }) + await vi.advanceTimersByTimeAsync(1500) + await recovered + expect(budget.retriesUsed).toBe(2) + request.mockRejectedValue({ status: 503 }) + await expect(retryWithPolicy(request, { budget })).rejects.toMatchObject({ attempts: 3 }) + expect(request).toHaveBeenCalledTimes(6) +}) + +it("T33 wraps a frozen error without changing its metadata", async () => { + const original = Object.freeze(new EmbeddingError("unavailable", { status: 503 }, { retryAfterMs: 2000 })) + const properties = Object.getOwnPropertyDescriptors(original) + const request = vi.fn().mockRejectedValue(original) + const pending = retryWithPolicy(request, { activity: "embed" }).catch((error) => error) + await vi.advanceTimersByTimeAsync(4000) + const error = await pending + expect(error).toBeInstanceOf(IndexingRetryError) + expect(error).toMatchObject({ + message: original.message, + attempts: 3, + classification: "server", + statusCode: 503, + retryAfterMs: 2000, + activity: "embed", + }) + expect((error as IndexingRetryError).cause).toBe(original) + expect(Object.getOwnPropertyDescriptors(original)).toEqual(properties) + expect(request).toHaveBeenCalledTimes(3) +}) + +it("T33 keeps independent metadata when one error is reused across operations", async () => { + const original = Object.assign(new Error("unavailable"), { status: 503 }) + const properties = Object.getOwnPropertyDescriptors(original) + const request = vi.fn().mockRejectedValue(original) + const firstPending = retryWithPolicy(request, { activity: "upsert" }).catch((error) => error) + await vi.advanceTimersByTimeAsync(1500) + const first = await firstPending + const second = await retryWithPolicy(request, { activity: "verify", budget: { retriesUsed: 2 } }).catch( + (error) => error, + ) + expect(first).toBeInstanceOf(IndexingRetryError) + expect(second).toBeInstanceOf(IndexingRetryError) + expect(first).not.toBe(second) + expect(first).toMatchObject({ attempts: 3, activity: "upsert", classification: "server", statusCode: 503 }) + expect(second).toMatchObject({ attempts: 3, activity: "verify", classification: "server", statusCode: 503 }) + expect((first as IndexingRetryError).cause).toBe(original) + expect((second as IndexingRetryError).cause).toBe(original) + expect(Object.getOwnPropertyDescriptors(original)).toEqual(properties) + expect(original).not.toHaveProperty("attempts") + expect(original).not.toHaveProperty("activity") +}) + +it.each([ + new DOMException("cancelled", "AbortError"), + new APIUserAbortError(), + Object.freeze(Object.assign(new Error("cancelled"), { name: "APIUserAbortError" })), +])("T33 preserves live-signal cancellation identity: %s", async (original) => { + const properties = Object.getOwnPropertyDescriptors(original) + const request = vi.fn().mockRejectedValue(original) + const onFailure = vi.fn() + await expect( + retryWithPolicy(request, { signal: new AbortController().signal, activity: "embed", onFailure }), + ).rejects.toBe(original) + expect(Object.getOwnPropertyDescriptors(original)).toEqual(properties) + expect(original).not.toHaveProperty("attempts") + expect(original).not.toHaveProperty("activity") + expect(onFailure).toHaveBeenCalledWith(expect.objectContaining({ classification: "permanent", willRetry: false })) + expect(request).toHaveBeenCalledOnce() +}) + +it("T33 rethrows signal.reason rather than an operation rejection after abort", async () => { + const controller = new AbortController() + const reason = Object.freeze(new DOMException("stopped", "AbortError")) + const rejection = new Error("operation rejected") + const request = vi.fn(async () => { + controller.abort(reason) + throw rejection + }) + const onFailure = vi.fn() + await expect(retryWithPolicy(request, { signal: controller.signal, activity: "embed", onFailure })).rejects.toBe( + controller.signal.reason, + ) + expect(reason.name).toBe("AbortError") + expect(reason).not.toHaveProperty("attempts") + expect(reason).not.toHaveProperty("activity") + expect(rejection).not.toHaveProperty("attempts") + expect(rejection).not.toHaveProperty("activity") + expect(onFailure).not.toHaveBeenCalled() + expect(request).toHaveBeenCalledOnce() +}) diff --git a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts index bf6c732a92..115c324e44 100644 --- a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts +++ b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts @@ -1,4 +1,61 @@ -import { sanitizeErrorMessage } from "../validation-helpers" +import * as fs from "fs" +import * as path from "path" +import { formatEmbeddingError, sanitizeErrorMessage } from "../validation-helpers" +import { EmbeddingError } from "../indexing-errors" +import { t } from "../../../../i18n" + +vi.mock("../../../../i18n", () => ({ t: vi.fn((key: string) => key) })) + +describe("formatEmbeddingError", () => { + it.each([ + [401, "permanent", "embeddings:authenticationFailed"], + [429, "rate-limit", "embeddings:failedWithStatus"], + [503, "server", "embeddings:failedWithStatus"], + ] as const)( + "keeps classification and delay for HTTP %s as a single attempt", + (status, classification, key) => { + const cause = Object.assign(new Error("failure"), { status, headers: new Headers({ "retry-after": "2" }) }) + const error = formatEmbeddingError(cause) + expect(error).toBeInstanceOf(EmbeddingError) + expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: 2000, cause }) + expect(error.message).toBe(key) + if (status !== 401) { + expect(t).toHaveBeenLastCalledWith(key, { attempts: 1, statusCode: status, errorMessage: "failure" }) + } + }, + ) + + it("preserves connection errors with no headers", () => { + const cause = new TypeError("fetch failed", { cause: new Error("network") }) + const error = formatEmbeddingError(cause) + expect(error).toMatchObject({ classification: "connection", cause }) + expect(error.retryAfterMs).toBeUndefined() + expect(t).toHaveBeenLastCalledWith("embeddings:failedWithError", { attempts: 1, errorMessage: "fetch failed" }) + }) + + it("supplies every placeholder used by the failure messages in all locales", () => { + formatEmbeddingError(Object.assign(new Error("failure"), { status: 503 })) + const statusArgs = vi.mocked(t).mock.lastCall?.[1] ?? {} + formatEmbeddingError(new TypeError("fetch failed")) + const errorArgs = vi.mocked(t).mock.lastCall?.[1] ?? {} + const localesRoot = path.resolve(__dirname, "../../../../i18n/locales") + const locales = fs.readdirSync(localesRoot) + expect(locales).toHaveLength(18) + for (const locale of locales) { + const messages: Record = JSON.parse( + fs.readFileSync(path.join(localesRoot, locale, "embeddings.json"), "utf8"), + ) + for (const [key, args] of [ + ["failedWithStatus", statusArgs], + ["failedWithError", errorArgs], + ] as const) { + const placeholders = [...String(messages[key]).matchAll(/\{\{(\w+)\}\}/g)].map((match) => match[1]) + expect(placeholders.length, `${locale} ${key}`).toBeGreaterThan(0) + for (const placeholder of placeholders) expect(args, `${locale} ${key}`).toHaveProperty(placeholder) + } + } + }) +}) describe("sanitizeErrorMessage", () => { it("should sanitize Unix-style file paths", () => { @@ -63,11 +120,15 @@ describe("sanitizeErrorMessage", () => { expect(sanitizeErrorMessage(input)).toBe(expected) }) - it("should handle non-string inputs gracefully", () => { - expect(sanitizeErrorMessage(null as any)).toBe("null") - expect(sanitizeErrorMessage(undefined as any)).toBe("undefined") - expect(sanitizeErrorMessage(123 as any)).toBe("123") - expect(sanitizeErrorMessage({} as any)).toBe("[object Object]") + it.each([ + [null, "null"], + [undefined, "undefined"], + [123, "123"], + [{}, "[object Object]"], + ])("handles runtime non-string input %o", (input, expected) => { + // Exercise untyped callers without weakening the production string contract. + const sanitizeRuntimeInput = sanitizeErrorMessage as unknown as (value: unknown) => string + expect(sanitizeRuntimeInput(input)).toBe(expected) }) it("should preserve non-sensitive error messages", () => { @@ -90,3 +151,34 @@ describe("sanitizeErrorMessage", () => { expect(sanitizeErrorMessage(input)).toBe(expected) }) }) + +describe("validation rate limits", () => { + it("rethrows the original 429 before custom handling", async () => { + const { withValidationErrorHandling } = await import("../validation-helpers") + const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }) + const handler = vi.fn() + await expect( + withValidationErrorHandling( + async () => { + throw error + }, + "openai", + { + beforeStandardHandling: handler, + }, + ), + ).rejects.toBe(error) + expect(handler).not.toHaveBeenCalled() + }) + it("still accepts valid configuration and converts permanent failure", async () => { + const { withValidationErrorHandling } = await import("../validation-helpers") + await expect(withValidationErrorHandling(async () => ({ valid: true }), "openai")).resolves.toEqual({ + valid: true, + }) + await expect( + withValidationErrorHandling(async () => { + throw new Error("bad key") + }, "openai"), + ).resolves.toEqual({ valid: false, error: "bad key" }) + }) +}) diff --git a/src/services/code-index/shared/indexing-errors.ts b/src/services/code-index/shared/indexing-errors.ts new file mode 100644 index 0000000000..d3b09b8c3a --- /dev/null +++ b/src/services/code-index/shared/indexing-errors.ts @@ -0,0 +1,167 @@ +import { APIConnectionError, APIUserAbortError } from "openai" +import type { FileFailureActivity } from "../file-failure-registry" + +export type TransientClass = "rate-limit" | "server" | "connection" +export type IndexingErrorClassification = TransientClass | "permanent" + +type ErrorDetails = { + classification?: IndexingErrorClassification + statusCode?: number + retryAfterMs?: number +} + +type ErrorShape = ErrorDetails & { + name?: string + code?: string + status?: number + headers?: RetryAfterHeaders + response?: { status?: number } + $metadata?: { httpStatusCode?: number } + cause?: unknown +} + +type RetryAfterHeaders = Headers | Record + +function nonnegativeNumber(value: unknown): number | undefined { + if (typeof value !== "number" && typeof value !== "string") return undefined + if (typeof value === "string" && !value.trim()) return undefined + const number = Number(value) + return Number.isFinite(number) && number >= 0 ? number : undefined +} + +export function parseRetryAfterMs(source: unknown, now = Date.now()): number | undefined { + if (typeof source === "number") { + const seconds = nonnegativeNumber(source) + return seconds !== undefined && Number.isFinite(seconds * 1000) ? seconds * 1000 : undefined + } + if (!source || typeof source !== "object") return undefined + const headers = source as RetryAfterHeaders + let milliseconds: unknown + let retryAfter: unknown + if (typeof headers.get === "function") { + milliseconds = headers.get("retry-after-ms") + retryAfter = headers.get("retry-after") + } else { + for (const [name, value] of Object.entries(source)) { + if (name.toLowerCase() === "retry-after-ms") milliseconds = value + if (name.toLowerCase() === "retry-after") retryAfter = value + } + } + const parsedMilliseconds = nonnegativeNumber(milliseconds) + if (parsedMilliseconds !== undefined) return parsedMilliseconds + const seconds = nonnegativeNumber(retryAfter) + if (seconds !== undefined && Number.isFinite(seconds * 1000)) return seconds * 1000 + if (typeof retryAfter !== "string" || !/[a-z]/i.test(retryAfter)) return undefined + const delay = Date.parse(retryAfter) - now + return Number.isFinite(delay) && delay > 0 ? delay : undefined +} + +function errorChain(error: unknown): ErrorShape[] { + const chain: ErrorShape[] = [] + const visited = new Set() + while (error && typeof error === "object" && !visited.has(error)) { + visited.add(error) + chain.push(error as ErrorShape) + error = (error as ErrorShape).cause + } + return chain +} + +function numericStatus(error: ErrorShape): number | undefined { + return [error.status, error.statusCode, error.response?.status, error.$metadata?.httpStatusCode].find( + (status) => typeof status === "number" && Number.isFinite(status), + ) +} + +export function getIndexingErrorStatusCode(error: unknown): number | undefined { + for (const entry of errorChain(error)) { + const status = numericStatus(entry) + if (status !== undefined) return status + } + return undefined +} + +export function classifyIndexingError(error: unknown, signal?: AbortSignal): IndexingErrorClassification { + const chain = errorChain(error) + if ( + signal?.aborted || + error instanceof APIUserAbortError || + chain[0]?.name === "AbortError" || + chain[0]?.name === "APIUserAbortError" + ) { + return "permanent" + } + for (const entry of chain) { + if ( + entry.classification === "rate-limit" || + entry.classification === "server" || + entry.classification === "connection" || + entry.classification === "permanent" + ) { + return entry.classification + } + } + if (chain.some((entry) => entry instanceof APIUserAbortError || entry.name === "AbortError")) return "permanent" + const status = getIndexingErrorStatusCode(error) + if (status !== undefined) { + if (status === 429) return "rate-limit" + return status >= 500 && status < 600 ? "server" : "permanent" + } + const networkCodes = new Set(["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "EPIPE"]) + for (const entry of chain) { + if ( + (typeof entry.code === "string" && + (networkCodes.has(entry.code) || + entry.code.startsWith("UND_ERR_") || + entry.code.startsWith("ERR_HTTP2_"))) || + (entry instanceof TypeError && entry.cause !== undefined) || + entry instanceof APIConnectionError || + entry.name === "APIConnectionError" || + entry.name === "APIConnectionTimeoutError" || + entry.name === "TimeoutError" || + entry.name === "QdrantClientTimeoutError" + ) { + return "connection" + } + } + return "permanent" +} + +class IndexingError extends Error { + readonly classification: IndexingErrorClassification + readonly statusCode?: number + + readonly retryAfterMs?: number + + constructor(message: string, cause: unknown, details?: ErrorDetails) { + super(message, { cause }) + this.name = new.target.name + this.classification = details?.classification ?? classifyIndexingError(cause) + this.statusCode = details?.statusCode ?? getIndexingErrorStatusCode(cause) + this.retryAfterMs = details?.retryAfterMs + if (this.retryAfterMs === undefined) { + for (const entry of errorChain(cause)) { + this.retryAfterMs = nonnegativeNumber(entry.retryAfterMs) + // Indexing-owned causes already checked for valid, missing or invalid headers. + if (entry instanceof IndexingError) break + this.retryAfterMs ??= parseRetryAfterMs(entry.headers) + if (this.retryAfterMs !== undefined) break + } + } + } +} + +export class EmbeddingError extends IndexingError {} +export class VectorStoreError extends IndexingError {} +export class SearchProviderRateLimitError extends IndexingError {} + +export class IndexingRetryError extends IndexingError { + constructor( + cause: unknown, + readonly attempts: number, + classification: IndexingErrorClassification, + readonly activity?: FileFailureActivity, + ) { + super(cause instanceof Error ? cause.message : String(cause), cause, { classification }) + } +} diff --git a/src/services/code-index/shared/provider-wait-message.ts b/src/services/code-index/shared/provider-wait-message.ts new file mode 100644 index 0000000000..0fc0e4d590 --- /dev/null +++ b/src/services/code-index/shared/provider-wait-message.ts @@ -0,0 +1,18 @@ +import type { ProviderWaitStatus } from "@roo-code/types" +import { t } from "../../../i18n" + +export function formatProviderWaitMessage(waits: readonly ProviderWaitStatus[]): string { + return waits + .map((wait) => + t( + wait.resumeAt === undefined + ? "embeddings:orchestrator.providerWaiting" + : "embeddings:orchestrator.providerWaitingUntil", + { + service: wait.service, + time: wait.resumeAt === undefined ? undefined : new Date(wait.resumeAt).toISOString(), + }, + ), + ) + .join("\n") +} diff --git a/src/services/code-index/shared/retry-policy.ts b/src/services/code-index/shared/retry-policy.ts new file mode 100644 index 0000000000..490ef34ff6 --- /dev/null +++ b/src/services/code-index/shared/retry-policy.ts @@ -0,0 +1,65 @@ +import { APIUserAbortError } from "openai" +import { FILE_RETRY_ATTEMPTS, FILE_RETRY_BASE_DELAY_MS } from "../constants" +import { waitForRetry } from "../embedders/cancellation" +import { classifyIndexingError, IndexingRetryError, type IndexingErrorClassification } from "./indexing-errors" +import type { FileFailureActivity } from "../file-failure-registry" + +export interface RetryBudget { + retriesUsed: number +} + +export interface RetryFailure { + error: unknown + attempt: number + classification: IndexingErrorClassification + willRetry: boolean + delayMs: number + delaySource: "default" | "provider" +} + +export interface RetryPolicyOptions { + signal?: AbortSignal + activity?: FileFailureActivity + budget?: RetryBudget + onFailure?: (failure: RetryFailure) => void +} + +export async function retryWithPolicy( + operation: () => Promise, + { signal, activity, budget = { retriesUsed: 0 }, onFailure }: RetryPolicyOptions = {}, +): Promise { + while (true) { + signal?.throwIfAborted() + try { + return await operation() + } catch (error) { + signal?.throwIfAborted() + const classification = classifyIndexingError(error, signal) + if (classification === "rate-limit") throw error + const attempt = budget.retriesUsed + 1 + const willRetry = classification !== "permanent" && attempt < FILE_RETRY_ATTEMPTS + const delayMs = willRetry ? FILE_RETRY_BASE_DELAY_MS * 2 ** budget.retriesUsed : 0 + if (willRetry) budget.retriesUsed++ + onFailure?.({ + error, + attempt, + classification, + willRetry, + delayMs, + delaySource: "default", + }) + if (!willRetry) { + if ( + error instanceof APIUserAbortError || + (error && + typeof error === "object" && + "name" in error && + (error.name === "AbortError" || error.name === "APIUserAbortError")) + ) + throw error + throw new IndexingRetryError(error, attempt, classification, activity) + } + await waitForRetry(delayMs, signal) + } + } +} diff --git a/src/services/code-index/shared/validation-helpers.ts b/src/services/code-index/shared/validation-helpers.ts index 025d004aa1..d93cb0141e 100644 --- a/src/services/code-index/shared/validation-helpers.ts +++ b/src/services/code-index/shared/validation-helpers.ts @@ -1,5 +1,6 @@ import { t } from "../../../i18n" import { serializeError } from "serialize-error" +import { classifyIndexingError, EmbeddingError, getIndexingErrorStatusCode } from "./indexing-errors" /** * Sanitizes error messages by removing sensitive information like file paths and URLs @@ -48,11 +49,19 @@ export function sanitizeErrorMessage(errorMessage: string): string { */ export interface HttpError extends Error { status?: number + headers?: Headers response?: { status?: number } } +export interface ValidationError { + status?: number + message?: string + name?: string + code?: string +} + /** * Maps HTTP status codes to appropriate error messages */ @@ -198,6 +207,7 @@ export async function withValidationErrorHandling>() + private diagnosticContext: { source: string; attempt: number | string } = { source: "state-manager", attempt: 0 } + + public setDiagnosticContext(context: { source: string; attempt: number | string }): void { + this.diagnosticContext = { ...context } + } + + private logTransition(oldState: IndexingState, source?: string): void { + codeIndexLog.info("State transition", { + oldState, + newState: this._systemStatus, + message: this._statusMessage, + ...this.diagnosticContext, + source: source ?? this.diagnosticContext.source, + }) + } + // --- Public API --- public readonly onProgressUpdate = this._progressEmitter.event @@ -30,11 +47,12 @@ export class CodeIndexStateManager { // --- State Management --- - public setSystemState(newState: IndexingState, message?: string): void { + public setSystemState(newState: IndexingState, message?: string, source?: string): void { const stateChanged = newState !== this._systemStatus || (message !== undefined && message !== this._statusMessage) if (stateChanged) { + const oldState = this._systemStatus this._systemStatus = newState if (message !== undefined) { this._statusMessage = message @@ -51,15 +69,15 @@ export class CodeIndexStateManager { if (newState === "Error" && message === undefined) this._statusMessage = "An error occurred." } + this.logTransition(oldState, source) this._progressEmitter.fire(this.getCurrentStatus()) } } - public reportBlockIndexingProgress(processedItems: number, totalItems: number): void { + public reportBlockIndexingProgress(processedItems: number, totalItems: number, source = "block-progress"): void { const progressChanged = processedItems !== this._processedItems || totalItems !== this._totalItems - // Don't override Stopping state with progress updates - if (this._systemStatus === "Stopping") return + if (this._systemStatus === "Error" || this._systemStatus === "Stopping") return // Update if progress changes OR if the system wasn't already in 'Indexing' state if (progressChanged || this._systemStatus !== "Indexing") { this._processedItems = processedItems @@ -73,6 +91,7 @@ export class CodeIndexStateManager { this._systemStatus = "Indexing" // Ensure state is Indexing this._statusMessage = message + if (oldStatus !== this._systemStatus) this.logTransition(oldStatus, source) // Only fire update if status, message or progress actually changed if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { this._progressEmitter.fire(this.getCurrentStatus()) @@ -80,15 +99,20 @@ export class CodeIndexStateManager { } } - public reportFileQueueProgress(processedFiles: number, totalFiles: number, currentFileBasename?: string): void { + public reportFileQueueProgress( + processedFiles: number, + totalFiles: number, + currentFileBasename?: string, + source = "file-queue-progress", + ): void { const progressChanged = processedFiles !== this._processedItems || totalFiles !== this._totalItems - // Don't override Stopping state with progress updates - if (this._systemStatus === "Stopping") return + if (this._systemStatus === "Error" || this._systemStatus === "Stopping") return if (progressChanged || this._systemStatus !== "Indexing") { this._processedItems = processedFiles this._totalItems = totalFiles this._currentItemUnit = "files" + const oldStatus = this._systemStatus this._systemStatus = "Indexing" let message: string @@ -102,10 +126,10 @@ export class CodeIndexStateManager { message = `File queue processed.` } - const oldStatus = this._systemStatus const oldMessage = this._statusMessage this._statusMessage = message + if (oldStatus !== this._systemStatus) this.logTransition(oldStatus, source) if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { this._progressEmitter.fire(this.getCurrentStatus()) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts new file mode 100644 index 0000000000..4ee365d748 --- /dev/null +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts @@ -0,0 +1,178 @@ +import "../../__tests__/mock-code-index-log" +import { createServer } from "node:http" +import path from "node:path" +import type { AddressInfo } from "node:net" +import nock from "nock" +import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" +import { QdrantVectorStore } from "../qdrant-client" +import { VectorStoreError } from "../../shared/indexing-errors" +import { pacedTransportWriter, waitForTransport } from "../../embedders/__tests__/paced-transport-writer" +import type { IEmbedder } from "../../interfaces" +import { LATE_WRITE_GRACE_MS } from "../../index-write-gate" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +const now = Date.parse("2026-11-11T12:00:00Z") +const operations = ["upsert", "delete", "verify"] as const +const matchesOperation = (url: string, operation: (typeof operations)[number]) => + operation === "verify" + ? url.includes("/count") + : operation === "delete" + ? url.includes("/delete") + : /\/points\?/.test(url) + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }) + vi.setSystemTime(now) + vi.spyOn(Math, "random").mockReturnValue(0) + nock.enableNetConnect("127.0.0.1") +}) +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + nock.disableNetConnect() +}) + +const scenarios = [ + ...operations.flatMap((activity) => + [7, 60, 61, 120].map((seconds) => ({ + activity, + scenario: `numeric ${seconds} seconds, four limits`, + header: String(seconds) as string | undefined, + delay: seconds * 1000, + limits: 4, + })), + ), + { + activity: "upsert", + scenario: "HTTP date 7 seconds ahead", + header: new Date(now + 7000).toUTCString(), + delay: 7000, + limits: 1, + }, + { + activity: "upsert", + scenario: "HTTP date 120 seconds ahead", + header: new Date(now + 120_000).toUTCString(), + delay: 120_000, + limits: 1, + }, + { + activity: "upsert", + scenario: "past date", + header: new Date(now - 7000).toUTCString(), + delay: undefined, + limits: 1, + }, + { activity: "upsert", scenario: "numeric 61 seconds", header: "61", delay: 61_000, limits: 1 }, + { activity: "upsert", scenario: "missing header", header: undefined, delay: undefined, limits: 1 }, +] as const + +it.each(scenarios)("T26/T32 HTTP $activity $scenario", async ({ activity, header, delay, limits }) => { + let requests = 0 + let healthComplete!: () => void + const health = new Promise((resolve) => { + healthComplete = resolve + }) + const perAttempt = activity === "verify" ? 2 : 1 + const server = createServer((request, response) => { + response.setHeader("content-type", "application/json") + if (request.url === "/") { + response.end(JSON.stringify({ title: "qdrant", version: "1.15.0" }), healthComplete) + return + } + const matches = matchesOperation(request.url!, activity) + if (matches) requests++ + let body = "" + request.on("data", (chunk) => { + body += chunk + }) + request.on("end", () => { + if (matches && requests <= limits * perAttempt) { + response.writeHead(429, header === undefined ? {} : { "retry-after": header }) + response.end(JSON.stringify({ status: { error: "limited" }, time: 0 })) + } else { + const count = request.url!.includes("/count") + ? JSON.parse(body).filter.must.length === 2 + ? 1 + : 0 + : undefined + response.end( + JSON.stringify({ + result: count === undefined ? { status: "completed", operation_id: 1 } : { count }, + status: "ok", + time: 0, + }), + ) + } + }) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + const store = new QdrantVectorStore(path.resolve("/workspace"), endpoint, 2) + await health + const method = + activity === "verify" ? "countPointsForFile" : activity === "delete" ? "deletePointsByFilePath" : "upsertPoints" + const operation = vi.spyOn(store, method) + const { replace, failures, cache, pacers, gate } = pacedTransportWriter(store) + const grace = vi.spyOn(gate, "waitForGrace") + const controller = new AbortController() + const embedding = { + rateLimitScope: "embedder:test", + createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[1, 0]] }), + validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), + embedderInfo: { name: "openai" }, + } satisfies IEmbedder + const pending = replace(embedding, controller.signal).catch((error: unknown) => error) + try { + for (const index of Array.from({ length: limits }, (_, index) => index)) { + await waitForTransport(() => { + expect(operation).toHaveBeenCalledTimes(index + 1) + expect(pacers.get(store.rateLimitScope).state().pausedUntil).toBe(Date.now() + (delay ?? 500)) + expect(pacers.get(store.rateLimitScope).state().waitingPaths).toEqual(["test.ts"]) + }) + await waitForTransport(() => expect(requests).toBe((index + 1) * perAttempt)) + const error = (await operation.mock.results[index].value.catch( + (error: unknown) => error, + )) as VectorStoreError + expect(error).toBeInstanceOf(VectorStoreError) + expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429 }) + if (index === 0) { + expect(error.retryAfterMs).toBe(delay) + if (header !== undefined && /^\d+$/.test(header)) { + expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) + expect(error.cause).toMatchObject({ retry_after: Number(header) }) + } else { + const rawError = error.cause as Error & { status: number; headers: Headers } + expect(rawError).toMatchObject({ name: "Error", status: 429 }) + expect(rawError.headers.get("retry-after")).toBe(header ?? null) + } + } + expect(failures.list()).toEqual([]) + expect(cache.completeFile).not.toHaveBeenCalled() + vi.advanceTimersByTime((delay ?? 500) - 1) + expect(requests).toBe((index + 1) * perAttempt) + vi.advanceTimersByTime(1) + } + await waitForTransport(() => expect(requests).toBe((limits + 1) * perAttempt)) + await operation.mock.results[limits].value + // Successful probes stage subsequent cleanup and verification requests. + await waitForTransport(() => { + if (cache.completeFile.mock.calls.length + grace.mock.calls.length === 0) { + vi.advanceTimersByTime(125) + } + expect(cache.completeFile.mock.calls.length + grace.mock.calls.length).toBe(1) + }) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await waitForTransport(() => expect(cache.completeFile).toHaveBeenCalledOnce()) + await pending + expect(requests).toBe((limits + 1) * perAttempt) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() + expect(embedding.createEmbeddings).toHaveBeenCalledOnce() + } finally { + controller.abort() + await pending + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + } +}) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts new file mode 100644 index 0000000000..1499b7d133 --- /dev/null +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts @@ -0,0 +1,150 @@ +import "../../__tests__/mock-code-index-log" +import path from "node:path" +import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" +import { QdrantVectorStore } from "../qdrant-client" +import { pacedTransportWriter } from "../../embedders/__tests__/paced-transport-writer" +import { VectorStoreError } from "../../shared/indexing-errors" +import type { IEmbedder } from "../../interfaces" +import { LATE_WRITE_GRACE_MS } from "../../index-write-gate" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +const now = Date.parse("2026-11-11T12:00:00Z") +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(now) + vi.spyOn(Math, "random").mockReturnValue(0) +}) +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +const operations = ["upsert", "delete", "verify"] as const +const matchesOperation = (url: string, operation: (typeof operations)[number]) => + operation === "verify" + ? url.includes("/count") + : operation === "delete" + ? url.includes("/delete") + : /\/points\?/.test(url) + +function installTransport(activity: (typeof operations)[number], header: string | undefined, limitedAttempts = 1) { + const requests = vi.fn() + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, options?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input) + const matches = matchesOperation(url, activity) + if (matches) requests() + const limited = matches && requests.mock.calls.length <= limitedAttempts * (activity === "verify" ? 2 : 1) + const body = input instanceof Request ? await input.clone().text() : String(options?.body ?? "{}") + const count = url.includes("/count") ? (JSON.parse(body).filter.must.length === 2 ? 1 : 0) : undefined + return new Response( + JSON.stringify( + limited + ? { status: { error: "limited" }, time: 0 } + : url.endsWith("/") + ? { title: "qdrant", version: "1.15.0" } + : { + result: count === undefined ? { status: "completed", operation_id: 1 } : { count }, + status: "ok", + time: 0, + }, + ), + { + status: limited ? 429 : 200, + headers: { + "content-type": "application/json", + ...(header === undefined ? {} : { "retry-after": header }), + }, + }, + ) + }), + ) + return requests +} + +const embedder = () => + ({ + rateLimitScope: "embedder:test", + createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[1, 0]] }), + validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), + embedderInfo: { name: "openai" }, + }) satisfies IEmbedder + +describe.each(operations)("T26 Qdrant %s pacing", (activity) => { + it.each([7, 60, 61, 120])( + "honors %s seconds across four limits and completes without failures", + async (seconds) => { + const requests = installTransport(activity, String(seconds), 4) + const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const { replace, failures, cache } = pacedTransportWriter(store) + const embedding = embedder() + const pending = replace(embedding) + await vi.advanceTimersByTimeAsync(0) + const perAttempt = activity === "verify" ? 2 : 1 + for (const index of [0, 1, 2, 3]) { + expect(requests).toHaveBeenCalledTimes((index + 1) * perAttempt) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(seconds * 1000 - 1) + expect(requests).toHaveBeenCalledTimes((index + 1) * perAttempt) + await vi.advanceTimersByTimeAsync(1) + } + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await pending + expect(requests).toHaveBeenCalledTimes(5 * perAttempt) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() + expect(embedding.createEmbeddings).toHaveBeenCalledOnce() + }, + ) +}) + +it.each([ + { scenario: "HTTP date 7 seconds ahead", header: new Date(now + 7000).toUTCString(), delay: 7000 }, + { scenario: "HTTP date 120 seconds ahead", header: new Date(now + 120_000).toUTCString(), delay: 120_000 }, + { scenario: "past HTTP date", header: new Date(now - 7000).toUTCString(), delay: undefined }, + { scenario: "numeric 61 seconds", header: "61", delay: 61_000 }, + { scenario: "missing header", header: undefined, delay: undefined }, +])("T32 $scenario preserves the raw error and waits without a cap", async ({ header, delay }) => { + const requests = installTransport("upsert", header) + const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const upsert = vi.spyOn(store, "upsertPoints") + const { replace, failures, cache } = pacedTransportWriter(store) + const pending = replace(embedder()) + await vi.advanceTimersByTimeAsync(0) + const result = upsert.mock.results[0] + const error = (await result.value.catch((error: unknown) => error)) as VectorStoreError + expect(error).toBeInstanceOf(VectorStoreError) + expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429 }) + expect(error.retryAfterMs).toBe(delay) + if (header === "61") { + expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) + expect(error.cause).toMatchObject({ retry_after: 61 }) + } else { + const rawError = error.cause as Error & { status: number; headers: Headers } + expect(rawError).toMatchObject({ name: "Error", status: 429 }) + expect(rawError.headers.get("retry-after")).toBe(header ?? null) + } + expect(requests).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + await vi.advanceTimersByTimeAsync((delay ?? 500) - 1) + expect(requests).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await pending + expect(requests).toHaveBeenCalledTimes(2) + expect(failures.list()).toEqual([]) + expect(cache.completeFile).toHaveBeenCalledOnce() +}) + +it("scopes Qdrant by normalized URL, independently of workspace and key", () => { + installTransport("upsert", undefined, 0) + const first = new QdrantVectorStore("/first", "http://EXAMPLE.test:6333/", 2, "first-key") + const same = new QdrantVectorStore("/second", "example.test:6333", 2, "second-key") + const other = new QdrantVectorStore("/first", "http://example.test:6334", 2) + expect(first.rateLimitScope).toBe(same.rateLimitScope) + expect(first.rateLimitScope).not.toBe(other.rateLimitScope) + expect(first.rateLimitScope).toMatch(/^qdrant:[a-f0-9]{64}$/) +}) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts new file mode 100644 index 0000000000..2c50f297eb --- /dev/null +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts @@ -0,0 +1,200 @@ +import { createServer, type Server } from "node:http" +import { createRequire } from "node:module" +import path from "node:path" +import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" +import { QdrantVectorStore } from "../qdrant-client" +import { VectorStoreError } from "../../shared/indexing-errors" +import { retryWithPolicy, type RetryFailure } from "../../shared/retry-policy" +import { waitForRetry } from "../../embedders/cancellation" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../../code-index-log", () => ({ + codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})) +vi.mock("../../embedders/cancellation", () => ({ waitForRetry: vi.fn().mockResolvedValue(undefined) })) + +let server: Server +let store: QdrantVectorStore +let responseStatus: number +let retryAfter: string | undefined +let requests: Array<{ method: string | undefined; url: string; body: unknown }> +const workspacePath = path.resolve("qdrant-contract-workspace") + +beforeEach(async () => { + vi.clearAllMocks() + responseStatus = 200 + retryAfter = undefined + requests = [] + server = createServer((incoming, outgoing) => { + const chunks: Buffer[] = [] + incoming.on("data", (chunk: Buffer) => chunks.push(chunk)) + incoming.on("end", () => { + const body = Buffer.concat(chunks).toString() + const url = incoming.url ?? "" + if (url === "/") { + outgoing.writeHead(200, { "content-type": "application/json" }) + outgoing.end(JSON.stringify({ title: "qdrant", version: "1.20.0" })) + return + } + requests.push({ method: incoming.method, url, body: body ? JSON.parse(body) : undefined }) + outgoing.writeHead(responseStatus, { + "content-type": "application/json", + ...(retryAfter === undefined ? {} : { "retry-after": retryAfter }), + }) + const result = url.includes("/count") ? { count: 2 } : { status: "completed", operation_id: 1 } + outgoing.end( + JSON.stringify({ result, status: responseStatus === 200 ? "ok" : { error: "failure" }, time: 0 }), + ) + }) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const address = server.address() + if (!address || typeof address === "string") throw new Error("Expected a TCP test listener") + store = new QdrantVectorStore(workspacePath, `http://127.0.0.1:${address.port}`, 2) +}) + +afterEach(async () => { + vi.restoreAllMocks() + server.closeAllConnections() + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) +}) + +function operation(activity: "upsert" | "delete" | "verify"): Promise { + if (activity === "upsert") { + return store.upsertPoints([ + { id: "00000000-0000-4000-8000-000000000001", vector: [1, 0], payload: { filePath: "file.ts" } }, + ]) + } + if (activity === "delete") return store.deletePointsByFilePath("file.ts", 2) + return store.countPointsForFile("file.ts", 2) +} + +it("runs the exact pinned package rather than a stale install", () => { + const require = createRequire(import.meta.url) + const installed: unknown = require( + path.resolve(path.dirname(require.resolve("@qdrant/js-client-rest")), "../../package.json"), + ) + expect(installed).toMatchObject({ version: "1.20.0" }) +}) + +it.each([ + ["upsert", 60], + ["delete", 60], + ["verify", 60], + ["upsert", 61], + ["delete", 61], + ["verify", 61], +] as const)("%s preserves the complete Retry-After %s header through real HTTP", async (activity, seconds) => { + responseStatus = 429 + retryAfter = String(seconds) + const failures: RetryFailure[] = [] + const budget = { retriesUsed: 2 } + const error: unknown = await retryWithPolicy(() => operation(activity), { + activity, + budget, + onFailure: (failure) => failures.push(failure), + }).catch((failure: unknown) => failure) + expect(waitForRetry).not.toHaveBeenCalled() + expect(failures).toEqual([]) + expect(budget.retriesUsed).toBe(2) + expect(error).toBeInstanceOf(VectorStoreError) + if (!(error instanceof VectorStoreError)) throw new Error("Expected a classified vector-store error") + expect(error).toMatchObject({ retryAfterMs: seconds * 1000, classification: "rate-limit", statusCode: 429 }) + expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) + expect(error.cause).toMatchObject({ retry_after: seconds }) + expect(requests).toHaveLength(activity === "verify" ? 2 : 1) +}) + +it.each(["upsert", "delete", "verify"] as const)("%s is a single attempt without the policy", async (activity) => { + responseStatus = 503 + await expect(operation(activity)).rejects.toMatchObject({ classification: "server", statusCode: 503 }) + expect(requests).toHaveLength(activity === "verify" ? 2 : 1) + expect(waitForRetry).not.toHaveBeenCalled() +}) + +it.each([400, 401, 403, 408, 409])("HTTP %s is permanent and never retried", async (status) => { + responseStatus = status + await expect(retryWithPolicy(() => operation("upsert"))).rejects.toMatchObject({ + classification: "permanent", + attempts: 1, + statusCode: status, + }) + expect(requests).toHaveLength(1) + expect(waitForRetry).not.toHaveBeenCalled() +}) + +it("deletes exact file paths only below the generation cutoff", async () => { + await store.deletePointsByMultipleFilePaths([path.join(workspacePath, "file.ts"), "other.ts"], 2) + expect(requests).toHaveLength(1) + expect(requests[0].body).toEqual({ + filter: { + must: [ + { key: "filePath", match: { any: ["file.ts", "other.ts"] } }, + { should: [{ key: "generation", range: { lt: 2 } }, { is_empty: { key: "generation" } }] }, + ], + }, + }) + expect(requests[0].url).toContain("wait=true") +}) + +it("counts the exact current generation separately from all other points", async () => { + await expect(store.countPointsForFile(path.join(workspacePath, "file.ts"), 2)).resolves.toEqual({ + current: 2, + other: 2, + }) + expect(requests.map(({ body }) => body)).toEqual([ + { + filter: { + must: [ + { key: "filePath", match: { value: "file.ts" } }, + { key: "generation", match: { value: 2 } }, + ], + }, + exact: true, + }, + { + filter: { + must: [{ key: "filePath", match: { value: "file.ts" } }], + must_not: [{ key: "generation", match: { value: 2 } }], + }, + exact: true, + }, + ]) +}) + +it.each([-1, 0.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])( + "rejects unsafe generation %s before either delete entry point", + async (generation) => { + await expect(store.deletePointsByFilePath("file.ts", generation)).rejects.toThrow("generation cutoff") + await expect(store.deletePointsByMultipleFilePaths(["file.ts"], generation)).rejects.toThrow( + "generation cutoff", + ) + expect(requests).toHaveLength(0) + }, +) + +it("accepts generation zero through both delete entry points and skips an empty file list", async () => { + await store.deletePointsByFilePath("file.ts", 0) + await store.deletePointsByMultipleFilePaths(["file.ts"], 0) + await store.deletePointsByMultipleFilePaths([], 0) + expect(requests).toHaveLength(2) +}) + +it("creates the required verification indexes before the optional indexes", async () => { + await store["_createPayloadIndexes"]() + expect(requests.slice(0, 2).map(({ body }) => body)).toEqual([ + { field_name: "filePath", field_schema: "keyword" }, + { field_name: "generation", field_schema: "integer" }, + ]) + expect(requests).toHaveLength(8) + expect(requests.every(({ url }) => url.includes("wait=true"))).toBe(true) +}) + +it.each(["filePath", "generation"])("does not swallow a required %s index failure", async (fieldName) => { + const createIndex = vi.spyOn(store["client"], "createPayloadIndex") + if (fieldName === "generation") createIndex.mockResolvedValueOnce({ status: "completed", operation_id: 1 }) + const error = new Error("required index failed") + createIndex.mockRejectedValueOnce(error) + await expect(store["_createPayloadIndexes"]()).rejects.toBe(error) + expect(createIndex).toHaveBeenCalledTimes(fieldName === "generation" ? 2 : 1) +}) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 12c92f1aad..38a8302521 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -1,4 +1,7 @@ -import { QdrantClient, Schemas } from "@qdrant/js-client-rest" +import { codeIndexLog } from "../code-index-log" +import { parseRetryAfterMs, VectorStoreError } from "../shared/indexing-errors" +import { makeRateLimitScope } from "../provider-pacer" +import { QdrantClient, QdrantClientResourceExhaustedError, Schemas } from "@qdrant/js-client-rest" import { createHash } from "crypto" import * as path from "path" import { v5 as uuidv5 } from "uuid" @@ -7,10 +10,21 @@ import { Payload, VectorStoreSearchResult } from "../interfaces" import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" import { t } from "../../../i18n" +function normalizeVectorStoreError(error: unknown): VectorStoreError { + return new VectorStoreError( + error instanceof Error ? error.message : String(error), + error, + error instanceof QdrantClientResourceExhaustedError + ? { classification: "rate-limit", statusCode: 429, retryAfterMs: parseRetryAfterMs(error.retry_after) } + : undefined, + ) +} + /** * Qdrant implementation of the vector store interface */ export class QdrantVectorStore implements IVectorStore { + readonly rateLimitScope: string private readonly vectorSize!: number private readonly DISTANCE_METRIC = "Cosine" @@ -30,6 +44,7 @@ export class QdrantVectorStore implements IVectorStore { // Store the resolved URL for our property this.qdrantUrl = parsedUrl + this.rateLimitScope = makeRateLimitScope("qdrant", parsedUrl) this.workspacePath = workspacePath try { @@ -133,7 +148,7 @@ export class QdrantVectorStore implements IVectorStore { return collectionInfo } catch (error: unknown) { if (error instanceof Error) { - console.warn( + codeIndexLog.warn( `[QdrantVectorStore] Warning during getCollectionInfo for "${this.collectionName}". Collection may not exist or another error occurred:`, error.message, ) @@ -197,7 +212,7 @@ export class QdrantVectorStore implements IVectorStore { return created } catch (error: any) { const errorMessage = error?.message || error - console.error( + codeIndexLog.error( `[QdrantVectorStore] Failed to initialize Qdrant collection "${this.collectionName}":`, errorMessage, ) @@ -220,7 +235,7 @@ export class QdrantVectorStore implements IVectorStore { * @returns Promise resolving to boolean indicating if a new collection was created */ private async _recreateCollectionWithNewDimension(existingVectorSize: number): Promise { - console.warn( + codeIndexLog.warn( `[QdrantVectorStore] Collection ${this.collectionName} exists with vector size ${existingVectorSize}, but expected ${this.vectorSize}. Recreating collection.`, ) @@ -229,10 +244,10 @@ export class QdrantVectorStore implements IVectorStore { try { // Step 1: Attempt to delete the existing collection - console.log(`[QdrantVectorStore] Deleting existing collection ${this.collectionName}...`) + codeIndexLog.info(`[QdrantVectorStore] Deleting existing collection ${this.collectionName}...`) await this.client.deleteCollection(this.collectionName) deletionSucceeded = true - console.log(`[QdrantVectorStore] Successfully deleted collection ${this.collectionName}`) + codeIndexLog.info(`[QdrantVectorStore] Successfully deleted collection ${this.collectionName}`) // Step 2: Wait a brief moment to ensure deletion is processed await new Promise((resolve) => setTimeout(resolve, 100)) @@ -244,7 +259,7 @@ export class QdrantVectorStore implements IVectorStore { } // Step 4: Create the new collection with correct dimensions - console.log( + codeIndexLog.info( `[QdrantVectorStore] Creating new collection ${this.collectionName} with vector size ${this.vectorSize}...`, ) recreationAttempted = true @@ -260,7 +275,7 @@ export class QdrantVectorStore implements IVectorStore { on_disk: true, }, }) - console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) + codeIndexLog.info(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) return true } catch (recreationError) { const errorMessage = recreationError instanceof Error ? recreationError.message : String(recreationError) @@ -275,7 +290,7 @@ export class QdrantVectorStore implements IVectorStore { contextualErrorMessage = `Deleted existing collection but failed to create new collection with vector size ${this.vectorSize}. ${errorMessage}` } - console.error( + codeIndexLog.error( `[QdrantVectorStore] CRITICAL: Failed to recreate collection ${this.collectionName} for dimension change (${existingVectorSize} -> ${this.vectorSize}). ${contextualErrorMessage}`, ) @@ -296,16 +311,29 @@ export class QdrantVectorStore implements IVectorStore { * Creates payload indexes for the collection, handling errors gracefully. */ private async _createPayloadIndexes(): Promise { + // Completion verification depends on these indexes; initialization must fail if creation fails. + await this.client.createPayloadIndex(this.collectionName, { + field_name: "filePath", + field_schema: "keyword", + wait: true, + }) + await this.client.createPayloadIndex(this.collectionName, { + field_name: "generation", + field_schema: "integer", + wait: true, + }) + // Create index for the 'type' field to enable metadata filtering try { await this.client.createPayloadIndex(this.collectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) } catch (indexError: any) { const errorMessage = (indexError?.message || "").toLowerCase() if (!errorMessage.includes("already exists")) { - console.warn( + codeIndexLog.warn( `[QdrantVectorStore] Could not create payload index for type on ${this.collectionName}. Details:`, indexError?.message || indexError, ) @@ -313,17 +341,18 @@ export class QdrantVectorStore implements IVectorStore { } // Create indexes for pathSegments fields - for (let i = 0; i <= 4; i++) { + for (let segmentIndex = 0; segmentIndex <= 4; segmentIndex++) { try { await this.client.createPayloadIndex(this.collectionName, { - field_name: `pathSegments.${i}`, + field_name: `pathSegments.${segmentIndex}`, field_schema: "keyword", + wait: true, }) } catch (indexError: any) { const errorMessage = (indexError?.message || "").toLowerCase() if (!errorMessage.includes("already exists")) { - console.warn( - `[QdrantVectorStore] Could not create payload index for pathSegments.${i} on ${this.collectionName}. Details:`, + codeIndexLog.warn( + `[QdrantVectorStore] Could not create payload index for pathSegments.${segmentIndex} on ${this.collectionName}. Details:`, indexError?.message || indexError, ) } @@ -369,8 +398,8 @@ export class QdrantVectorStore implements IVectorStore { wait: true, }) } catch (error) { - console.error("Failed to upsert points:", error) - throw error + codeIndexLog.error("Failed to upsert points:", error) + throw normalizeVectorStoreError(error) } } @@ -462,81 +491,64 @@ export class QdrantVectorStore implements IVectorStore { return filteredPoints as VectorStoreSearchResult[] } catch (error) { - console.error("Failed to search points:", error) + codeIndexLog.error("Failed to search points:", error) throw error } } - /** - * Deletes points by file path - * @param filePath Path of the file to delete points for - */ - async deletePointsByFilePath(filePath: string): Promise { - return this.deletePointsByMultipleFilePaths([filePath]) + private relativeFilePath(filePath: string): string { + return path.normalize(path.isAbsolute(filePath) ? path.relative(this.workspacePath, filePath) : filePath) } - async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { - if (filePaths.length === 0) { - return - } - - try { - // First check if the collection exists - const collectionExists = await this.collectionExists() - if (!collectionExists) { - console.warn( - `[QdrantVectorStore] Skipping deletion - collection "${this.collectionName}" does not exist`, - ) - return - } - - const workspaceRoot = this.workspacePath - - // Build filters using pathSegments to match the indexed fields - const filters = filePaths.map((filePath) => { - // IMPORTANT: Use the relative path to match what's stored in upsertPoints - // upsertPoints stores the relative filePath, not the absolute path - const relativePath = path.isAbsolute(filePath) ? path.relative(workspaceRoot, filePath) : filePath - - // Normalize the relative path - const normalizedRelativePath = path.normalize(relativePath) - - // Split the path into segments like we do in upsertPoints - const segments = normalizedRelativePath.split(path.sep).filter(Boolean) - - // Create a filter that matches all segments of the path - // This ensures we only delete points that match the exact file path - const mustConditions = segments.map((segment, index) => ({ - key: `pathSegments.${index}`, - match: { value: segment }, - })) - - return { must: mustConditions } - }) + async deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise { + return this.deletePointsByMultipleFilePaths([filePath], beforeGeneration) + } - // Use 'should' to match any of the file paths (OR condition) - const filter = filters.length === 1 ? filters[0] : { should: filters } + async deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise { + if (!Number.isSafeInteger(beforeGeneration) || beforeGeneration < 0) { + throw new Error("A non-negative safe integer generation cutoff is required") + } + if (filePaths.length === 0) return - await this.client.delete(this.collectionName, { - filter, + await this.client + .delete(this.collectionName, { + filter: { + must: [ + { + key: "filePath", + match: { any: filePaths.map((filePath) => this.relativeFilePath(filePath)) }, + }, + { + should: [ + { key: "generation", range: { lt: beforeGeneration } }, + { is_empty: { key: "generation" } }, + ], + }, + ], + }, wait: true, }) - } catch (error: any) { - // Extract more detailed error information - const errorMessage = error?.message || String(error) - const errorStatus = error?.status || error?.response?.status || error?.statusCode - const errorDetails = error?.response?.data || error?.data || "" - - console.error(`[QdrantVectorStore] Failed to delete points by file paths:`, { - error: errorMessage, - status: errorStatus, - details: errorDetails, - collection: this.collectionName, - fileCount: filePaths.length, - // Include first few file paths for debugging (avoid logging too many) - samplePaths: filePaths.slice(0, 3), + .catch((error: unknown) => { + throw normalizeVectorStoreError(error) }) - } + } + + async countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> { + const fileCondition = { key: "filePath", match: { value: this.relativeFilePath(filePath) } } + const generationCondition = { key: "generation", match: { value: generation } } + const [current, other] = await Promise.all([ + this.client.count(this.collectionName, { + filter: { must: [fileCondition, generationCondition] }, + exact: true, + }), + this.client.count(this.collectionName, { + filter: { must: [fileCondition], must_not: [generationCondition] }, + exact: true, + }), + ]).catch((error: unknown) => { + throw normalizeVectorStoreError(error) + }) + return { current: current.count, other: other.count } } /** @@ -549,7 +561,7 @@ export class QdrantVectorStore implements IVectorStore { await this.client.deleteCollection(this.collectionName) } } catch (error) { - console.error(`[QdrantVectorStore] Failed to delete collection ${this.collectionName}:`, error) + codeIndexLog.error(`[QdrantVectorStore] Failed to delete collection ${this.collectionName}:`, error) throw error // Re-throw to allow calling code to handle it } } @@ -566,7 +578,7 @@ export class QdrantVectorStore implements IVectorStore { wait: true, }) } catch (error) { - console.error("Failed to clear collection:", error) + codeIndexLog.error("Failed to clear collection:", error) throw error } } @@ -610,12 +622,12 @@ export class QdrantVectorStore implements IVectorStore { // Backward compatibility: No marker exists (old index or pre-marker version) // Fall back to old logic - assume complete if collection has points - console.log( + codeIndexLog.info( "[QdrantVectorStore] No indexing metadata marker found. Using backward compatibility mode (checking points_count > 0).", ) return pointsCount > 0 } catch (error) { - console.warn("[QdrantVectorStore] Failed to check if collection has data:", error) + codeIndexLog.warn("[QdrantVectorStore] Failed to check if collection has data:", error) return false } } @@ -644,9 +656,9 @@ export class QdrantVectorStore implements IVectorStore { ], wait: true, }) - console.log("[QdrantVectorStore] Marked indexing as complete") + codeIndexLog.info("[QdrantVectorStore] Marked indexing as complete") } catch (error) { - console.error("[QdrantVectorStore] Failed to mark indexing as complete:", error) + codeIndexLog.error("[QdrantVectorStore] Failed to mark indexing as complete:", error) throw error } } @@ -675,9 +687,9 @@ export class QdrantVectorStore implements IVectorStore { ], wait: true, }) - console.log("[QdrantVectorStore] Marked indexing as incomplete (in progress)") + codeIndexLog.info("[QdrantVectorStore] Marked indexing as incomplete (in progress)") } catch (error) { - console.error("[QdrantVectorStore] Failed to mark indexing as incomplete:", error) + codeIndexLog.error("[QdrantVectorStore] Failed to mark indexing as incomplete:", error) throw error } } diff --git a/src/services/code-index/workspace-indexing-enablement-manager.ts b/src/services/code-index/workspace-indexing-enablement-manager.ts index 78f179ad4a..3552aab210 100644 --- a/src/services/code-index/workspace-indexing-enablement-manager.ts +++ b/src/services/code-index/workspace-indexing-enablement-manager.ts @@ -9,7 +9,6 @@ export class WorkspaceIndexingEnablementManager { | "setWorkspaceEnabled" | "isFeatureEnabled" | "isFeatureConfigured" - | "initialize" | "startIndexing" | "stopIndexing" | "getCurrentStatus" @@ -22,10 +21,9 @@ export class WorkspaceIndexingEnablementManager { ): Promise { await this.manager.setWorkspaceEnabled(enabled) if (enabled && this.manager.isFeatureEnabled && this.manager.isFeatureConfigured) { - await this.manager.initialize(provider.contextProxy) - void this.manager.startIndexing().catch((error) => provider.log(`Indexing error: ${error}`)) + await this.manager.startIndexing(provider.contextProxy) } else if (!enabled) { - this.manager.stopIndexing() + await this.manager.stopIndexing() } await provider.postMessageToWebview({ type: "indexingStatusUpdate", diff --git a/src/workers/codeIndexParse.ts b/src/workers/codeIndexParse.ts new file mode 100644 index 0000000000..a660fc2a3e --- /dev/null +++ b/src/workers/codeIndexParse.ts @@ -0,0 +1 @@ +import "../services/code-index/parse-worker" diff --git a/webview-ui/src/components/chat/CodeIndexPopover.tsx b/webview-ui/src/components/chat/CodeIndexPopover.tsx index 62c811e4b0..0770a77de3 100644 --- a/webview-ui/src/components/chat/CodeIndexPopover.tsx +++ b/webview-ui/src/components/chat/CodeIndexPopover.tsx @@ -15,6 +15,7 @@ import { TriangleAlert } from "lucide-react" import { type IndexingStatus, type EmbedderProvider, + type IndexingStatusUpdateMessage, CODEBASE_INDEX_DEFAULTS, providerIdentifiers, } from "@roo-code/types" @@ -206,6 +207,9 @@ export const CodeIndexPopover: React.FC = ({ const [isSetupSettingsOpen, setIsSetupSettingsOpen] = useState(false) const [indexingStatus, setIndexingStatus] = useState(externalIndexingStatus) + const failedFileSummary = indexingStatus.failedFileSummary + const failedFiles = indexingStatus.failedFiles ?? [] + const hasFailedFiles = failedFiles.length > 0 || (failedFileSummary?.count ?? 0) > 0 const [saveStatus, setSaveStatus] = useState<"idle" | "saving" | "saved" | "error">("idle") const [saveError, setSaveError] = useState(null) @@ -318,16 +322,14 @@ export const CodeIndexPopover: React.FC = ({ // Listen for indexing status updates and save responses useEffect(() => { - const handleMessage = (event: MessageEvent) => { + const handleMessage = ( + event: MessageEvent< + IndexingStatusUpdateMessage | { type: "codeIndexSettingsSaved"; success: boolean; error?: string } + >, + ) => { if (event.data.type === "indexingStatusUpdate") { if (!event.data.values.workspacePath || event.data.values.workspacePath === cwd) { - setIndexingStatus({ - systemStatus: event.data.values.systemStatus, - message: event.data.values.message || "", - processedItems: event.data.values.processedItems, - totalItems: event.data.values.totalItems, - currentItemUnit: event.data.values.currentItemUnit || "items", - }) + setIndexingStatus(event.data.values) } } else if (event.data.type === "codeIndexSettingsSaved") { if (event.data.success) { @@ -671,16 +673,91 @@ export const CodeIndexPopover: React.FC = ({
{t(`settings:codeIndex.indexingStatuses.${indexingStatus.systemStatus.toLowerCase()}`)} {indexingStatus.message ? ` - ${indexingStatus.message}` : ""}
+ {indexingStatus.providerWaits?.map((wait, index) => ( +
+

+ {t( + indexingStatus.systemStatus === "Indexing" + ? wait.resumeAt === undefined + ? "settings:codeIndex.providerWaiting" + : "settings:codeIndex.providerWaitingUntil" + : wait.resumeAt === undefined + ? "settings:codeIndex.providerPaused" + : "settings:codeIndex.providerPausedUntil", + { + service: wait.service, + time: + wait.resumeAt === undefined + ? undefined + : new Date(wait.resumeAt).toLocaleString(), + }, + )} +

+
    + {wait.waitingFiles.map((file) => ( +
  • {file}
    • + ))} +
+
+ ))} + + {hasFailedFiles && ( +
+ {failedFileSummary && failedFileSummary.count > 0 && ( +

+ {t( + failedFileSummary.state === "retrying" + ? "settings:codeIndex.indexingFilesRetrying" + : "settings:codeIndex.indexingFilesFailed", + { ...failedFileSummary }, + )} +

+ )} +
+ + {t("settings:codeIndex.failedFileDetails")} ({failedFiles.length}) + +
    + {failedFiles.map((failedFile) => ( +
  • +
    + {failedFile.path} +
    +
    + {t( + `settings:codeIndex.failedFileActivities.${failedFile.activity}`, + )} +
    +
    + {t(`settings:codeIndex.failedFileStates.${failedFile.state}`)} +
    +
    + {t("settings:codeIndex.failedFileReason")}: {failedFile.reason} +
    +
    • + ))} +
+
+
+ )} + {indexingStatus.systemStatus === "Indexing" && (
= ({
{currentSettings.codebaseIndexEnabled && - (indexingStatus.systemStatus === "Error" || + (hasFailedFiles || + indexingStatus.systemStatus === "Error" || indexingStatus.systemStatus === "Standby") && ( )} diff --git a/webview-ui/src/components/chat/IndexingStatusBadge.tsx b/webview-ui/src/components/chat/IndexingStatusBadge.tsx index 22dbe8f9d5..d49aefb8c6 100644 --- a/webview-ui/src/components/chat/IndexingStatusBadge.tsx +++ b/webview-ui/src/components/chat/IndexingStatusBadge.tsx @@ -57,7 +57,10 @@ export const IndexingStatusBadge: React.FC = ({ classN [indexingStatus.processedItems, indexingStatus.totalItems], ) - const tooltipText = useMemo(() => { + const hasFailedFiles = + (indexingStatus.failedFiles?.length ?? 0) > 0 || (indexingStatus.failedFileSummary?.count ?? 0) > 0 + + const statusTooltip = useMemo(() => { switch (indexingStatus.systemStatus) { case "Standby": return t("chat:indexingStatus.ready") @@ -74,7 +77,32 @@ export const IndexingStatusBadge: React.FC = ({ classN } }, [indexingStatus.systemStatus, progressPercentage, t]) + const tooltipText = useMemo(() => { + if (!hasFailedFiles || indexingStatus.systemStatus === "Error" || indexingStatus.systemStatus === "Stopping") { + return statusTooltip + } + + const summary = indexingStatus.failedFileSummary + const failureText = + summary && summary.count > 0 + ? t( + summary.state === "retrying" + ? "settings:codeIndex.indexingFilesRetrying" + : "settings:codeIndex.indexingFilesFailed", + { ...summary }, + ) + : `${t("settings:codeIndex.failedFileDetails")} (${indexingStatus.failedFiles?.length ?? 0})` + return `${statusTooltip} - ${failureText}` + }, [hasFailedFiles, indexingStatus, statusTooltip, t]) + const statusColorClass = useMemo(() => { + if ( + hasFailedFiles && + (indexingStatus.systemStatus === "Standby" || indexingStatus.systemStatus === "Indexed") + ) { + return "bg-amber-500" + } + const statusColors = { Standby: "bg-vscode-descriptionForeground/60", Indexing: "bg-yellow-500 animate-pulse", @@ -84,7 +112,7 @@ export const IndexingStatusBadge: React.FC = ({ classN } return statusColors[indexingStatus.systemStatus as keyof typeof statusColors] || statusColors.Standby - }, [indexingStatus.systemStatus]) + }, [hasFailedFiles, indexingStatus.systemStatus]) return ( diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx index e27a10d51c..40b4b758f0 100644 --- a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx +++ b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx @@ -1,13 +1,36 @@ import React from "react" +import { createInstance } from "i18next" +import englishSettings from "@/i18n/locales/en/settings.json" +import germanSettings from "@/i18n/locales/de/settings.json" -import type { EmbedderProvider, ExtensionState } from "@roo-code/types" +import type { + EmbedderProvider, + ExtensionState, + FailedFileStatus, + IndexingStatus, + IndexingStatusUpdateMessage, +} from "@roo-code/types" import { providerIdentifiers } from "@roo-code/types" -import { fireEvent, renderWithExtensionState, screen } from "@/utils/test-utils" +import { act, fireEvent, renderWithExtensionState, screen, within } from "@/utils/test-utils" +import { vscode } from "@src/utils/vscode" import { useOpenRouterModelProviders } from "@src/components/ui/hooks/useOpenRouterModelProviders" import { CodeIndexPopover, createValidationSchema } from "../CodeIndexPopover" +vi.mock("@src/utils/vscode", () => ({ vscode: { postMessage: vi.fn() } })) + +vi.mock("@src/i18n/TranslationContext", () => ({ + useAppTranslation: () => ({ + t: (key: string, parameters?: Record) => + key === "settings:codeIndex.indexingFilesRetrying" || key === "settings:codeIndex.indexingFilesFailed" + ? `${key}: ${parameters?.count} / ${parameters?.path} / ${parameters?.reason}` + : key.startsWith("settings:codeIndex.provider") + ? translation.t(key, parameters) + : key, + }), +})) + vi.mock("@src/components/ui/hooks/useOpenRouterModelProviders", () => ({ OPENROUTER_DEFAULT_PROVIDER_NAME: "Auto", useOpenRouterModelProviders: vi.fn(() => ({ data: undefined })), @@ -43,8 +66,8 @@ vi.mock("@src/components/ui", async (importOriginal) => { } }) -const indexingStatus = { - systemStatus: "Standby" as const, +const indexingStatus: IndexingStatus = { + systemStatus: "Standby", message: "", processedItems: 0, totalItems: 0, @@ -58,6 +81,15 @@ const baseSettings = { } const translate = (key: string) => key +const translation = createInstance() + +beforeAll(async () => { + await translation.init({ + lng: "en", + resources: { en: { settings: englishSettings }, de: { settings: germanSettings } }, + interpolation: { escapeValue: false }, + }) +}) describe("CodeIndexPopover validation", () => { it.each([ @@ -228,3 +260,266 @@ describe("CodeIndexPopover OpenRouter provider lookup", () => { expect(screen.getByText("settings:codeIndex.validation.geminiApiKeyRequired")).toBeInTheDocument() }) }) + +const failedStatus = ( + state: FailedFileStatus["state"], + systemStatus = state === "retrying" ? "Indexing" : "Standby", +): IndexingStatus => { + const failure: FailedFileStatus = { + path: "src/failed.ts", + activity: "embed", + reason: "Provider unavailable", + state, + } + return { + ...indexingStatus, + systemStatus, + workspacePath: "/workspace", + workspaceEnabled: true, + autoEnableDefault: false, + processedItems: 2, + totalItems: 8, + failedFiles: [failure], + failedFileSummary: { ...failure, count: 1 }, + } +} + +const renderStatusPopover = (status: IndexingStatus) => + renderWithExtensionState( + + + , + { + state: { + cwd: "/workspace", + codebaseIndexConfig: { ...baseSettings, codebaseIndexEmbedderProvider: providerIdentifiers.openai }, + codebaseIndexModels: {}, + platform: "win32", + arch: "x64", + }, + }, + ) + +const publishStatus = (values: IndexingStatus) => { + const data: IndexingStatusUpdateMessage = { type: "indexingStatusUpdate", values } + act(() => window.dispatchEvent(new MessageEvent("message", { data }))) +} + +describe("CodeIndexPopover indexing reliability", () => { + beforeEach(async () => { + vi.clearAllMocks() + await translation.changeLanguage("en") + }) + + it.each(["Indexing", "Indexed", "Standby", "Stopping", "Error"])( + "localizes timed and untimed waits without false resume promises in %s", + async (systemStatus) => { + await translation.changeLanguage("de") + const resumeAt = Date.parse("2026-11-11T12:02:00Z") + renderStatusPopover({ + ...indexingStatus, + systemStatus, + providerWaits: [ + { service: "openai", resumeAt, waitingFiles: ["waiting.ts"] }, + { service: "qdrant", waitingFiles: [] }, + ], + }) + const active = systemStatus === "Indexing" + const timed = active ? "providerWaitingUntil" : "providerPausedUntil" + const untimed = active ? "providerWaiting" : "providerPaused" + expect( + screen.getByText( + translation.t(`settings:codeIndex.${timed}`, { + service: "openai", + time: new Date(resumeAt).toLocaleString(), + }), + ), + ).toBeVisible() + expect( + screen.getByText(translation.t(`settings:codeIndex.${untimed}`, { service: "qdrant" })), + ).toBeVisible() + expect(screen.queryByText(/Waiting for/)).not.toBeInTheDocument() + if (!active) expect(screen.queryByText(/automatisch fortgesetzt/)).not.toBeInTheDocument() + }, + ) + + it("keeps two provider waits separate from genuine failures and Stop enabled", () => { + renderStatusPopover({ + ...failedStatus("retrying"), + providerWaits: [ + { service: "openai", resumeAt: Date.now() + 120_000, waitingFiles: ["src/waiting-embed.ts"] }, + { service: "qdrant", waitingFiles: ["src/waiting-store.ts"] }, + ], + }) + expect(screen.getByText(/Waiting for openai; indexing resumes automatically at/)).toBeVisible() + expect(screen.getByText("Waiting for qdrant; indexing resumes automatically")).toBeVisible() + for (const path of ["src/waiting-embed.ts", "src/waiting-store.ts"]) { + expect(screen.getByText(path)).toBeVisible() + expect(screen.getByRole("status")).not.toHaveTextContent(path) + } + expect(screen.getByRole("status")).toHaveTextContent("1 / src/failed.ts") + const stop = screen.getByRole("button", { name: "settings:codeIndex.stopIndexingButton" }) + expect(stop).toBeEnabled() + fireEvent.click(stop) + expect(vscode.postMessage).toHaveBeenCalledWith({ type: "stopIndexing" }) + publishStatus({ ...indexingStatus, providerWaits: [], failedFiles: [] }) + expect(screen.queryByText(/Waiting for/)).not.toBeInTheDocument() + expect(screen.queryByRole("status")).not.toBeInTheDocument() + }) + + it.each(["empty", "omitted"])("clears waiting-only snapshots when waits are %s", (snapshot) => { + renderStatusPopover({ + ...indexingStatus, + systemStatus: "Indexing", + providerWaits: [{ service: "openai", waitingFiles: ["waiting.ts"] }], + }) + expect(screen.getByText("waiting.ts")).toBeVisible() + expect(screen.queryByRole("status")).not.toBeInTheDocument() + expect(screen.queryByText(/settings:codeIndex.failedFileDetails/)).not.toBeInTheDocument() + expect(screen.queryByRole("button", { name: "settings:codeIndex.startIndexingButton" })).not.toBeInTheDocument() + expect(screen.getByRole("button", { name: "settings:codeIndex.stopIndexingButton" })).toBeEnabled() + publishStatus({ ...indexingStatus, providerWaits: snapshot === "empty" ? [] : undefined }) + expect(screen.queryByText("waiting.ts")).not.toBeInTheDocument() + expect(screen.queryByText(/Waiting for/)).not.toBeInTheDocument() + expect(screen.queryByRole("status")).not.toBeInTheDocument() + expect(screen.getByRole("button", { name: "settings:codeIndex.startIndexingButton" })).toBeEnabled() + }) + + it("replaces waits from parent snapshots and ignores waits for other workspaces", () => { + const view = renderStatusPopover({ + ...indexingStatus, + providerWaits: [{ service: "openai", waitingFiles: ["waiting.ts"] }], + }) + publishStatus({ ...indexingStatus, workspacePath: "/other", providerWaits: [] }) + expect(screen.getByText("waiting.ts")).toBeVisible() + view.rerender( + + + , + ) + expect(screen.queryByText("waiting.ts")).not.toBeInTheDocument() + }) + + it.each(["retrying", "exhausted"] as const)("shows %s summary and expandable file details", (state) => { + renderStatusPopover(failedStatus(state)) + const summaryKey = state === "retrying" ? "indexingFilesRetrying" : "indexingFilesFailed" + expect(screen.getByRole("status")).toHaveTextContent( + `settings:codeIndex.${summaryKey}: 1 / src/failed.ts / Provider unavailable`, + ) + const summary = screen.getByText("settings:codeIndex.failedFileDetails (1)") + const details = summary.closest("details")! + expect(details).not.toHaveAttribute("open") + fireEvent.click(summary) + expect(within(details).getByText("src/failed.ts")).toBeInTheDocument() + expect(within(details).getByText("settings:codeIndex.failedFileActivities.embed")).toBeInTheDocument() + expect(within(details).getByText(`settings:codeIndex.failedFileStates.${state}`)).toBeInTheDocument() + expect( + within(details).getByText("settings:codeIndex.failedFileReason: Provider unavailable"), + ).toBeInTheDocument() + }) + + it("renders all six failure activities and both retry states", () => { + const activities: FailedFileStatus["activity"][] = ["read", "parse", "embed", "upsert", "delete", "verify"] + const failedFiles: FailedFileStatus[] = activities.map((activity, index) => ({ + path: `src/${activity}.ts`, + activity, + reason: `${activity} failed`, + state: index % 2 === 0 ? "retrying" : "exhausted", + })) + renderStatusPopover({ ...failedStatus("retrying"), failedFiles, failedFileSummary: undefined }) + const details = screen.getByText("settings:codeIndex.failedFileDetails (6)").closest("details")! + fireEvent.click(within(details).getByText("settings:codeIndex.failedFileDetails (6)")) + for (const failure of failedFiles) { + const entry = within(details).getByText(failure.path).closest("li")! + expect( + within(entry).getByText(`settings:codeIndex.failedFileActivities.${failure.activity}`), + ).toBeInTheDocument() + expect(within(entry).getByText(`settings:codeIndex.failedFileStates.${failure.state}`)).toBeInTheDocument() + expect( + within(entry).getByText(`settings:codeIndex.failedFileReason: ${failure.reason}`), + ).toBeInTheDocument() + } + }) + + it("allows explicit Start for idle partial Standby", () => { + renderStatusPopover(failedStatus("exhausted")) + const start = screen.getByRole("button", { name: "settings:codeIndex.startIndexingButton" }) + expect(start).toBeEnabled() + fireEvent.click(start) + expect(vscode.postMessage).toHaveBeenCalledWith({ type: "startIndexing" }) + }) + + it.each(["Indexing", "Stopping"])("keeps Start discoverable but disabled during %s", (systemStatus) => { + renderStatusPopover(failedStatus("retrying", systemStatus)) + const start = screen.getByRole("button", { name: "settings:codeIndex.startIndexingButton" }) + expect(start).toBeDisabled() + fireEvent.click(start) + expect(vscode.postMessage).not.toHaveBeenCalledWith({ type: "startIndexing" }) + if (systemStatus === "Stopping") { + expect(screen.getByRole("button", { name: "settings:codeIndex.stoppingButton" })).toBeDisabled() + } else { + fireEvent.click(screen.getByRole("button", { name: "settings:codeIndex.stopIndexingButton" })) + expect(vscode.postMessage).toHaveBeenCalledWith({ type: "stopIndexing" }) + } + }) + + it.each([ + ["Indexing", "Embedding batch 2 of 8"], + ["Error", "Parser timeout at src/hung.ts. Start indexing to retry or disable indexing. Not a provider error."], + ["Stopping", "Stopping parser and pending retries"], + ["Standby", "Indexing stopped. Partial index data preserved."], + ])("preserves the backend %s message alongside failures", (systemStatus, message) => { + renderStatusPopover({ ...failedStatus("exhausted", systemStatus), message }) + expect( + screen.getByText(`settings:codeIndex.indexingStatuses.${systemStatus.toLowerCase()} - ${message}`), + ).toBeInTheDocument() + expect(screen.getByRole("status")).toHaveTextContent("settings:codeIndex.indexingFilesFailed") + if (systemStatus === "Indexing") { + expect(screen.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "25") + } + }) + + it.each([{ failedFiles: undefined }, { failedFiles: [] }])( + "omits failure UI for zero failures: %s", + ({ failedFiles }) => { + renderStatusPopover({ + ...failedStatus("exhausted", "Indexed"), + failedFiles, + failedFileSummary: { ...failedStatus("exhausted").failedFileSummary!, count: 0 }, + }) + expect(screen.queryByRole("status")).not.toBeInTheDocument() + expect(screen.queryByText(/settings:codeIndex.failedFileDetails/)).not.toBeInTheDocument() + expect( + screen.queryByRole("button", { name: "settings:codeIndex.startIndexingButton" }), + ).not.toBeInTheDocument() + }, + ) + + it("preserves whole status messages and filters other workspaces", () => { + renderStatusPopover(indexingStatus) + const status = { ...failedStatus("exhausted"), message: "Partial index available", autoEnableDefault: true } + publishStatus(status) + expect(screen.getByText("settings:codeIndex.failedFileDetails (1)")).toBeInTheDocument() + expect(screen.getByRole("checkbox", { name: "settings:codeIndex.workspaceToggleLabel" })).toBeChecked() + expect(screen.getByRole("checkbox", { name: "settings:codeIndex.autoEnableDefaultLabel" })).toBeChecked() + publishStatus({ ...indexingStatus, systemStatus: "Error", message: "Wrong workspace", workspacePath: "/other" }) + expect(screen.queryByText(/Wrong workspace/)).not.toBeInTheDocument() + expect(screen.getByText(/Partial index available/)).toBeInTheDocument() + publishStatus({ ...indexingStatus, systemStatus: "Indexed" }) + expect(screen.queryByText(/settings:codeIndex.failedFileDetails/)).not.toBeInTheDocument() + expect(screen.getByText("settings:codeIndex.indexingStatuses.indexed")).toBeInTheDocument() + expect(screen.getByRole("checkbox", { name: "settings:codeIndex.workspaceToggleLabel" })).not.toBeChecked() + }) + + it("accepts whole status updates from the badge prop", () => { + const view = renderStatusPopover(indexingStatus) + view.rerender( + + + , + ) + expect(screen.getByRole("status")).toHaveTextContent("settings:codeIndex.indexingFilesRetrying") + expect(screen.getByRole("checkbox", { name: "settings:codeIndex.workspaceToggleLabel" })).toBeChecked() + expect(screen.getByRole("progressbar")).toHaveAttribute("aria-valuenow", "25") + }) +}) diff --git a/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx b/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx index 04ba9cc60e..2eb7f24337 100644 --- a/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx +++ b/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx @@ -1,264 +1,233 @@ import React from "react" -import { render, screen, fireEvent, waitFor, act } from "@/utils/test-utils" - +import type { IndexingStatus, IndexingStatusUpdateMessage } from "@roo-code/types" +import { render, screen, fireEvent, act } from "@/utils/test-utils" import { vscode } from "@src/utils/vscode" - +import { CodeIndexPopover } from "../CodeIndexPopover" import { IndexingStatusBadge } from "../IndexingStatusBadge" -vi.mock("@/i18n/setup", () => ({ - __esModule: true, - default: { - use: vi.fn().mockReturnThis(), - init: vi.fn().mockReturnThis(), - addResourceBundle: vi.fn(), - language: "en", - changeLanguage: vi.fn(), - }, -})) +vi.mock("@src/utils/vscode", () => ({ vscode: { postMessage: vi.fn() } })) -vi.mock("react-i18next", () => ({ - useTranslation: () => ({ - t: (key: string, params?: any) => { - const translations: Record = { - "indexingStatus.ready": "Index ready", - "indexingStatus.indexing": params?.progress !== undefined ? `Indexing ${params.progress}%` : "Indexing", - "indexingStatus.error": "Index error", - "indexingStatus.indexed": "Indexed", - "indexingStatus.tooltip.ready": "The codebase index is ready for use", - "indexingStatus.tooltip.indexing": - params?.progress !== undefined - ? `Indexing in progress: ${params.progress}% complete` - : "Indexing in progress", - "indexingStatus.tooltip.error": "An error occurred during indexing", - "indexingStatus.tooltip.indexed": "Codebase has been successfully indexed", - "indexingStatus.tooltip.clickToSettings": "Click to open indexing settings", - } - return translations[key] || key - }, - i18n: { - language: "en", - changeLanguage: vi.fn(), - }, - }), - initReactI18next: { - type: "3rdParty", - init: vi.fn(), - }, - Trans: ({ children }: { children: React.ReactNode }) => <>{children}, +vi.mock("@src/context/ExtensionStateContext", () => ({ + useExtensionState: () => ({ cwd: "/workspace" }), + ExtensionStateContextProvider: ({ children }: React.PropsWithChildren) => <>{children}, })) -// Mock vscode API -vi.mock("@src/utils/vscode", () => ({ - vscode: { - postMessage: vi.fn(), - }, +vi.mock("../CodeIndexPopover", () => ({ + CodeIndexPopover: vi.fn(({ children }: React.PropsWithChildren<{ indexingStatus: IndexingStatus }>) => ( + <>{children} + )), })) -// Mock the ExtensionStateContext -vi.mock("@/context/ExtensionStateContext", () => ({ - useExtensionState: () => ({ - version: "1.0.0", - clineMessages: [], - taskHistory: [], - shouldShowAnnouncement: false, - language: "en", - }), - ExtensionStateContextProvider: ({ children }: { children: React.ReactNode }) => <>{children}, -})) +vi.mock("@src/components/ui", async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + PopoverTrigger: ({ children }: React.PropsWithChildren) => <>{children}, + StandardTooltip: ({ children }: React.PropsWithChildren) => <>{children}, + } +}) -// Mock TranslationContext to provide t function directly vi.mock("@/i18n/TranslationContext", () => ({ useAppTranslation: () => ({ - t: (key: string, params?: any) => { - // Remove namespace prefix if present - const cleanKey = key.includes(":") ? key.split(":")[1] : key - + t: (key: string, parameters?: Record) => { const translations: Record = { - "indexingStatus.ready": "Index ready", - "indexingStatus.indexing": - params?.percentage !== undefined ? `Indexing ${params.percentage}%` : "Indexing", - "indexingStatus.error": "Index error", - "indexingStatus.indexed": "Indexed", - "indexingStatus.tooltip.ready": "The codebase index is ready for use", - "indexingStatus.tooltip.indexing": - params?.percentage !== undefined - ? `Indexing in progress: ${params.percentage}% complete` - : "Indexing in progress", - "indexingStatus.tooltip.error": "An error occurred during indexing", - "indexingStatus.tooltip.indexed": "Codebase has been successfully indexed", - "indexingStatus.tooltip.clickToSettings": "Click to open indexing settings", + "chat:indexingStatus.ready": "Index ready", + "chat:indexingStatus.indexing": `Indexing ${parameters?.percentage}%`, + "chat:indexingStatus.indexed": "Indexed", + "chat:indexingStatus.error": "Index error", + "chat:indexingStatus.stopping": "Stopping indexing...", + "chat:indexingStatus.status": "Index status", + "settings:codeIndex.indexingFilesRetrying": `Retrying ${parameters?.count}: ${parameters?.path}: ${parameters?.reason}`, + "settings:codeIndex.indexingFilesFailed": `Failed ${parameters?.count}: ${parameters?.path}: ${parameters?.reason}`, + "settings:codeIndex.failedFileDetails": "Failed file details", } - return translations[cleanKey] || cleanKey + return translations[key] ?? key }, }), })) -describe("IndexingStatusBadge", () => { - const renderComponent = (props = {}) => { - return render() +const baseStatus: IndexingStatus = { + systemStatus: "Standby", + processedItems: 0, + totalItems: 0, + currentItemUnit: "files", +} + +const failureStatus = (state: "retrying" | "exhausted"): IndexingStatus => { + const failure = { path: "src/failed.ts", activity: "embed", reason: "Provider unavailable", state } as const + return { + ...baseStatus, + systemStatus: state === "retrying" ? "Indexing" : "Standby", + processedItems: 25, + totalItems: 100, + workspacePath: "/workspace", + workspaceEnabled: true, + autoEnableDefault: false, + message: "Backend status message", + failedFiles: [failure], + failedFileSummary: { ...failure, count: 1 }, } +} - beforeEach(() => { - vi.clearAllMocks() - }) - - it("renders the status dot", () => { - renderComponent() - const button = screen.getByRole("button") - expect(button).toBeInTheDocument() - }) - - it("shows standby status by default", () => { - renderComponent() - const button = screen.getByRole("button") - expect(button).toHaveAttribute("aria-label", "Index ready") - }) - - it("opens popover when clicked", () => { - // Mock window.postMessage - const postMessageSpy = vi.spyOn(window, "postMessage") +const publishStatus = (values: IndexingStatus) => { + const data: IndexingStatusUpdateMessage = { type: "indexingStatusUpdate", values } + act(() => window.dispatchEvent(new MessageEvent("message", { data }))) +} - renderComponent() +const statusDot = () => screen.getByRole("button").querySelector("span") - const button = screen.getByRole("button") - fireEvent.click(button) +const lastPopoverStatus = () => vi.mocked(CodeIndexPopover).mock.calls.at(-1)?.[0].indexingStatus - // The button should be clickable and not post a message anymore - // Since the popover is rendered conditionally, we just verify the button is clickable - // and doesn't throw any errors. The actual popover rendering is tested in integration tests. - expect(postMessageSpy).not.toHaveBeenCalled() +describe("IndexingStatusBadge", () => { + beforeEach(() => vi.clearAllMocks()) - postMessageSpy.mockRestore() + it("renders an accessible, clickable standby control and requests initial status", () => { + render() + fireEvent.click(screen.getByRole("button", { name: "Index ready" })) + expect(vscode.postMessage).toHaveBeenCalledWith({ type: "requestIndexingStatus" }) + expect(statusDot()).toHaveClass("bg-vscode-descriptionForeground/60") }) - it("requests indexing status on mount", () => { - renderComponent() - - expect(vscode.postMessage).toHaveBeenCalledWith({ - type: "requestIndexingStatus", - }) + it.each([ + ["Standby", "Index ready", "bg-vscode-descriptionForeground/60"], + ["Indexed", "Indexed", "bg-green-500"], + ["Error", "Index error", "bg-red-500"], + ["Stopping", "Stopping indexing...", "bg-amber-500"], + ["Unknown", "Index status", "bg-vscode-descriptionForeground/60"], + ])("preserves %s status without failures", (systemStatus, label, color) => { + render() + publishStatus({ ...baseStatus, systemStatus }) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", label) + expect(statusDot()).toHaveClass(color) }) - it("updates status when receiving indexingStatusUpdate message", async () => { - renderComponent() - - // Simulate receiving an indexing status update - const event = new MessageEvent("message", { - data: { - type: "indexingStatusUpdate", - values: { - systemStatus: "Indexing", - processedItems: 50, - totalItems: 100, - currentItemUnit: "files", - }, - }, - }) - - act(() => { - window.dispatchEvent(event) - }) - - await waitFor(() => { - const button = screen.getByRole("button") - expect(button).toHaveAttribute("aria-label", "Indexing 50%") - }) + it.each([ + [0, 100, 0], + [25, 100, 25], + [1, 3, 33], + [100, 100, 100], + [0, 0, 0], + ])("keeps progress for %s/%s items", (processedItems, totalItems, percentage) => { + render() + publishStatus({ ...baseStatus, systemStatus: "Indexing", processedItems, totalItems }) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", `Indexing ${percentage}%`) + expect(statusDot()).toHaveClass("bg-yellow-500", "animate-pulse") }) - it("shows error status correctly", async () => { - renderComponent() - - // Simulate error status - const event = new MessageEvent("message", { - data: { - type: "indexingStatusUpdate", - values: { - systemStatus: "Error", - processedItems: 0, - totalItems: 0, - currentItemUnit: "files", - }, - }, - }) - - act(() => { - window.dispatchEvent(event) - }) - - await waitFor(() => { - const button = screen.getByRole("button") - expect(button).toHaveAttribute("aria-label", "Index error") - }) + it.each(["retrying", "exhausted"] as const)( + "indicates %s failures and passes the whole status to the popover", + (state) => { + render() + const status = failureStatus(state) + publishStatus(status) + const prefix = state === "retrying" ? "Indexing 25% - Retrying" : "Index ready - Failed" + expect(screen.getByRole("button")).toHaveAttribute( + "aria-label", + `${prefix} 1: src/failed.ts: Provider unavailable`, + ) + expect(statusDot()).toHaveClass(state === "retrying" ? "bg-yellow-500" : "bg-amber-500") + expect(lastPopoverStatus()).toEqual(status) + }, + ) + + it.each(["Error", "Stopping"])("does not replace %s with a failure summary", (systemStatus) => { + render() + const status = { ...failureStatus("retrying"), systemStatus, message: "Fatal or stopping message" } + publishStatus(status) + expect(screen.getByRole("button")).toHaveAttribute( + "aria-label", + systemStatus === "Error" ? "Index error" : "Stopping indexing...", + ) + expect(statusDot()).toHaveClass(systemStatus === "Error" ? "bg-red-500" : "bg-amber-500") + expect(lastPopoverStatus()).toEqual(status) }) - it("shows indexed status correctly", async () => { - renderComponent() - - // Simulate indexed status - const event = new MessageEvent("message", { - data: { - type: "indexingStatusUpdate", - values: { - systemStatus: "Indexed", - processedItems: 100, - totalItems: 100, - currentItemUnit: "files", - }, - }, - }) - - act(() => { - window.dispatchEvent(event) - }) - - await waitFor(() => { - const button = screen.getByRole("button") - expect(button).toHaveAttribute("aria-label", "Indexed") - }) + it("indicates list-only failures without inventing a retry state", () => { + render() + publishStatus({ ...failureStatus("exhausted"), failedFileSummary: undefined }) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Index ready - Failed file details (1)") + expect(statusDot()).toHaveClass("bg-amber-500") }) - it("cleans up event listener on unmount", () => { - const { unmount } = renderComponent() - const removeEventListenerSpy = vi.spyOn(window, "removeEventListener") - - unmount() - - expect(removeEventListenerSpy).toHaveBeenCalledWith("message", expect.any(Function)) + it.each([{ failedFiles: undefined }, { failedFiles: [] }])( + "keeps zero-failure compatibility: %s", + ({ failedFiles }) => { + render() + publishStatus({ + ...failureStatus("exhausted"), + systemStatus: "Indexed", + failedFiles, + failedFileSummary: { ...failureStatus("exhausted").failedFileSummary!, count: 0 }, + }) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Indexed") + expect(statusDot()).toHaveClass("bg-green-500") + }, + ) + + it.each(["empty", "omitted"])("forwards waits and clears them when %s in a new snapshot", (snapshot) => { + render() + const waiting = { + ...baseStatus, + systemStatus: "Indexing", + providerWaits: [{ service: "openai", resumeAt: 123_000, waitingFiles: ["waiting.ts"] }], + } + publishStatus(waiting) + expect(lastPopoverStatus()).toEqual(waiting) + expect(statusDot()).toHaveClass("bg-yellow-500") + const recovered = { + ...baseStatus, + systemStatus: "Indexed", + providerWaits: snapshot === "empty" ? [] : undefined, + } + publishStatus(recovered) + expect(lastPopoverStatus()).toEqual(recovered) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Indexed") + expect(statusDot()).toHaveClass("bg-green-500") }) - it("calculates progress percentage correctly", async () => { - renderComponent() + it("preserves Indexed for a search-only pause without file work", () => { + render() + const status = { + ...baseStatus, + systemStatus: "Indexed", + providerWaits: [{ service: "openai", waitingFiles: [] }], + } + publishStatus(status) + expect(lastPopoverStatus()).toEqual(status) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Indexed") + expect(statusDot()).toHaveClass("bg-green-500") + }) - // Test various progress scenarios - const testCases = [ - { processed: 0, total: 100, expected: 0 }, - { processed: 25, total: 100, expected: 25 }, - { processed: 33, total: 100, expected: 33 }, - { processed: 100, total: 100, expected: 100 }, - { processed: 0, total: 0, expected: 0 }, - ] + it("replaces failure fields when recovery arrives", () => { + render() + publishStatus(failureStatus("retrying")) + const recovered = { ...baseStatus, systemStatus: "Indexed", failedFiles: [] } + publishStatus(recovered) + expect(lastPopoverStatus()).toEqual(recovered) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Indexed") + }) - for (const testCase of testCases) { - const event = new MessageEvent("message", { - data: { - type: "indexingStatusUpdate", - values: { - systemStatus: "Indexing", - processedItems: testCase.processed, - totalItems: testCase.total, - currentItemUnit: "files", - }, - }, - }) + it("accepts this workspace and unscoped updates but ignores other workspaces", () => { + render() + const status = failureStatus("exhausted") + publishStatus(status) + publishStatus({ ...baseStatus, systemStatus: "Error", workspacePath: "/other" }) + expect(lastPopoverStatus()).toEqual(status) + publishStatus({ ...baseStatus, systemStatus: "Indexed" }) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Indexed") + }) - act(() => { - window.dispatchEvent(event) - }) + it("ignores unrelated messages", () => { + render() + act(() => window.dispatchEvent(new MessageEvent("message", { data: { type: "workspaceUpdated" } }))) + expect(screen.getByRole("button")).toHaveAttribute("aria-label", "Index ready") + }) - await waitFor(() => { - const button = screen.getByRole("button") - expect(button).toHaveAttribute("aria-label", `Indexing ${testCase.expected}%`) - }) - } + it("cleans up its event listener on unmount", () => { + const { unmount } = render() + const removeListener = vi.spyOn(window, "removeEventListener") + unmount() + expect(removeListener).toHaveBeenCalledWith("message", expect.any(Function)) + removeListener.mockRestore() }) }) diff --git a/webview-ui/src/components/chat/__tests__/provider-wait-locales.spec.ts b/webview-ui/src/components/chat/__tests__/provider-wait-locales.spec.ts new file mode 100644 index 0000000000..2a818dfbd2 --- /dev/null +++ b/webview-ui/src/components/chat/__tests__/provider-wait-locales.spec.ts @@ -0,0 +1,78 @@ +import { createInstance } from "i18next" + +interface WaitMessages { + providerWaiting: string + providerWaitingUntil: string + providerPaused: string + providerPausedUntil: string +} + +interface SettingsMessages { + codeIndex: WaitMessages +} + +interface EmbeddingMessages { + searchProviderRateLimited: string + orchestrator: Pick +} + +const settingsLocales = import.meta.glob("../../../i18n/locales/*/settings.json", { + eager: true, + import: "default", +}) +const embeddingLocales = import.meta.glob("../../../../../src/i18n/locales/*/embeddings.json", { + eager: true, + import: "default", +}) +const locales = [ + "ca", + "de", + "en", + "es", + "fr", + "hi", + "id", + "it", + "ja", + "ko", + "nl", + "pl", + "pt-BR", + "ru", + "tr", + "vi", + "zh-CN", + "zh-TW", +] + +it.each(locales)("provides matching provider-wait and search keys in %s", async (locale) => { + const settings = settingsLocales[`../../../i18n/locales/${locale}/settings.json`] + const embeddings = embeddingLocales[`../../../../../src/i18n/locales/${locale}/embeddings.json`] + expect(settings).toBeDefined() + expect(embeddings).toBeDefined() + const translation = createInstance() + await translation.init({ + lng: locale, + fallbackLng: false, + resources: { [locale]: { settings, embeddings } }, + interpolation: { escapeValue: false }, + }) + for (const key of ["providerWaiting", "providerWaitingUntil"] as const) { + expect(settings.codeIndex[key]).toBe(embeddings.orchestrator[key]) + } + for (const key of ["providerWaiting", "providerWaitingUntil", "providerPaused", "providerPausedUntil"] as const) { + const message = settings.codeIndex[key] + expect(message).toContain("{{service}}") + expect(message.includes("{{time}}")).toBe(key.endsWith("Until")) + const rendered = translation.t(`settings:codeIndex.${key}`, { service: "test-service", time: "local-time" }) + expect(rendered).toContain("test-service") + expect(rendered).not.toContain("{{") + if (key.endsWith("Until")) expect(rendered).toContain("local-time") + } + expect(embeddings.searchProviderRateLimited).toBeTruthy() + expect(translation.t("embeddings:searchProviderRateLimited")).toBe(embeddings.searchProviderRateLimited) + if (locale !== "en") { + expect(settings.codeIndex.providerWaiting).not.toContain("indexing resumes automatically") + expect(embeddings.searchProviderRateLimited).not.toContain("Search provider rate-limited") + } +}) diff --git a/webview-ui/src/i18n/locales/ca/settings.json b/webview-ui/src/i18n/locales/ca/settings.json index 41bbcc4b0f..0ad5bdcc8c 100644 --- a/webview-ui/src/i18n/locales/ca/settings.json +++ b/webview-ui/src/i18n/locales/ca/settings.json @@ -167,6 +167,10 @@ "description": "Configura les indicacions de suport utilitzades per a accions ràpides com millorar indicacions, explicar codi i solucionar problemes. Aquestes indicacions ajuden Zoo a proporcionar millor assistència per a tasques comunes de desenvolupament." }, "codeIndex": { + "providerWaiting": "Esperant {{service}}; la indexació es reprèn automàticament", + "providerWaitingUntil": "Esperant {{service}}; la indexació es reprèn automàticament a les {{time}}", + "providerPaused": "Esperant {{service}}", + "providerPausedUntil": "Esperant {{service}} fins a les {{time}}", "title": "Indexació de codi", "enableLabel": "Habilitar indexació de codi", "enableDescription": "Habilita la indexació de codi per millorar la cerca i la comprensió del context", @@ -211,6 +215,22 @@ }, "description": "Configureu la configuració d'indexació de la base de codi per habilitar la cerca semàntica del vostre projecte. <0>Més informació", "statusTitle": "Estat", + "indexingFilesRetrying": "Fitxers amb errors d'indexació: {{count}}. S'està reintentant {{path}}: {{reason}}", + "indexingFilesFailed": "Fitxers amb errors d'indexació: {{count}}. El reintent automàtic ha finalitzat per a {{path}}: {{reason}}", + "failedFileDetails": "Detalls dels fitxers amb errors", + "failedFileReason": "Motiu", + "failedFileStates": { + "retrying": "Reintent automàtic pendent", + "exhausted": "Reintent automàtic finalitzat" + }, + "failedFileActivities": { + "read": "Llegir el fitxer", + "parse": "Analitzar el fitxer", + "embed": "Crear embeddings", + "upsert": "Desar els blocs indexats", + "delete": "Eliminar els blocs indexats", + "verify": "Verificar el resultat indexat" + }, "embedderProviderLabel": "Proveïdor d'embeddings", "modelPlaceholder": "Introduïu el nom del model", "selectModel": "Seleccioneu un model", @@ -227,6 +247,7 @@ "saving": "Desant...", "saveSettings": "Desar", "indexingStatuses": { + "stopping": "S'està aturant", "standby": "En espera", "indexing": "Indexant", "indexed": "Indexat", diff --git a/webview-ui/src/i18n/locales/de/settings.json b/webview-ui/src/i18n/locales/de/settings.json index a4a8b8c606..3b5710d12b 100644 --- a/webview-ui/src/i18n/locales/de/settings.json +++ b/webview-ui/src/i18n/locales/de/settings.json @@ -167,9 +167,29 @@ "description": "Konfiguriere Support-Prompts, die für schnelle Aktionen wie das Verbessern von Prompts, das Erklären von Code und das Beheben von Problemen verwendet werden. Diese Prompts helfen Zoo dabei, bessere Unterstützung für häufige Entwicklungsaufgaben zu bieten." }, "codeIndex": { + "providerWaiting": "Warten auf {{service}}; die Indexierung wird automatisch fortgesetzt", + "providerWaitingUntil": "Warten auf {{service}}; die Indexierung wird um {{time}} automatisch fortgesetzt", + "providerPaused": "Warten auf {{service}}", + "providerPausedUntil": "Warten auf {{service}} bis {{time}}", "title": "Codebase-Indexierung", "description": "Konfiguriere Codebase-Indexierungseinstellungen, um semantische Suche in deinem Projekt zu aktivieren. <0>Mehr erfahren", "statusTitle": "Status", + "indexingFilesRetrying": "Dateien mit Indexierungsfehlern: {{count}}. Erneuter Versuch für {{path}}: {{reason}}", + "indexingFilesFailed": "Dateien mit Indexierungsfehlern: {{count}}. Automatische Wiederholung für {{path}} beendet: {{reason}}", + "failedFileDetails": "Details zu fehlgeschlagenen Dateien", + "failedFileReason": "Grund", + "failedFileStates": { + "retrying": "Automatische Wiederholung ausstehend", + "exhausted": "Automatische Wiederholung beendet" + }, + "failedFileActivities": { + "read": "Datei lesen", + "parse": "Datei parsen", + "embed": "Embeddings erstellen", + "upsert": "Indexierte Blöcke speichern", + "delete": "Indexierte Blöcke entfernen", + "verify": "Indexiertes Ergebnis überprüfen" + }, "enableLabel": "Codebase-Indexierung aktivieren", "enableDescription": "Aktiviere die Code-Indizierung für eine verbesserte Suche und ein besseres Kontextverständnis", "embedderProviderLabel": "Embedder-Anbieter", @@ -227,6 +247,7 @@ "saving": "Speichern...", "saveSettings": "Speichern", "indexingStatuses": { + "stopping": "Wird gestoppt", "standby": "Bereitschaft", "indexing": "Indexierung", "indexed": "Indexiert", diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index a5cd604aed..f1c6a6a600 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -243,6 +243,26 @@ "title": "Codebase Indexing", "description": "Configure codebase indexing settings to enable semantic search of your project. <0>Learn more", "statusTitle": "Status", + "indexingFilesRetrying": "Files with indexing failures: {{count}}. Retrying {{path}}: {{reason}}", + "indexingFilesFailed": "Files with indexing failures: {{count}}. Automatic retry ended for {{path}}: {{reason}}", + "providerWaiting": "Waiting for {{service}}; indexing resumes automatically", + "providerWaitingUntil": "Waiting for {{service}}; indexing resumes automatically at {{time}}", + "providerPaused": "Waiting for {{service}}", + "providerPausedUntil": "Waiting for {{service}} until {{time}}", + "failedFileDetails": "Failed file details", + "failedFileReason": "Reason", + "failedFileStates": { + "retrying": "Automatic retry pending", + "exhausted": "Automatic retry ended" + }, + "failedFileActivities": { + "read": "Read file", + "parse": "Parse file", + "embed": "Create embeddings", + "upsert": "Save indexed blocks", + "delete": "Remove indexed blocks", + "verify": "Verify indexed result" + }, "enableLabel": "Enable Codebase Indexing", "enableDescription": "Enable code indexing for improved search and context understanding", "embedderProviderLabel": "Embedder Provider", @@ -310,7 +330,8 @@ "standby": "Standby", "indexing": "Indexing", "indexed": "Indexed", - "error": "Error" + "error": "Error", + "stopping": "Stopping" }, "validation": { "qdrantUrlRequired": "Qdrant URL is required", diff --git a/webview-ui/src/i18n/locales/es/settings.json b/webview-ui/src/i18n/locales/es/settings.json index a100b767f9..aa560ad957 100644 --- a/webview-ui/src/i18n/locales/es/settings.json +++ b/webview-ui/src/i18n/locales/es/settings.json @@ -167,9 +167,29 @@ "description": "Configura indicaciones de soporte que se utilizan para acciones rápidas como mejorar indicaciones, explicar código y solucionar problemas. Estas indicaciones ayudan a Zoo a brindar mejor asistencia para tareas comunes de desarrollo." }, "codeIndex": { + "providerWaiting": "Esperando a {{service}}; la indexación se reanuda automáticamente", + "providerWaitingUntil": "Esperando a {{service}}; la indexación se reanuda automáticamente a las {{time}}", + "providerPaused": "Esperando a {{service}}", + "providerPausedUntil": "Esperando a {{service}} hasta las {{time}}", "title": "Indexación de código", "description": "Configura los ajustes de indexación de código para habilitar búsqueda semántica en tu proyecto. <0>Más información", "statusTitle": "Estado", + "indexingFilesRetrying": "Archivos con errores de indexación: {{count}}. Reintentando {{path}}: {{reason}}", + "indexingFilesFailed": "Archivos con errores de indexación: {{count}}. El reintento automático terminó para {{path}}: {{reason}}", + "failedFileDetails": "Detalles de archivos con errores", + "failedFileReason": "Motivo", + "failedFileStates": { + "retrying": "Reintento automático pendiente", + "exhausted": "Reintento automático terminado" + }, + "failedFileActivities": { + "read": "Leer archivo", + "parse": "Analizar archivo", + "embed": "Crear embeddings", + "upsert": "Guardar bloques indexados", + "delete": "Eliminar bloques indexados", + "verify": "Verificar resultado indexado" + }, "enableLabel": "Habilitar indexación de código", "enableDescription": "Habilita la indexación de código para mejorar la búsqueda y la comprensión del contexto", "embedderProviderLabel": "Proveedor de embedder", @@ -227,6 +247,7 @@ "saving": "Guardando...", "saveSettings": "Guardar", "indexingStatuses": { + "stopping": "Deteniendo", "standby": "En espera", "indexing": "Indexando", "indexed": "Indexado", diff --git a/webview-ui/src/i18n/locales/fr/settings.json b/webview-ui/src/i18n/locales/fr/settings.json index aba006610b..c413f407cf 100644 --- a/webview-ui/src/i18n/locales/fr/settings.json +++ b/webview-ui/src/i18n/locales/fr/settings.json @@ -167,9 +167,29 @@ "description": "Configurez les invites de support utilisées pour les actions rapides comme l'amélioration des invites, l'explication du code et la résolution des problèmes. Ces invites aident Zoo à fournir une meilleure assistance pour les tâches de développement courantes." }, "codeIndex": { + "providerWaiting": "En attente de {{service}} ; l’indexation reprend automatiquement", + "providerWaitingUntil": "En attente de {{service}} ; l’indexation reprend automatiquement à {{time}}", + "providerPaused": "En attente de {{service}}", + "providerPausedUntil": "En attente de {{service}} jusqu’à {{time}}", "title": "Indexation de la base de code", "description": "Configurez les paramètres d'indexation de la base de code pour activer la recherche sémantique dans votre projet. <0>En savoir plus", "statusTitle": "Statut", + "indexingFilesRetrying": "Fichiers en échec d'indexation : {{count}}. Nouvelle tentative pour {{path}} : {{reason}}", + "indexingFilesFailed": "Fichiers en échec d'indexation : {{count}}. Nouvelle tentative automatique terminée pour {{path}} : {{reason}}", + "failedFileDetails": "Détails des fichiers en échec", + "failedFileReason": "Raison", + "failedFileStates": { + "retrying": "Nouvelle tentative automatique en attente", + "exhausted": "Nouvelle tentative automatique terminée" + }, + "failedFileActivities": { + "read": "Lire le fichier", + "parse": "Analyser le fichier", + "embed": "Créer les embeddings", + "upsert": "Enregistrer les blocs indexés", + "delete": "Supprimer les blocs indexés", + "verify": "Vérifier le résultat indexé" + }, "enableLabel": "Activer l'indexation de la base de code", "enableDescription": "Activer l'indexation du code pour une recherche et une compréhension du contexte améliorées", "embedderProviderLabel": "Fournisseur d'embedder", @@ -227,6 +247,7 @@ "saving": "Sauvegarde...", "saveSettings": "Sauvegarder", "indexingStatuses": { + "stopping": "Arrêt en cours", "standby": "En attente", "indexing": "Indexation", "indexed": "Indexé", diff --git a/webview-ui/src/i18n/locales/hi/settings.json b/webview-ui/src/i18n/locales/hi/settings.json index 274d0b4217..27cf6f7f6f 100644 --- a/webview-ui/src/i18n/locales/hi/settings.json +++ b/webview-ui/src/i18n/locales/hi/settings.json @@ -167,6 +167,10 @@ "description": "प्रॉम्प्ट्स को बेहतर बनाना, कोड की व्याख्या करना और समस्याओं को ठीक करना जैसी त्वरित कार्रवाइयों के लिए उपयोग किए जाने वाले सहायक प्रॉम्प्ट्स को कॉन्फ़िगर करें। ये प्रॉम्प्ट्स Zoo को सामान्य विकास कार्यों के लिए बेहतर सहायता प्रदान करने में मदद करते हैं।" }, "codeIndex": { + "providerWaiting": "{{service}} की प्रतीक्षा है; इंडेक्सिंग अपने आप फिर शुरू होगी", + "providerWaitingUntil": "{{service}} की प्रतीक्षा है; इंडेक्सिंग {{time}} पर अपने आप फिर शुरू होगी", + "providerPaused": "{{service}} की प्रतीक्षा है", + "providerPausedUntil": "{{time}} तक {{service}} की प्रतीक्षा है", "title": "कोडबेस इंडेक्सिंग", "enableLabel": "कोडबेस इंडेक्सिंग सक्षम करें", "enableDescription": "बेहतर खोज और संदर्भ समझने के लिए कोड इंडेक्सिंग सक्षम करें", @@ -211,6 +215,22 @@ }, "description": "अपने प्रोजेक्ट की सिमेंटिक खोज को सक्षम करने के लिए कोडबेस इंडेक्सिंग सेटिंग्स कॉन्फ़िगर करें। <0>और जानें", "statusTitle": "स्थिति", + "indexingFilesRetrying": "इंडेक्सिंग में विफल फ़ाइलें: {{count}}। {{path}} के लिए फिर से कोशिश की जा रही है: {{reason}}", + "indexingFilesFailed": "इंडेक्सिंग में विफल फ़ाइलें: {{count}}। {{path}} के लिए स्वचालित पुनः प्रयास समाप्त: {{reason}}", + "failedFileDetails": "विफल फ़ाइलों का विवरण", + "failedFileReason": "कारण", + "failedFileStates": { + "retrying": "स्वचालित पुनः प्रयास लंबित", + "exhausted": "स्वचालित पुनः प्रयास समाप्त" + }, + "failedFileActivities": { + "read": "फ़ाइल पढ़ना", + "parse": "फ़ाइल पार्स करना", + "embed": "एम्बेडिंग बनाना", + "upsert": "इंडेक्स किए गए ब्लॉक सहेजना", + "delete": "इंडेक्स किए गए ब्लॉक हटाना", + "verify": "इंडेक्स किए गए परिणाम की पुष्टि करना" + }, "embedderProviderLabel": "एम्बेडर प्रदाता", "modelPlaceholder": "मॉडल नाम दर्ज करें", "selectModel": "एक मॉडल चुनें", @@ -227,6 +247,7 @@ "saving": "सहेज रहे हैं...", "saveSettings": "सहेजें", "indexingStatuses": { + "stopping": "रोका जा रहा है", "standby": "स्टैंडबाई", "indexing": "इंडेक्सिंग", "indexed": "इंडेक्स किया गया", diff --git a/webview-ui/src/i18n/locales/id/settings.json b/webview-ui/src/i18n/locales/id/settings.json index f88ddf8500..ec2412e12b 100644 --- a/webview-ui/src/i18n/locales/id/settings.json +++ b/webview-ui/src/i18n/locales/id/settings.json @@ -167,6 +167,10 @@ "description": "Konfigurasi support prompt yang digunakan untuk aksi cepat seperti meningkatkan prompt, menjelaskan kode, dan memperbaiki masalah. Prompt ini membantu Zoo memberikan bantuan yang lebih baik untuk tugas pengembangan umum." }, "codeIndex": { + "providerWaiting": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis", + "providerWaitingUntil": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis pada {{time}}", + "providerPaused": "Menunggu {{service}}", + "providerPausedUntil": "Menunggu {{service}} hingga {{time}}", "title": "Pengindeksan Codebase", "enableLabel": "Aktifkan Pengindeksan Codebase", "enableDescription": "Aktifkan pengindeksan kode untuk pencarian dan pemahaman konteks yang lebih baik", @@ -211,6 +215,22 @@ }, "description": "Konfigurasi pengaturan pengindeksan codebase untuk mengaktifkan pencarian semantik proyek kamu. <0>Pelajari lebih lanjut", "statusTitle": "Status", + "indexingFilesRetrying": "File dengan kegagalan pengindeksan: {{count}}. Mencoba ulang {{path}}: {{reason}}", + "indexingFilesFailed": "File dengan kegagalan pengindeksan: {{count}}. Percobaan ulang otomatis berakhir untuk {{path}}: {{reason}}", + "failedFileDetails": "Detail file yang gagal", + "failedFileReason": "Alasan", + "failedFileStates": { + "retrying": "Percobaan ulang otomatis tertunda", + "exhausted": "Percobaan ulang otomatis berakhir" + }, + "failedFileActivities": { + "read": "Baca file", + "parse": "Urai file", + "embed": "Buat embedding", + "upsert": "Simpan blok yang diindeks", + "delete": "Hapus blok yang diindeks", + "verify": "Verifikasi hasil yang diindeks" + }, "embedderProviderLabel": "Provider Embedder", "modelPlaceholder": "Masukkan nama model", "selectModel": "Pilih model", @@ -227,6 +247,7 @@ "saving": "Menyimpan...", "saveSettings": "Simpan", "indexingStatuses": { + "stopping": "Menghentikan", "standby": "Siaga", "indexing": "Mengindeks", "indexed": "Terindeks", diff --git a/webview-ui/src/i18n/locales/it/settings.json b/webview-ui/src/i18n/locales/it/settings.json index ee1b282786..c9ad3d5f89 100644 --- a/webview-ui/src/i18n/locales/it/settings.json +++ b/webview-ui/src/i18n/locales/it/settings.json @@ -167,6 +167,10 @@ "description": "Configura i prompt di supporto utilizzati per azioni rapide come il miglioramento dei prompt, la spiegazione del codice e la risoluzione dei problemi. Questi prompt aiutano Zoo a fornire una migliore assistenza per le attività di sviluppo comuni." }, "codeIndex": { + "providerWaiting": "In attesa di {{service}}; l’indicizzazione riprende automaticamente", + "providerWaitingUntil": "In attesa di {{service}}; l’indicizzazione riprende automaticamente alle {{time}}", + "providerPaused": "In attesa di {{service}}", + "providerPausedUntil": "In attesa di {{service}} fino alle {{time}}", "title": "Indicizzazione del codice", "enableLabel": "Abilita indicizzazione del codice", "enableDescription": "Abilita l'indicizzazione del codice per una ricerca e una comprensione del contesto migliorate", @@ -211,6 +215,22 @@ }, "description": "Configura le impostazioni di indicizzazione del codebase per abilitare la ricerca semantica del tuo progetto. <0>Scopri di più", "statusTitle": "Stato", + "indexingFilesRetrying": "File con errori di indicizzazione: {{count}}. Nuovo tentativo per {{path}}: {{reason}}", + "indexingFilesFailed": "File con errori di indicizzazione: {{count}}. Nuovo tentativo automatico terminato per {{path}}: {{reason}}", + "failedFileDetails": "Dettagli dei file con errori", + "failedFileReason": "Motivo", + "failedFileStates": { + "retrying": "Nuovo tentativo automatico in attesa", + "exhausted": "Nuovo tentativo automatico terminato" + }, + "failedFileActivities": { + "read": "Leggere il file", + "parse": "Analizzare il file", + "embed": "Creare gli embedding", + "upsert": "Salvare i blocchi indicizzati", + "delete": "Rimuovere i blocchi indicizzati", + "verify": "Verificare il risultato indicizzato" + }, "embedderProviderLabel": "Provider Embedder", "modelPlaceholder": "Inserisci il nome del modello", "selectModel": "Seleziona un modello", @@ -227,6 +247,7 @@ "saving": "Salvataggio...", "saveSettings": "Salva", "indexingStatuses": { + "stopping": "Arresto in corso", "standby": "In attesa", "indexing": "Indicizzazione", "indexed": "Indicizzato", diff --git a/webview-ui/src/i18n/locales/ja/settings.json b/webview-ui/src/i18n/locales/ja/settings.json index a277b5485c..20ca24f1d0 100644 --- a/webview-ui/src/i18n/locales/ja/settings.json +++ b/webview-ui/src/i18n/locales/ja/settings.json @@ -167,6 +167,10 @@ "description": "プロンプトの強化、コードの説明、問題の修正などの迅速なアクションに使用されるサポートプロンプトを設定します。これらのプロンプトは、Zooが一般的な開発タスクでより良いサポートを提供するのに役立ちます。" }, "codeIndex": { + "providerWaiting": "{{service}} を待機中です。インデックス作成は自動的に再開されます", + "providerWaitingUntil": "{{service}} を待機中です。インデックス作成は {{time}} に自動的に再開されます", + "providerPaused": "{{service}} を待機中です", + "providerPausedUntil": "{{time}} まで {{service}} を待機します", "title": "コードベースのインデックス作成", "enableLabel": "コードベースのインデックス作成を有効化", "enableDescription": "コードのインデックス作成を有効にして、検索とコンテキストの理解を向上させます", @@ -211,6 +215,22 @@ }, "description": "プロジェクトのセマンティック検索を有効にするためのコードベースインデックス設定を構成します。<0>詳細はこちら", "statusTitle": "ステータス", + "indexingFilesRetrying": "インデックス作成に失敗したファイル数：{{count}}。{{path}} を再試行中：{{reason}}", + "indexingFilesFailed": "インデックス作成に失敗したファイル数：{{count}}。{{path}} の自動再試行は終了しました：{{reason}}", + "failedFileDetails": "失敗したファイルの詳細", + "failedFileReason": "理由", + "failedFileStates": { + "retrying": "自動再試行待ち", + "exhausted": "自動再試行終了" + }, + "failedFileActivities": { + "read": "ファイルの読み込み", + "parse": "ファイルの解析", + "embed": "埋め込みの作成", + "upsert": "インデックス化したブロックの保存", + "delete": "インデックス化したブロックの削除", + "verify": "インデックス化した結果の検証" + }, "embedderProviderLabel": "エンベッダープロバイダー", "modelPlaceholder": "モデル名を入力", "selectModel": "モデルを選択", @@ -227,6 +247,7 @@ "saving": "保存中...", "saveSettings": "保存", "indexingStatuses": { + "stopping": "停止中", "standby": "スタンバイ", "indexing": "インデックス中", "indexed": "インデックス済み", diff --git a/webview-ui/src/i18n/locales/ko/settings.json b/webview-ui/src/i18n/locales/ko/settings.json index 87bd718210..d47fd671a7 100644 --- a/webview-ui/src/i18n/locales/ko/settings.json +++ b/webview-ui/src/i18n/locales/ko/settings.json @@ -167,6 +167,10 @@ "description": "프롬프트 향상, 코드 설명, 문제 해결과 같은 빠른 작업에 사용되는 지원 프롬프트를 구성합니다. 이러한 프롬프트는 Zoo가 일반적인 개발 작업에 대해 더 나은 지원을 제공하는 데 도움이 됩니다." }, "codeIndex": { + "providerWaiting": "{{service}} 대기 중; 인덱싱이 자동으로 재개됩니다", + "providerWaitingUntil": "{{service}} 대기 중; {{time}}에 인덱싱이 자동으로 재개됩니다", + "providerPaused": "{{service}} 대기 중", + "providerPausedUntil": "{{time}}까지 {{service}} 대기 중", "title": "코드베이스 인덱싱", "enableLabel": "코드베이스 인덱싱 활성화", "enableDescription": "향상된 검색 및 컨텍스트 이해를 위해 코드 인덱싱 활성화", @@ -211,6 +215,22 @@ }, "description": "프로젝트의 시맨틱 검색을 활성화하기 위한 코드베이스 인덱싱 설정을 구성합니다. <0>자세히 알아보기", "statusTitle": "상태", + "indexingFilesRetrying": "인덱싱에 실패한 파일: {{count}}. {{path}} 재시도 중: {{reason}}", + "indexingFilesFailed": "인덱싱에 실패한 파일: {{count}}. {{path}} 자동 재시도 종료: {{reason}}", + "failedFileDetails": "실패한 파일 세부 정보", + "failedFileReason": "이유", + "failedFileStates": { + "retrying": "자동 재시도 대기 중", + "exhausted": "자동 재시도 종료" + }, + "failedFileActivities": { + "read": "파일 읽기", + "parse": "파일 파싱", + "embed": "임베딩 생성", + "upsert": "인덱싱된 블록 저장", + "delete": "인덱싱된 블록 제거", + "verify": "인덱싱된 결과 검증" + }, "embedderProviderLabel": "임베더 제공자", "modelPlaceholder": "모델 이름을 입력하세요", "selectModel": "모델 선택", @@ -227,6 +247,7 @@ "saving": "저장 중...", "saveSettings": "저장", "indexingStatuses": { + "stopping": "중지 중", "standby": "대기", "indexing": "인덱싱 중", "indexed": "인덱싱됨", diff --git a/webview-ui/src/i18n/locales/nl/settings.json b/webview-ui/src/i18n/locales/nl/settings.json index 9185cd1ba8..7db47697a1 100644 --- a/webview-ui/src/i18n/locales/nl/settings.json +++ b/webview-ui/src/i18n/locales/nl/settings.json @@ -167,6 +167,10 @@ "description": "Configureer ondersteuningsprompts die worden gebruikt voor snelle acties zoals het verbeteren van prompts, het uitleggen van code en het oplossen van problemen. Deze prompts helpen Zoo om betere ondersteuning te bieden voor veelvoorkomende ontwikkelingstaken." }, "codeIndex": { + "providerWaiting": "Wachten op {{service}}; de indexering wordt automatisch hervat", + "providerWaitingUntil": "Wachten op {{service}}; de indexering wordt automatisch hervat om {{time}}", + "providerPaused": "Wachten op {{service}}", + "providerPausedUntil": "Wachten op {{service}} tot {{time}}", "title": "Codebase indexering", "enableLabel": "Codebase indexering inschakelen", "enableDescription": "Code-indexering inschakelen voor verbeterde zoekresultaten en contextbegrip", @@ -211,6 +215,22 @@ }, "description": "Configureer codebase-indexeringsinstellingen om semantisch zoeken voor je project in te schakelen. <0>Meer informatie", "statusTitle": "Status", + "indexingFilesRetrying": "Bestanden met indexeringsfouten: {{count}}. Nieuwe poging voor {{path}}: {{reason}}", + "indexingFilesFailed": "Bestanden met indexeringsfouten: {{count}}. Automatische nieuwe poging beëindigd voor {{path}}: {{reason}}", + "failedFileDetails": "Details van mislukte bestanden", + "failedFileReason": "Reden", + "failedFileStates": { + "retrying": "Automatische nieuwe poging in afwachting", + "exhausted": "Automatische nieuwe poging beëindigd" + }, + "failedFileActivities": { + "read": "Bestand lezen", + "parse": "Bestand parseren", + "embed": "Embeddings maken", + "upsert": "Geïndexeerde blokken opslaan", + "delete": "Geïndexeerde blokken verwijderen", + "verify": "Geïndexeerd resultaat controleren" + }, "embedderProviderLabel": "Embedder Provider", "modelPlaceholder": "Voer modelnaam in", "selectModel": "Selecteer een model", @@ -227,6 +247,7 @@ "saving": "Opslaan...", "saveSettings": "Opslaan", "indexingStatuses": { + "stopping": "Wordt gestopt", "standby": "Stand-by", "indexing": "Indexeren", "indexed": "Geïndexeerd", diff --git a/webview-ui/src/i18n/locales/pl/settings.json b/webview-ui/src/i18n/locales/pl/settings.json index a3dee217d0..cab9cf9e1e 100644 --- a/webview-ui/src/i18n/locales/pl/settings.json +++ b/webview-ui/src/i18n/locales/pl/settings.json @@ -167,6 +167,10 @@ "description": "Skonfiguruj podpowiedzi wsparcia używane do szybkich działań, takich jak ulepszanie podpowiedzi, wyjaśnianie kodu i rozwiązywanie problemów. Te podpowiedzi pomagają Zoo zapewnić lepsze wsparcie dla typowych zadań programistycznych." }, "codeIndex": { + "providerWaiting": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione", + "providerWaitingUntil": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione o {{time}}", + "providerPaused": "Oczekiwanie na {{service}}", + "providerPausedUntil": "Oczekiwanie na {{service}} do {{time}}", "title": "Indeksowanie kodu", "enableLabel": "Włącz indeksowanie kodu", "enableDescription": "Włącz indeksowanie kodu, aby poprawić wyszukiwanie i zrozumienie kontekstu", @@ -211,6 +215,22 @@ }, "description": "Skonfiguruj ustawienia indeksowania bazy kodu, aby włączyć wyszukiwanie semantyczne w swoim projekcie. <0>Dowiedz się więcej", "statusTitle": "Status", + "indexingFilesRetrying": "Pliki z błędami indeksowania: {{count}}. Ponawianie próby dla {{path}}: {{reason}}", + "indexingFilesFailed": "Pliki z błędami indeksowania: {{count}}. Automatyczne ponawianie zakończone dla {{path}}: {{reason}}", + "failedFileDetails": "Szczegóły plików z błędami", + "failedFileReason": "Powód", + "failedFileStates": { + "retrying": "Oczekiwanie na automatyczne ponowienie", + "exhausted": "Automatyczne ponawianie zakończone" + }, + "failedFileActivities": { + "read": "Odczyt pliku", + "parse": "Parsowanie pliku", + "embed": "Tworzenie embeddingów", + "upsert": "Zapisywanie zaindeksowanych bloków", + "delete": "Usuwanie zaindeksowanych bloków", + "verify": "Weryfikacja zaindeksowanego wyniku" + }, "embedderProviderLabel": "Dostawca Embeddera", "modelPlaceholder": "Wprowadź nazwę modelu", "selectModel": "Wybierz model", @@ -227,6 +247,7 @@ "saving": "Zapisywanie...", "saveSettings": "Zapisz", "indexingStatuses": { + "stopping": "Zatrzymywanie", "standby": "Gotowość", "indexing": "Indeksowanie", "indexed": "Zaindeksowane", diff --git a/webview-ui/src/i18n/locales/pt-BR/settings.json b/webview-ui/src/i18n/locales/pt-BR/settings.json index c5e103eec0..020712a9af 100644 --- a/webview-ui/src/i18n/locales/pt-BR/settings.json +++ b/webview-ui/src/i18n/locales/pt-BR/settings.json @@ -167,6 +167,10 @@ "description": "Configure prompts de suporte usados para ações rápidas como melhorar prompts, explicar código e corrigir problemas. Esses prompts ajudam o Zoo a fornecer melhor assistência para tarefas comuns de desenvolvimento." }, "codeIndex": { + "providerWaiting": "Aguardando {{service}}; a indexação é retomada automaticamente", + "providerWaitingUntil": "Aguardando {{service}}; a indexação é retomada automaticamente às {{time}}", + "providerPaused": "Aguardando {{service}}", + "providerPausedUntil": "Aguardando {{service}} até {{time}}", "title": "Indexação de Código", "enableLabel": "Ativar Indexação de Código", "enableDescription": "Ative a indexação de código para pesquisa e compreensão de contexto aprimoradas", @@ -211,6 +215,22 @@ }, "description": "Configure as configurações de indexação da base de código para habilitar a pesquisa semântica do seu projeto. <0>Saiba mais", "statusTitle": "Status", + "indexingFilesRetrying": "Arquivos com falhas de indexação: {{count}}. Tentando novamente {{path}}: {{reason}}", + "indexingFilesFailed": "Arquivos com falhas de indexação: {{count}}. Nova tentativa automática encerrada para {{path}}: {{reason}}", + "failedFileDetails": "Detalhes dos arquivos com falhas", + "failedFileReason": "Motivo", + "failedFileStates": { + "retrying": "Nova tentativa automática pendente", + "exhausted": "Nova tentativa automática encerrada" + }, + "failedFileActivities": { + "read": "Ler arquivo", + "parse": "Analisar arquivo", + "embed": "Criar embeddings", + "upsert": "Salvar blocos indexados", + "delete": "Remover blocos indexados", + "verify": "Verificar resultado indexado" + }, "embedderProviderLabel": "Provedor de Embedder", "modelPlaceholder": "Insira o nome do modelo", "selectModel": "Selecione um modelo", @@ -227,6 +247,7 @@ "saving": "Salvando...", "saveSettings": "Salvar", "indexingStatuses": { + "stopping": "Parando", "standby": "Em espera", "indexing": "Indexando", "indexed": "Indexado", diff --git a/webview-ui/src/i18n/locales/ru/settings.json b/webview-ui/src/i18n/locales/ru/settings.json index 8c944f3797..86496f45e3 100644 --- a/webview-ui/src/i18n/locales/ru/settings.json +++ b/webview-ui/src/i18n/locales/ru/settings.json @@ -167,6 +167,10 @@ "description": "Настройте промпты поддержки, используемые для быстрых действий, таких как улучшение промптов, объяснение кода и исправление проблем. Эти промпты помогают Zoo обеспечить лучшую поддержку для общих задач разработки." }, "codeIndex": { + "providerWaiting": "Ожидание {{service}}; индексация возобновится автоматически", + "providerWaitingUntil": "Ожидание {{service}}; индексация возобновится автоматически в {{time}}", + "providerPaused": "Ожидание {{service}}", + "providerPausedUntil": "Ожидание {{service}} до {{time}}", "title": "Индексация кодовой базы", "enableLabel": "Включить индексацию кодовой базы", "enableDescription": "Включите индексацию кода для улучшения поиска и понимания контекста", @@ -211,6 +215,22 @@ }, "description": "Настройте параметры индексации кодовой базы для включения семантического поиска в вашем проекте. <0>Узнать больше", "statusTitle": "Статус", + "indexingFilesRetrying": "Файлов с ошибками индексации: {{count}}. Повторная попытка для {{path}}: {{reason}}", + "indexingFilesFailed": "Файлов с ошибками индексации: {{count}}. Автоматические повторные попытки для {{path}} завершены: {{reason}}", + "failedFileDetails": "Подробности файлов с ошибками", + "failedFileReason": "Причина", + "failedFileStates": { + "retrying": "Ожидается автоматическая повторная попытка", + "exhausted": "Автоматические повторные попытки завершены" + }, + "failedFileActivities": { + "read": "Чтение файла", + "parse": "Разбор файла", + "embed": "Создание эмбеддингов", + "upsert": "Сохранение проиндексированных блоков", + "delete": "Удаление проиндексированных блоков", + "verify": "Проверка результата индексации" + }, "embedderProviderLabel": "Провайдер эмбеддера", "modelPlaceholder": "Введите название модели", "selectModel": "Выберите модель", @@ -227,6 +247,7 @@ "saving": "Сохранение...", "saveSettings": "Сохранить", "indexingStatuses": { + "stopping": "Остановка", "standby": "Ожидание", "indexing": "Индексация", "indexed": "Проиндексировано", diff --git a/webview-ui/src/i18n/locales/tr/settings.json b/webview-ui/src/i18n/locales/tr/settings.json index 2872f7f9f3..a234a66d24 100644 --- a/webview-ui/src/i18n/locales/tr/settings.json +++ b/webview-ui/src/i18n/locales/tr/settings.json @@ -167,6 +167,10 @@ "description": "Prompt geliştirme, kod açıklama ve sorun çözme gibi hızlı eylemler için kullanılan destek promptlarını yapılandırın. Bu promptlar, Zoo'nun yaygın geliştirme görevleri için daha iyi destek sağlamasına yardımcı olur." }, "codeIndex": { + "providerWaiting": "{{service}} bekleniyor; dizinleme otomatik olarak devam eder", + "providerWaitingUntil": "{{service}} bekleniyor; dizinleme {{time}} zamanında otomatik olarak devam eder", + "providerPaused": "{{service}} bekleniyor", + "providerPausedUntil": "{{time}} zamanına kadar {{service}} bekleniyor", "title": "Kod Tabanı İndeksleme", "enableLabel": "Kod Tabanı İndekslemeyi Etkinleştir", "enableDescription": "Geliştirilmiş arama ve bağlam anlayışı için kod indekslemeyi etkinleştirin", @@ -211,6 +215,22 @@ }, "description": "Projenizin anlamsal aramasını etkinleştirmek için kod tabanı indeksleme ayarlarını yapılandırın. <0>Daha fazla bilgi", "statusTitle": "Durum", + "indexingFilesRetrying": "İndeksleme hatası olan dosyalar: {{count}}. {{path}} yeniden deneniyor: {{reason}}", + "indexingFilesFailed": "İndeksleme hatası olan dosyalar: {{count}}. {{path}} için otomatik yeniden deneme sona erdi: {{reason}}", + "failedFileDetails": "Başarısız dosya ayrıntıları", + "failedFileReason": "Neden", + "failedFileStates": { + "retrying": "Otomatik yeniden deneme bekliyor", + "exhausted": "Otomatik yeniden deneme sona erdi" + }, + "failedFileActivities": { + "read": "Dosya okuma", + "parse": "Dosya ayrıştırma", + "embed": "Gömme oluşturma", + "upsert": "İndekslenen blokları kaydetme", + "delete": "İndekslenen blokları kaldırma", + "verify": "İndekslenen sonucu doğrulama" + }, "embedderProviderLabel": "Gömücü Sağlayıcı", "modelPlaceholder": "Model adını girin", "selectModel": "Bir model seçin", @@ -227,6 +247,7 @@ "saving": "Kaydediliyor...", "saveSettings": "Kaydet", "indexingStatuses": { + "stopping": "Durduruluyor", "standby": "Bekleme", "indexing": "İndeksleniyor", "indexed": "İndekslendi", diff --git a/webview-ui/src/i18n/locales/vi/settings.json b/webview-ui/src/i18n/locales/vi/settings.json index b8c769f584..5344ab8789 100644 --- a/webview-ui/src/i18n/locales/vi/settings.json +++ b/webview-ui/src/i18n/locales/vi/settings.json @@ -167,6 +167,10 @@ "description": "Cấu hình các lời nhắc hỗ trợ được sử dụng cho các hành động nhanh như cải thiện lời nhắc, giải thích mã và khắc phục sự cố. Những lời nhắc này giúp Zoo cung cấp hỗ trợ tốt hơn cho các tác vụ phát triển phổ biến." }, "codeIndex": { + "providerWaiting": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục", + "providerWaitingUntil": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục lúc {{time}}", + "providerPaused": "Đang chờ {{service}}", + "providerPausedUntil": "Đang chờ {{service}} đến {{time}}", "title": "Lập chỉ mục mã nguồn", "enableLabel": "Bật lập chỉ mục mã nguồn", "enableDescription": "Bật lập chỉ mục mã để cải thiện tìm kiếm và sự hiểu biết về ngữ cảnh", @@ -211,6 +215,22 @@ }, "description": "Cấu hình cài đặt lập chỉ mục mã nguồn để kích hoạt tìm kiếm ngữ nghĩa cho dự án của bạn. <0>Tìm hiểu thêm", "statusTitle": "Trạng thái", + "indexingFilesRetrying": "Số tệp bị lỗi lập chỉ mục: {{count}}. Đang thử lại {{path}}: {{reason}}", + "indexingFilesFailed": "Số tệp bị lỗi lập chỉ mục: {{count}}. Đã kết thúc thử lại tự động cho {{path}}: {{reason}}", + "failedFileDetails": "Chi tiết tệp bị lỗi", + "failedFileReason": "Lý do", + "failedFileStates": { + "retrying": "Đang chờ thử lại tự động", + "exhausted": "Đã kết thúc thử lại tự động" + }, + "failedFileActivities": { + "read": "Đọc tệp", + "parse": "Phân tích tệp", + "embed": "Tạo embedding", + "upsert": "Lưu các khối đã lập chỉ mục", + "delete": "Xóa các khối đã lập chỉ mục", + "verify": "Xác minh kết quả đã lập chỉ mục" + }, "embedderProviderLabel": "Nhà cung cấp Embedder", "modelPlaceholder": "Nhập tên mô hình", "selectModel": "Chọn một mô hình", @@ -227,6 +247,7 @@ "saving": "Đang lưu...", "saveSettings": "Lưu", "indexingStatuses": { + "stopping": "Đang dừng", "standby": "Chờ", "indexing": "Đang lập chỉ mục", "indexed": "Đã lập chỉ mục", diff --git a/webview-ui/src/i18n/locales/zh-CN/settings.json b/webview-ui/src/i18n/locales/zh-CN/settings.json index 01c6cf7948..445ae62dae 100644 --- a/webview-ui/src/i18n/locales/zh-CN/settings.json +++ b/webview-ui/src/i18n/locales/zh-CN/settings.json @@ -167,9 +167,29 @@ "description": "配置用于快速操作的支持提示词，如增强提示词、解释代码和修复问题。这些提示词帮助 Zoo 为常见开发任务提供更好的支持。" }, "codeIndex": { + "providerWaiting": "正在等待 {{service}}；索引将自动恢复", + "providerWaitingUntil": "正在等待 {{service}}；索引将于 {{time}} 自动恢复", + "providerPaused": "正在等待 {{service}}", + "providerPausedUntil": "正在等待 {{service}}，直到 {{time}}", "title": "代码库索引", "description": "配置代码库索引设置以启用项目的语义搜索。<0>了解更多", "statusTitle": "状态", + "indexingFilesRetrying": "索引失败的文件数：{{count}}。正在重试 {{path}}：{{reason}}", + "indexingFilesFailed": "索引失败的文件数：{{count}}。{{path}} 的自动重试已结束：{{reason}}", + "failedFileDetails": "失败文件详情", + "failedFileReason": "原因", + "failedFileStates": { + "retrying": "等待自动重试", + "exhausted": "自动重试已结束" + }, + "failedFileActivities": { + "read": "读取文件", + "parse": "解析文件", + "embed": "创建嵌入向量", + "upsert": "保存已索引的代码块", + "delete": "移除已索引的代码块", + "verify": "验证索引结果" + }, "enableLabel": "启用代码库索引", "enableDescription": "启用代码索引以改进搜索和上下文理解", "embedderProviderLabel": "嵌入器提供商", @@ -227,6 +247,7 @@ "saving": "保存中...", "saveSettings": "保存", "indexingStatuses": { + "stopping": "正在停止", "standby": "待机", "indexing": "索引中", "indexed": "已索引", diff --git a/webview-ui/src/i18n/locales/zh-TW/settings.json b/webview-ui/src/i18n/locales/zh-TW/settings.json index ceda034bf6..20964fc298 100644 --- a/webview-ui/src/i18n/locales/zh-TW/settings.json +++ b/webview-ui/src/i18n/locales/zh-TW/settings.json @@ -194,9 +194,29 @@ "description": "設定用於快速操作的支援提示詞，如增強提示詞、解釋程式碼和修復問題。這些提示詞幫助 Zoo 為常見開發工作提供更好的支援。" }, "codeIndex": { + "providerWaiting": "正在等待 {{service}}；索引將自動恢復", + "providerWaitingUntil": "正在等待 {{service}}；索引將於 {{time}} 自動恢復", + "providerPaused": "正在等待 {{service}}", + "providerPausedUntil": "正在等待 {{service}}，直到 {{time}}", "title": "程式碼庫索引", "description": "設定程式碼庫索引設定以啟用專案的語意搜尋。<0>了解更多", "statusTitle": "狀態", + "indexingFilesRetrying": "索引失敗的檔案數：{{count}}。正在重試 {{path}}：{{reason}}", + "indexingFilesFailed": "索引失敗的檔案數：{{count}}。{{path}} 的自動重試已結束：{{reason}}", + "failedFileDetails": "失敗檔案詳細資訊", + "failedFileReason": "原因", + "failedFileStates": { + "retrying": "等待自動重試", + "exhausted": "自動重試已結束" + }, + "failedFileActivities": { + "read": "讀取檔案", + "parse": "剖析檔案", + "embed": "建立嵌入向量", + "upsert": "儲存已索引的區塊", + "delete": "移除已索引的區塊", + "verify": "驗證索引結果" + }, "enableLabel": "啟用程式碼庫索引", "enableDescription": "啟用程式碼索引以改進搜尋和上下文理解", "embedderProviderLabel": "嵌入器供應商", @@ -260,6 +280,7 @@ "saving": "儲存中...", "saveSettings": "儲存", "indexingStatuses": { + "stopping": "正在停止", "standby": "待命", "indexing": "索引中", "indexed": "已索引", From bf1726023af70f69c3946d98c1954bb09b8a185c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 08:09:15 -0800 Subject: [PATCH 02/42] fix(code-index): address first CI run (format, knip entries, Vitest 4 class mocks, telemetry/nock test setup, built-worker spec in dist lane) --- knip.json | 2 +- src/scripts/verify-coverage-contract.mjs | 1 + .../code-index/__tests__/manager.spec.ts | 8 +++++-- .../__tests__/mock-code-index-log.ts | 4 ++++ .../__tests__/provider-pacer.spec.ts | 1 + .../__tests__/search-service.spec.ts | 7 ++++++ .../__tests__/state-manager.spec.ts | 5 ++-- src/services/code-index/constants/index.ts | 4 ---- .../__tests__/parser-constants.spec.ts | 14 ++++------- .../__tests__/validation-helpers.spec.ts | 23 ++++++++----------- .../code-index/shared/validation-helpers.ts | 7 ------ .../qdrant-client.retry-contract.spec.ts | 2 ++ src/vitest.dist.config.ts | 7 ++++-- src/vitest.services.config.ts | 1 + src/vitest.unit.config.ts | 1 + 15 files changed, 47 insertions(+), 40 deletions(-) diff --git a/knip.json b/knip.json index 1f4956faa6..00536e7b07 100644 --- a/knip.json +++ b/knip.json @@ -16,7 +16,7 @@ "playwright-ct": false, "workspaces": { "src": { - "entry": ["extension.ts!", "extension/api.ts!", "workers/countTokens.ts!"], + "entry": ["extension.ts!", "extension/api.ts!", "workers/countTokens.ts!", "workers/codeIndexParse.ts!"], "project": [ "**/*.ts!", "!**/__tests__/**!", diff --git a/src/scripts/verify-coverage-contract.mjs b/src/scripts/verify-coverage-contract.mjs index 985e514cb6..91cedf90b6 100644 --- a/src/scripts/verify-coverage-contract.mjs +++ b/src/scripts/verify-coverage-contract.mjs @@ -103,6 +103,7 @@ collectTests(root) const laneForTest = (file) => { if (file === "__tests__/dist_assets.spec.ts") return "dist" + if (file === "services/code-index/__tests__/parse-worker-client.real.spec.ts") return "dist" // Reads root-level config; runs outside Turbo in a dedicated CI step. if (file === "services/__tests__/pr-review-state-workflow.test.ts") return "workflow" if (file.startsWith("services/tree-sitter/")) return "tree-sitter" diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index f400fd284f..1c39135048 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -47,7 +47,9 @@ vi.mock("vscode", () => ({ })) vi.mock("../../../i18n", () => ({ t: (key: string) => key })) vi.mock("../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: vi.fn().mockImplementation(() => ({ initialize: vi.fn().mockResolvedValue(undefined) })), + RooIgnoreController: class { + initialize = vi.fn().mockResolvedValue(undefined) + }, })) vi.mock("fs/promises", () => ({ default: { readFile: vi.fn().mockResolvedValue("") } })) vi.mock("ignore", () => ({ default: () => ({ add: vi.fn() }) })) @@ -152,7 +154,9 @@ function semble() { state: "Indexed" as const, } // Semble's public lifecycle surface is sufficient; its CLI fields are not used by the manager. - vi.mocked(SembleProvider).mockReturnValue(provider as unknown as SembleProvider) + vi.mocked(SembleProvider).mockImplementation(function () { + return provider as unknown as SembleProvider + }) return provider } diff --git a/src/services/code-index/__tests__/mock-code-index-log.ts b/src/services/code-index/__tests__/mock-code-index-log.ts index 9ee6c3e07d..df797502fb 100644 --- a/src/services/code-index/__tests__/mock-code-index-log.ts +++ b/src/services/code-index/__tests__/mock-code-index-log.ts @@ -1,3 +1,7 @@ vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, })) +// Embedder failure paths report telemetry; an uninitialized singleton would turn a 429 into a permanent error. +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { instance: { captureEvent: vi.fn() } }, +})) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 573519d414..c5c50a276c 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -18,6 +18,7 @@ const limited = (retryAfterMs?: number) => ({ status: 429, retryAfterMs }) const flush = () => vi.advanceTimersByTimeAsync(0) beforeEach(() => { + vi.mocked(codeIndexLog.info).mockClear() vi.useFakeTimers() vi.setSystemTime(0) vi.spyOn(Math, "random").mockReturnValue(0) diff --git a/src/services/code-index/__tests__/search-service.spec.ts b/src/services/code-index/__tests__/search-service.spec.ts index 017cd2d8fe..345e3f591f 100644 --- a/src/services/code-index/__tests__/search-service.spec.ts +++ b/src/services/code-index/__tests__/search-service.spec.ts @@ -8,6 +8,13 @@ import type { IEmbedder } from "../interfaces/embedder" import type { IVectorStore } from "../interfaces/vector-store" import { EmbeddingError } from "../shared/indexing-errors" +vi.mock("vscode", () => ({ + EventEmitter: class { + event = vi.fn() + fire(): void {} + dispose(): void {} + }, +})) vi.mock("../../../i18n", () => ({ t: (key: string) => key })) vi.mock("../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), info: vi.fn() } })) diff --git a/src/services/code-index/__tests__/state-manager.spec.ts b/src/services/code-index/__tests__/state-manager.spec.ts index eaa519480b..f9605e32c9 100644 --- a/src/services/code-index/__tests__/state-manager.spec.ts +++ b/src/services/code-index/__tests__/state-manager.spec.ts @@ -6,8 +6,9 @@ vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) vi.mock("vscode", () => ({ EventEmitter: class { event = vi.fn() - fire = vi.fn() - dispose = vi.fn() + // Prototype methods give each vi.spyOn a fresh spy instead of an instance mock with earlier calls. + fire(): void {} + dispose(): void {} }, })) diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 0619fd6bfa..d5fec99e54 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -2,9 +2,6 @@ import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types" /**Parser */ export const MAX_BLOCK_CHARS = 1000 -export const MIN_BLOCK_CHARS = 50 -export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split -export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars export const PARSE_TIMEOUT_MS = 10_000 export const INDEX_STOP_LIMIT_MS = 2_000 @@ -25,7 +22,6 @@ export const PACER_INCREASE_AFTER = 20 export const PACER_MAX_BACKOFF_MS = 300_000 export const PACER_RESUME_SPACING_MS = 250 export const PARSING_CONCURRENCY = 10 -export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting /**OpenAI Embedder */ export const MAX_BATCH_TOKENS = 100000 diff --git a/src/services/code-index/processors/__tests__/parser-constants.spec.ts b/src/services/code-index/processors/__tests__/parser-constants.spec.ts index 07cdf42e7b..fceec3947b 100644 --- a/src/services/code-index/processors/__tests__/parser-constants.spec.ts +++ b/src/services/code-index/processors/__tests__/parser-constants.spec.ts @@ -1,20 +1,16 @@ import { build } from "esbuild" import * as path from "path" -import { readFile } from "fs/promises" import * as parserConstants from "../parser-constants" +import { MAX_BLOCK_CHARS } from "../../constants" -it("preserves the four foundation parser constants exactly", async () => { - const source = await readFile(path.resolve("services/code-index/constants/index.ts"), "utf8") - const expected = { +it("preserves the four foundation parser constants exactly", () => { + expect({ ...parserConstants }).toEqual({ MAX_BLOCK_CHARS: 1000, MIN_BLOCK_CHARS: 50, MIN_CHUNK_REMAINDER_CHARS: 200, MAX_CHARS_TOLERANCE_FACTOR: 1.15, - } - expect(parserConstants).toEqual(expected) - for (const [name, value] of Object.entries(expected)) { - expect(source).toMatch(new RegExp(`export const ${name} = ${String(value).replace(".", "\\.")}(?:\\s|$)`)) - } + }) + expect(parserConstants.MAX_BLOCK_CHARS).toBe(MAX_BLOCK_CHARS) }) it("bundles the actual parser worker without host or types/provider barrel inputs", async () => { diff --git a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts index 115c324e44..5263d1299c 100644 --- a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts +++ b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts @@ -11,19 +11,16 @@ describe("formatEmbeddingError", () => { [401, "permanent", "embeddings:authenticationFailed"], [429, "rate-limit", "embeddings:failedWithStatus"], [503, "server", "embeddings:failedWithStatus"], - ] as const)( - "keeps classification and delay for HTTP %s as a single attempt", - (status, classification, key) => { - const cause = Object.assign(new Error("failure"), { status, headers: new Headers({ "retry-after": "2" }) }) - const error = formatEmbeddingError(cause) - expect(error).toBeInstanceOf(EmbeddingError) - expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: 2000, cause }) - expect(error.message).toBe(key) - if (status !== 401) { - expect(t).toHaveBeenLastCalledWith(key, { attempts: 1, statusCode: status, errorMessage: "failure" }) - } - }, - ) + ] as const)("keeps classification and delay for HTTP %s as a single attempt", (status, classification, key) => { + const cause = Object.assign(new Error("failure"), { status, headers: new Headers({ "retry-after": "2" }) }) + const error = formatEmbeddingError(cause) + expect(error).toBeInstanceOf(EmbeddingError) + expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: 2000, cause }) + expect(error.message).toBe(key) + if (status !== 401) { + expect(t).toHaveBeenLastCalledWith(key, { attempts: 1, statusCode: status, errorMessage: "failure" }) + } + }) it("preserves connection errors with no headers", () => { const cause = new TypeError("fetch failed", { cause: new Error("network") }) diff --git a/src/services/code-index/shared/validation-helpers.ts b/src/services/code-index/shared/validation-helpers.ts index d93cb0141e..392c56940e 100644 --- a/src/services/code-index/shared/validation-helpers.ts +++ b/src/services/code-index/shared/validation-helpers.ts @@ -55,13 +55,6 @@ export interface HttpError extends Error { } } -export interface ValidationError { - status?: number - message?: string - name?: string - code?: string -} - /** * Maps HTTP status codes to appropriate error messages */ diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts index 2c50f297eb..8c9a63de15 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts @@ -1,6 +1,7 @@ import { createServer, type Server } from "node:http" import { createRequire } from "node:module" import path from "node:path" +import nock from "nock" import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" import { QdrantVectorStore } from "../qdrant-client" import { VectorStoreError } from "../../shared/indexing-errors" @@ -22,6 +23,7 @@ const workspacePath = path.resolve("qdrant-contract-workspace") beforeEach(async () => { vi.clearAllMocks() + nock.enableNetConnect("127.0.0.1") responseStatus = 200 retryAfter = undefined requests = [] diff --git a/src/vitest.dist.config.ts b/src/vitest.dist.config.ts index 0eae23958f..b47de6c460 100644 --- a/src/vitest.dist.config.ts +++ b/src/vitest.dist.config.ts @@ -6,8 +6,11 @@ export default mergeConfig( baseConfig, defineConfig({ test: { - // This smoke test validates emitted artifacts; source coverage remains in the unit lane. - include: ["__tests__/dist_assets.spec.ts"], + // These smoke tests validate emitted artifacts; source coverage remains in the unit lane. + include: [ + "__tests__/dist_assets.spec.ts", + "services/code-index/__tests__/parse-worker-client.real.spec.ts", + ], }, }), ) diff --git a/src/vitest.services.config.ts b/src/vitest.services.config.ts index eb3d53d0b0..c9ff735baa 100644 --- a/src/vitest.services.config.ts +++ b/src/vitest.services.config.ts @@ -9,6 +9,7 @@ export default mergeConfig( include: ["services/**/*.{test,spec}.{ts,tsx}"], exclude: [ "services/tree-sitter/**/*.{test,spec}.{ts,tsx}", + "services/code-index/__tests__/parse-worker-client.real.spec.ts", "services/__tests__/pr-review-state-workflow.test.ts", "services/__tests__/stale-pr-clock-workflow.test.ts", ], diff --git a/src/vitest.unit.config.ts b/src/vitest.unit.config.ts index f3a20e86b4..ed59587db8 100644 --- a/src/vitest.unit.config.ts +++ b/src/vitest.unit.config.ts @@ -9,6 +9,7 @@ export default mergeConfig( exclude: [ ...configDefaults.exclude, "__tests__/dist_assets.spec.ts", + "services/code-index/__tests__/parse-worker-client.real.spec.ts", "services/__tests__/pr-review-state-workflow.test.ts", "services/__tests__/stale-pr-clock-workflow.test.ts", ], From 947cea78a1011f17a2fc4f7caf84f820d3ded7e1 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 08:29:14 -0800 Subject: [PATCH 03/42] fix(code-index): address second CI run (orchestrator class mocks, watcher log double, lint, unused processors barrel, Qdrant spec doubles for required indexes and codeIndexLog) --- .../code-index/__tests__/manager.spec.ts | 10 +++-- src/services/code-index/code-index-log.ts | 2 +- .../processors/__tests__/file-watcher.spec.ts | 1 + src/services/code-index/processors/index.ts | 3 -- .../__tests__/provider-wait-message.spec.ts | 3 +- .../qdrant-client.retry-contract.spec.ts | 6 ++- .../__tests__/qdrant-client.spec.ts | 45 ++++++++++++++----- 7 files changed, 48 insertions(+), 22 deletions(-) delete mode 100644 src/services/code-index/processors/index.ts diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 1c39135048..225d0ca198 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -181,7 +181,9 @@ describe("CodeIndexManager lifecycle", () => { parser: manager["parser"]!, })) vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockResolvedValue({ valid: true }) - vi.mocked(CodeIndexOrchestrator).mockImplementation(() => orchestratorDouble()) + vi.mocked(CodeIndexOrchestrator).mockImplementation(function () { + return orchestratorDouble() + }) semble() }) @@ -1018,9 +1020,9 @@ describe("CodeIndexManager lifecycle", () => { const { EmbeddingError } = await import("../shared/indexing-errors") const { CodeIndexOrchestrator: ActualOrchestrator } = await vi.importActual("../orchestrator") - vi.mocked(CodeIndexOrchestrator).mockImplementation( - (...argumentsUsed) => new ActualOrchestrator(...argumentsUsed), - ) + vi.mocked(CodeIndexOrchestrator).mockImplementation(function (...argumentsUsed) { + return new ActualOrchestrator(...argumentsUsed) + }) const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) pacer.reportRateLimit( new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 1000 }), diff --git a/src/services/code-index/code-index-log.ts b/src/services/code-index/code-index-log.ts index de88f3a3c3..0a66d8ea54 100644 --- a/src/services/code-index/code-index-log.ts +++ b/src/services/code-index/code-index-log.ts @@ -35,7 +35,7 @@ function configure(configuration: CodeIndexLogConfiguration): void { .sort((left, right) => right.length - left.length) workspacePattern = roots.length ? new RegExp( - `(?:${roots.map((root) => root.split(/[\\/]/).map(escapePattern).join("[\\\\/]")).join("|")})(?:[\\\\/]|(?=$|[\\s\"',:)]))`, + `(?:${roots.map((root) => root.split(/[\\/]/).map(escapePattern).join("[\\\\/]")).join("|")})(?:[\\\\/]|(?=$|[\\s"',:)]))`, "gi", ) : undefined diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 2bc050354e..500089bb1b 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -48,6 +48,7 @@ vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ })) vi.mock("../../code-index-log", () => ({ codeIndexLog: { + info: vi.fn(), redact: (message: string) => message, createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), }, diff --git a/src/services/code-index/processors/index.ts b/src/services/code-index/processors/index.ts deleted file mode 100644 index c244d9b875..0000000000 --- a/src/services/code-index/processors/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./parser" -export * from "./scanner" -export * from "./file-watcher" diff --git a/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts b/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts index c32994861d..26b7d2a9bc 100644 --- a/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts +++ b/src/services/code-index/shared/__tests__/provider-wait-message.spec.ts @@ -1,10 +1,11 @@ import { formatProviderWaitMessage } from "../provider-wait-message" import { t } from "../../../../i18n" +import { clearAllMocks } from "../../../../test-utils/reset" vi.mock("../../../../i18n", () => ({ t: vi.fn((key: string) => key) })) describe("formatProviderWaitMessage", () => { - beforeEach(() => vi.clearAllMocks()) + beforeEach(() => clearAllMocks()) it.each([undefined, 123_000])("formats a provider wait with resume time %s", (resumeAt) => { const key = diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts index 8c9a63de15..af2bec52de 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts @@ -7,6 +7,7 @@ import { QdrantVectorStore } from "../qdrant-client" import { VectorStoreError } from "../../shared/indexing-errors" import { retryWithPolicy, type RetryFailure } from "../../shared/retry-policy" import { waitForRetry } from "../../embedders/cancellation" +import { clearAllMocks } from "../../../../test-utils/reset" vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) vi.mock("../../code-index-log", () => ({ @@ -22,7 +23,7 @@ let requests: Array<{ method: string | undefined; url: string; body: unknown }> const workspacePath = path.resolve("qdrant-contract-workspace") beforeEach(async () => { - vi.clearAllMocks() + clearAllMocks() nock.enableNetConnect("127.0.0.1") responseStatus = 200 retryAfter = undefined @@ -110,7 +111,8 @@ it.each([ it.each(["upsert", "delete", "verify"] as const)("%s is a single attempt without the policy", async (activity) => { responseStatus = 503 await expect(operation(activity)).rejects.toMatchObject({ classification: "server", statusCode: 503 }) - expect(requests).toHaveLength(activity === "verify" ? 2 : 1) + // Verify issues two concurrent counts; the rejection can arrive before the second reaches the server. + await vi.waitFor(() => expect(requests).toHaveLength(activity === "verify" ? 2 : 1)) expect(waitForRetry).not.toHaveBeenCalled() }) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index feb3cebafb..e397425188 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -11,6 +11,15 @@ import { clearAllMocks } from "../../../../test-utils/reset" vitest.mock("@qdrant/js-client-rest") vitest.mock("crypto") vitest.mock("../../../../utils/path") +// The store logs through codeIndexLog; forward to console so the existing console spies observe it. +vitest.mock("../../code-index-log", () => ({ + codeIndexLog: { + info: (...values: unknown[]) => console.log(...values), + warn: (...values: unknown[]) => console.warn(...values), + error: (...values: unknown[]) => console.error(...values), + debug: () => {}, + }, +})) vitest.mock("../../../../i18n", () => ({ t: (key: string, params?: any) => { // Mock translation function that includes parameters for testing @@ -550,14 +559,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) }) it("should not create a new collection if one exists with matching vectorSize and return false", async () => { // Mock getCollection to return existing collection info with matching vector size @@ -584,14 +595,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) }) it("should recreate collection if it exists but vectorSize mismatches and return true", async () => { const differentVectorSize = 768 @@ -641,14 +654,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() // Restore console.warn }) it("should log warning for non-404 errors but still create collection", async () => { @@ -662,7 +677,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled() - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) expect(console.warn).toHaveBeenCalledWith( expect.stringContaining(`Warning during getCollectionInfo for "${expectedCollectionName}"`), genericError.message, @@ -698,9 +713,14 @@ describe("QdrantVectorStore", () => { }) mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) - // Mock payload index creation to fail + // Required verification indexes succeed; only the optional indexes fail. const indexError = new Error("Index creation failed") - mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(indexError) + mockQdrantClientInstance.createPayloadIndex.mockImplementation( + async (_collection: string, options: { field_name: string }) => { + if (options.field_name === "filePath" || options.field_name === "generation") return {} + throw indexError + }, + ) vitest.spyOn(console, "warn").mockImplementation(function () {}) // Suppress console.warn const result = await vectorStore.initialize() @@ -709,8 +729,8 @@ describe("QdrantVectorStore", () => { expect(result).toBe(true) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - // Verify all payload index creations were attempted (6: type + 5 pathSegments) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + // Verify all payload index creations were attempted (2 required + type + 5 pathSegments) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) // Verify warnings were logged for each failed index (now 6) expect(console.warn).toHaveBeenCalledTimes(6) @@ -847,7 +867,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(2) expect(mockQdrantClientInstance.deleteCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() }) @@ -944,7 +964,7 @@ describe("QdrantVectorStore", () => { on_disk: true, }, }) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() }) @@ -1236,7 +1256,10 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.upsert.mockRejectedValue(upsertError) vitest.spyOn(console, "error").mockImplementation(function () {}) - await expect(vectorStore.upsertPoints(mockPoints)).rejects.toThrow(upsertError) + await expect(vectorStore.upsertPoints(mockPoints)).rejects.toMatchObject({ + message: "Upsert failed", + cause: upsertError, + }) expect(mockQdrantClientInstance.upsert).toHaveBeenCalledTimes(1) expect(console.error).toHaveBeenCalledWith("Failed to upsert points:", upsertError) From a6186aed7b0a5cd151a0c75b54865a29e7355d3c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 08:45:20 -0800 Subject: [PATCH 04/42] chore(lint): prune code-index ESLint suppressions made stale by the port --- src/eslint-suppressions.json | 44 ++---------------------------------- 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 7307fc29af..e1ac635c82 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1024,16 +1024,6 @@ "count": 1 } }, - "services/code-index/__tests__/manager.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 84 - } - }, - "services/code-index/__tests__/orchestrator.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 23 - } - }, "services/code-index/__tests__/service-factory.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 43 @@ -1074,36 +1064,11 @@ "count": 2 } }, - "services/code-index/orchestrator.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 2 - } - }, - "services/code-index/processors/__tests__/file-watcher.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 25 - } - }, "services/code-index/processors/__tests__/parser.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 22 } }, - "services/code-index/processors/__tests__/scanner.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 26 - } - }, - "services/code-index/processors/file-watcher.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 2 - } - }, - "services/code-index/processors/scanner.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 2 - } - }, "services/code-index/semble/__tests__/provider.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 12 @@ -1129,14 +1094,9 @@ "count": 3 } }, - "services/code-index/shared/__tests__/validation-helpers.spec.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 4 - } - }, "services/code-index/shared/validation-helpers.ts": { "@typescript-eslint/no-explicit-any": { - "count": 5 + "count": 4 } }, "services/code-index/vector-store/__tests__/qdrant-client.spec.ts": { @@ -1146,7 +1106,7 @@ }, "services/code-index/vector-store/qdrant-client.ts": { "@typescript-eslint/no-explicit-any": { - "count": 5 + "count": 4 } }, "services/glob/__tests__/gitignore-integration.spec.ts": { From b8fd8e1dce3e5d0f9dc327a2c3cee176ab7b8d4f Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 09:00:14 -0800 Subject: [PATCH 05/42] chore(lint): prune stale CodebaseSearchTool suppression removed by the port --- src/eslint-suppressions.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index e1ac635c82..e649abe268 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -769,11 +769,6 @@ "count": 1 } }, - "core/tools/CodebaseSearchTool.ts": { - "@typescript-eslint/no-explicit-any": { - "count": 1 - } - }, "core/tools/GenerateImageTool.ts": { "@typescript-eslint/no-explicit-any": { "count": 1 From c13334ab689c570d6bc9db471458ca22b85c8757 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:01:57 -0800 Subject: [PATCH 06/42] fix(code-index): resolve type errors in orchestrator and test doubles --- src/__tests__/extension.spec.ts | 6 ++++++ src/services/code-index/__tests__/manager.spec.ts | 2 +- .../code-index/__tests__/orchestrator.spec.ts | 4 ++-- .../workspace-indexing-enablement-manager.spec.ts | 2 ++ src/services/code-index/orchestrator.ts | 4 +--- .../code-index/processors/__tests__/scanner.spec.ts | 11 +++++------ 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/__tests__/extension.spec.ts b/src/__tests__/extension.spec.ts index abcd3c28a0..28dba2649e 100644 --- a/src/__tests__/extension.spec.ts +++ b/src/__tests__/extension.spec.ts @@ -388,6 +388,9 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() @@ -434,6 +437,9 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 225d0ca198..4c668b8029 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -921,7 +921,7 @@ describe("CodeIndexManager lifecycle", () => { it("keeps folder URI enablement distinct and honors auto-enable false", async () => { const other = new CodeIndexManager( "/workspace", - { ...vscode.Uri.file("/workspace"), toString: () => "remote:///workspace" }, + Object.assign(vscode.Uri.file("/workspace"), { toString: () => "remote:///workspace" }), context, new CodeIndexStateManager(), ) diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 70e2575d80..117838f198 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -87,7 +87,7 @@ describe("CodeIndexOrchestrator lifecycle", () => { let gate: IndexWriteGate let allocator: ReturnType["allocator"] let current: boolean - let fatal: ReturnType + let fatal: ReturnType void>> let orchestrator: CodeIndexOrchestrator let failures: FileFailureRegistry let pacers: ProviderPacerRegistry @@ -102,7 +102,7 @@ describe("CodeIndexOrchestrator lifecycle", () => { vi.spyOn(state, "reportFileQueueProgress") ;({ cache, store, scanner, watcher, worker, gate, allocator } = fixtures()) current = true - fatal = vi.fn() + fatal = vi.fn<(error: ParserFatalError, message: string) => void>() failures = new FileFailureRegistry() pacers = new ProviderPacerRegistry() orchestrator = createOrchestrator() diff --git a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts index 6008ba3d81..e3973d4de0 100644 --- a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts +++ b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts @@ -22,6 +22,8 @@ describe("WorkspaceIndexingEnablementManager", () => { workspaceEnabled: true, autoEnableDefault: false, failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], }), } return { diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 10e5cbd148..6ddabb23a5 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -310,9 +310,7 @@ export class CodeIndexOrchestrator { if (!this.isCurrentAttempt()) return Promise.resolve() const paths = signal ? this.writeGate.uncertainFiles(signal) : [] return Promise.all(paths.map((filePath) => this.cacheManager.invalidateFile(filePath))) - .then(() => { - if (this.isCurrentAttempt()) return this.cacheManager.flush() - }) + .then(() => (this.isCurrentAttempt() ? this.cacheManager.flush() : undefined)) .catch((error: unknown) => { if (this.isCurrentAttempt()) codeIndexLog.error("Stopped index cache persistence failed", { error }) }) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index bfb2f3ac86..560b5c0568 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -19,6 +19,9 @@ import { PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, MAX_FILE_SIZE_BYTES import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" vi.mock("fs/promises", () => ({ stat: vi.fn() })) + +// Stats has a private constructor in @types/node; the scanner reads only `size`. +const statsOfSize = (size: number): Stats => Object.assign(Object.create(Stats.prototype) as Stats, { size }) vi.mock("vscode", () => ({ workspace: { fs: { readFile: vi.fn() } }, Uri: { file: (filePath: string) => ({ fsPath: filePath }) }, @@ -185,9 +188,7 @@ function setup() { beforeEach(() => { vi.useFakeTimers() clearAllMocks() - const stats = new Stats() - stats.size = 100 - vi.mocked(stat).mockReset().mockResolvedValue(stats) + vi.mocked(stat).mockReset().mockResolvedValue(statsOfSize(100)) vi.mocked(listFiles) .mockReset() .mockResolvedValue([[fileA], false]) @@ -474,9 +475,7 @@ describe("DirectoryScanner", () => { if (selection === "deleted") vi.mocked(listFiles).mockResolvedValue([[], false]) if (selection === "excluded") ignored.add("a.ts") if (selection === "oversized") { - const stats = new Stats() - stats.size = MAX_FILE_SIZE_BYTES + 1 - vi.mocked(stat).mockResolvedValue(stats) + vi.mocked(stat).mockResolvedValue(statsOfSize(MAX_FILE_SIZE_BYTES + 1)) } expect((await scanner.scanDirectory(workspacePath)).incompleteFiles).toEqual([]) expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(fileA, expect.any(Number)) From d510201b5595fa31deecc60f190f99b6e1d16cda Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:01:58 -0800 Subject: [PATCH 07/42] build: drop unused p-limit; patch @qdrant/js-client-rest Retry-After parsing (D8) --- .gitattributes | 3 ++ package.json | 3 ++ patches/@qdrant__js-client-rest@1.20.0.patch | 48 ++++++++++++++++++++ pnpm-lock.yaml | 26 +++-------- src/package.json | 1 - 5 files changed, 61 insertions(+), 20 deletions(-) create mode 100644 patches/@qdrant__js-client-rest@1.20.0.patch diff --git a/.gitattributes b/.gitattributes index e9e36432cd..550c08fe69 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,9 @@ demo.gif filter=lfs diff=lfs merge=lfs -text assets/docs/demo.gif filter=lfs diff=lfs merge=lfs -text src/assets/docs/demo.gif filter=lfs diff=lfs merge=lfs -text +# pnpm hashes patch bytes; keep LF on every checkout. +patches/*.patch text eol=lf + # Test snapshot files - mark as linguist-generated to exclude from GitHub language statistics *.snap linguist-generated=true diff --git a/package.json b/package.json index 04922487f6..840700f103 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,9 @@ "zod": "4.6.7", "@lmstudio/sdk>zod": "3.25.76", "katex": "0.19.1" + }, + "patchedDependencies": { + "@qdrant/js-client-rest@1.20.0": "patches/@qdrant__js-client-rest@1.20.0.patch" } } } diff --git a/patches/@qdrant__js-client-rest@1.20.0.patch b/patches/@qdrant__js-client-rest@1.20.0.patch new file mode 100644 index 0000000000..42d905ae87 --- /dev/null +++ b/patches/@qdrant__js-client-rest@1.20.0.patch @@ -0,0 +1,48 @@ +diff --git a/dist/browser/index.cjs b/dist/browser/index.cjs +--- a/dist/browser/index.cjs ++++ b/dist/browser/index.cjs +@@ -991,7 +991,7 @@ + } + catch (error) { + if (error instanceof ApiError && error.status === 429) { +- const retryAfterHeader = error.headers.get('retry-after')?.[0]; ++ const retryAfterHeader = error.headers.get('retry-after'); + if (retryAfterHeader) { + throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); + } +diff --git a/dist/browser/index.js b/dist/browser/index.js +--- a/dist/browser/index.js ++++ b/dist/browser/index.js +@@ -989,7 +989,7 @@ + } + catch (error) { + if (error instanceof ApiError && error.status === 429) { +- const retryAfterHeader = error.headers.get('retry-after')?.[0]; ++ const retryAfterHeader = error.headers.get('retry-after'); + if (retryAfterHeader) { + throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); + } +diff --git a/dist/cjs/api-client.js b/dist/cjs/api-client.js +--- a/dist/cjs/api-client.js ++++ b/dist/cjs/api-client.js +@@ -51,7 +51,7 @@ + } + catch (error) { + if (error instanceof openapi_typescript_fetch_1.ApiError && error.status === 429) { +- const retryAfterHeader = error.headers.get('retry-after')?.[0]; ++ const retryAfterHeader = error.headers.get('retry-after'); + if (retryAfterHeader) { + throw new errors_js_1.QdrantClientResourceExhaustedError(error.message, retryAfterHeader); + } +diff --git a/dist/esm/api-client.js b/dist/esm/api-client.js +--- a/dist/esm/api-client.js ++++ b/dist/esm/api-client.js +@@ -47,7 +47,7 @@ + } + catch (error) { + if (error instanceof ApiError && error.status === 429) { +- const retryAfterHeader = error.headers.get('retry-after')?.[0]; ++ const retryAfterHeader = error.headers.get('retry-after'); + if (retryAfterHeader) { + throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c792efec3c..5f3e83f360 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,6 +46,11 @@ overrides: '@lmstudio/sdk>zod': 3.25.76 katex: 0.19.1 +patchedDependencies: + '@qdrant/js-client-rest@1.20.0': + hash: a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd + path: patches/@qdrant__js-client-rest@1.20.0.patch + importers: .: @@ -471,7 +476,7 @@ importers: version: 1.32.2(zod@4.6.7) '@qdrant/js-client-rest': specifier: 1.20.0 - version: 1.20.0(typescript@5.9.3) + version: 1.20.0(patch_hash=a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd)(typescript@5.9.3) '@roo-code/cloud': specifier: workspace:^ version: link:../packages/cloud @@ -577,9 +582,6 @@ importers: os-name: specifier: ^6.0.0 version: 6.1.0 - p-limit: - specifier: ^6.2.0 - version: 6.2.0 p-wait-for: specifier: ^5.0.2 version: 5.0.2 @@ -7118,10 +7120,6 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} - p-limit@6.2.0: - resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} - engines: {node: '>=18'} - p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -9047,10 +9045,6 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-queue@1.2.2: - resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} - engines: {node: '>=12.20'} - yoctocolors@2.2.0: resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} engines: {node: '>=18'} @@ -10697,7 +10691,7 @@ snapshots: '@protobufjs/utf8@1.1.2': {} - '@qdrant/js-client-rest@1.20.0(typescript@5.9.3)': + '@qdrant/js-client-rest@1.20.0(patch_hash=a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd)(typescript@5.9.3)': dependencies: '@qdrant/openapi-typescript-fetch': 1.2.6 typescript: 5.9.3 @@ -16148,10 +16142,6 @@ snapshots: dependencies: yocto-queue: 0.1.0 - p-limit@6.2.0: - dependencies: - yocto-queue: 1.2.2 - p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -18245,8 +18235,6 @@ snapshots: yocto-queue@0.1.0: {} - yocto-queue@1.2.2: {} - yoctocolors@2.2.0: {} yoga-layout@3.2.1: {} diff --git a/src/package.json b/src/package.json index 98578ff92a..37f9dd2f12 100644 --- a/src/package.json +++ b/src/package.json @@ -528,7 +528,6 @@ "ollama": "^0.6.0", "openai": "^5.12.2", "os-name": "^6.0.0", - "p-limit": "^6.2.0", "p-wait-for": "^5.0.2", "partial-json": "^0.1.7", "pdf-parse": "^1.1.1", From 9309d5e8dc53b5483e714473ddde25e0a3bf46c4 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:16:48 -0800 Subject: [PATCH 08/42] chore(code-index): remove production-dead code left by the port (knip --production) --- knip.json | 1 - src/services/code-index/constants/index.ts | 1 - .../__tests__/file-preparation.spec.ts | 327 ------------------ .../__tests__/parser-constants.spec.ts | 2 - .../processors/__tests__/parser.spec.ts | 11 +- .../processors/__tests__/parser.txt.spec.ts | 3 +- .../processors/__tests__/parser.vb.spec.ts | 2 +- .../__tests__/parser.worker-contract.spec.ts | 2 +- .../file-preparation-dependencies.ts | 15 - .../code-index/processors/file-preparation.ts | 100 ------ src/services/code-index/processors/parser.ts | 3 - .../__tests__/get-relative-path.spec.ts | 28 +- .../code-index/shared/get-relative-path.ts | 15 - .../code-index/shared/supported-extensions.ts | 26 +- 14 files changed, 7 insertions(+), 529 deletions(-) delete mode 100644 src/services/code-index/processors/__tests__/file-preparation.spec.ts delete mode 100644 src/services/code-index/processors/file-preparation-dependencies.ts delete mode 100644 src/services/code-index/processors/file-preparation.ts diff --git a/knip.json b/knip.json index 00536e7b07..e0ab62eb9d 100644 --- a/knip.json +++ b/knip.json @@ -22,7 +22,6 @@ "!**/__tests__/**!", "!vitest.setup.ts!", "!utils/vitest-verbosity.ts!", - "!utils/TaskSemaphore.ts!", "!utils/WorkspacePathResolver.ts!", "!test-utils/**!", "!utils/versionToken.ts!", diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index d5fec99e54..6eed825bc5 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -1,7 +1,6 @@ import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types" /**Parser */ -export const MAX_BLOCK_CHARS = 1000 export const PARSE_TIMEOUT_MS = 10_000 export const INDEX_STOP_LIMIT_MS = 2_000 diff --git a/src/services/code-index/processors/__tests__/file-preparation.spec.ts b/src/services/code-index/processors/__tests__/file-preparation.spec.ts deleted file mode 100644 index 22020a9fdd..0000000000 --- a/src/services/code-index/processors/__tests__/file-preparation.spec.ts +++ /dev/null @@ -1,327 +0,0 @@ -import { createHash } from "crypto" -import path from "path" -import { FileType, Uri } from "vscode" -import { v5 as uuidv5 } from "uuid" -import type { CodeBlock, ICodeParser, IEmbedder } from "../../interfaces" -import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../../constants" -import { FilePreparation } from "../file-preparation" -import type { FilePreparationDependencies } from "../file-preparation-dependencies" - -// Keep path spies local to the modules under test, not Node or the test runner. -vi.mock("path", async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, default: { ...actual } } -}) - -describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platform) => { - beforeEach(() => { - vi.spyOn(path, "resolve").mockImplementation(path[platform].resolve) - vi.spyOn(path, "relative").mockImplementation(path[platform].relative) - vi.spyOn(path, "normalize").mockImplementation(path[platform].normalize) - }) - - afterEach(() => vi.restoreAllMocks()) - - const filePath = "/workspace/src/file.ts" - const relativeFilePath = path[platform].join("src", "file.ts") - const normalizedFilePath = path[platform].resolve(filePath) - const content = "test content" - const hash = createHash("sha256").update(content).digest("hex") - const block: CodeBlock = { - file_path: filePath, - identifier: null, - type: "function", - start_line: 2, - end_line: 5, - content, - fileHash: hash, - segmentHash: "segment", - } - - function setup() { - return { - workspacePath: "/workspace", - ignoreController: { - validateAccess: vi - .fn() - .mockReturnValue(true), - }, - ignoreInstance: { ignores: vi.fn<(path: string) => boolean>().mockReturnValue(false) }, - fileSystem: { - stat: vi - .fn() - .mockResolvedValue({ type: FileType.File, ctime: 0, mtime: 0, size: 100 }), - readFile: vi - .fn() - .mockResolvedValue(Buffer.from(content)), - }, - cacheManager: { getHash: vi.fn() }, - parser: { parseFile: vi.fn().mockResolvedValue([{ ...block }]) }, - embedder: { - createEmbeddings: vi - .fn() - .mockResolvedValue({ embeddings: [[0.1, 0.2]] }), - }, - } satisfies FilePreparationDependencies - } - - it.each([".git/config", ".hidden/file.ts", "node_modules/pkg/file.ts", "dist/file.js"])( - "skips excluded directory %s before access or file reads", - async (relativePath) => { - const dependencies = setup() - const path = `/workspace/${relativePath}` - expect(await new FilePreparation(dependencies).prepareFile(path)).toEqual({ - path, - status: "skipped", - reason: "File is in an ignored directory", - }) - expect(dependencies.ignoreController.validateAccess).not.toHaveBeenCalled() - expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() - expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() - expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() - expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() - expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }, - ) - - it.each(["access", "gitignore"])("skips %s exclusions before reading", async (source) => { - const dependencies = setup() - dependencies.ignoreController.validateAccess.mockReturnValue(source !== "access") - dependencies.ignoreInstance.ignores.mockReturnValue(source === "gitignore") - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "skipped", - reason: "File is ignored by .rooignore or .gitignore", - }) - expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledWith(filePath) - if (source === "access") { - expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() - } else { - expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) - } - expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() - expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() - expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() - expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it("skips oversized files without reading them", async () => { - const dependencies = setup() - dependencies.fileSystem.stat.mockResolvedValue({ - type: FileType.File, - ctime: 0, - mtime: 0, - size: MAX_FILE_SIZE_BYTES + 1, - }) - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "skipped", - reason: "File is too large", - }) - expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() - expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() - expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it("skips unchanged content before parsing or embedding", async () => { - const dependencies = setup() - dependencies.cacheManager.getHash.mockReturnValue(hash) - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "skipped", - reason: "File has not changed", - }) - expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) - expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it("returns the hash and empty points when parsing produces no blocks", async () => { - const dependencies = setup() - dependencies.parser.parseFile.mockResolvedValue([]) - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: hash, - pointsToUpsert: [], - }) - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it("still parses and returns the hash without an embedder or gitignore", async () => { - const dependencies = setup() - expect( - await new FilePreparation({ ...dependencies, embedder: undefined, ignoreInstance: undefined }).prepareFile( - filePath, - ), - ).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: hash, - pointsToUpsert: [], - }) - expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) - }) - - it("preserves normalized paths, stable IDs and embedding order at the size limit", async () => { - const dependencies = setup() - dependencies.fileSystem.stat.mockResolvedValue({ - type: FileType.File, - ctime: 0, - mtime: 0, - size: MAX_FILE_SIZE_BYTES, - }) - dependencies.parser.parseFile.mockResolvedValue([ - { ...block, file_path: "src/../src/file.ts" }, - { ...block, start_line: 8, end_line: 10, content: "second" }, - ]) - dependencies.embedder.createEmbeddings.mockResolvedValue({ - embeddings: [ - [0.1, 0.2], - [0.3, 0.4], - ], - }) - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: hash, - pointsToUpsert: [ - { - id: uuidv5(`${normalizedFilePath}:2`, QDRANT_CODE_BLOCK_NAMESPACE), - vector: [0.1, 0.2], - payload: { filePath: relativeFilePath, codeChunk: content, startLine: 2, endLine: 5 }, - }, - { - id: uuidv5(`${normalizedFilePath}:8`, QDRANT_CODE_BLOCK_NAMESPACE), - vector: [0.3, 0.4], - payload: { filePath: relativeFilePath, codeChunk: "second", startLine: 8, endLine: 10 }, - }, - ], - }) - expect(dependencies.fileSystem.stat).toHaveBeenCalledWith(Uri.file(filePath)) - expect(dependencies.fileSystem.readFile).toHaveBeenCalledWith(Uri.file(filePath)) - expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) - expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledWith([content, "second"]) - }) - - it("calls service methods with their original receivers", async () => { - const dependencies = setup() - expect((await new FilePreparation(dependencies).prepareFile(filePath)).status).toBe("processed_for_batching") - expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledOnce() - expect(dependencies.ignoreController.validateAccess.mock.contexts[0]).toBe(dependencies.ignoreController) - expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledOnce() - expect(dependencies.ignoreInstance.ignores.mock.contexts[0]).toBe(dependencies.ignoreInstance) - expect(dependencies.fileSystem.stat).toHaveBeenCalledOnce() - expect(dependencies.fileSystem.stat.mock.contexts[0]).toBe(dependencies.fileSystem) - expect(dependencies.fileSystem.readFile).toHaveBeenCalledOnce() - expect(dependencies.fileSystem.readFile.mock.contexts[0]).toBe(dependencies.fileSystem) - expect(dependencies.cacheManager.getHash).toHaveBeenCalledOnce() - expect(dependencies.cacheManager.getHash.mock.contexts[0]).toBe(dependencies.cacheManager) - expect(dependencies.parser.parseFile).toHaveBeenCalledOnce() - expect(dependencies.parser.parseFile.mock.contexts[0]).toBe(dependencies.parser) - expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(dependencies.embedder.createEmbeddings.mock.contexts[0]).toBe(dependencies.embedder) - }) - - it("does not treat a hidden workspace ancestor as an excluded directory", async () => { - const dependencies = setup() - dependencies.workspacePath = "/.hidden/workspace" - expect((await new FilePreparation(dependencies).prepareFile("/.hidden/workspace/src/file.ts")).status).toBe( - "processed_for_batching", - ) - expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) - }) - - describe.each(["Buffer", "Uint8Array"] as const)("UTF-8 decoding from %s", (representation) => { - it.each([ - { name: "ASCII", bytes: [65, 66], expectedContent: "AB" }, - { - name: "Cyrillic, Spanish, CJK and emoji", - bytes: [...Buffer.from("Привет, español 中文 😀", "utf-8")], - expectedContent: "Привет, español 中文 😀", - }, - { name: "preserved BOM", bytes: [0xef, 0xbb, 0xbf, 65, 66], expectedContent: "\uFEFFAB" }, - { name: "empty content", bytes: [], expectedContent: "" }, - { - name: "invalid and truncated UTF-8 sequences", - bytes: [65, 0xc3, 0x28, 0xff, 0xe2, 0x82], - expectedContent: "A\uFFFD(\uFFFD\uFFFD", - }, - { - name: "sliced view with nonzero byteOffset", - bytes: [0x58, 65, 0xc3, 0xb1, 0x59], - expectedContent: "Añ", - sliced: true, - }, - ])("decodes and hashes $name, then skips the cached content", async ({ bytes, expectedContent, sliced }) => { - const dependencies = setup() - const backing = representation === "Buffer" ? Buffer.from(bytes) : new Uint8Array(bytes) - const fileContent = sliced ? backing.subarray(1, backing.length - 1) : backing - if (sliced) { - expect(fileContent.byteOffset).toBeGreaterThan(0) - expect(fileContent.byteLength).toBeLessThan(backing.byteLength) - } - dependencies.fileSystem.readFile.mockResolvedValue(fileContent) - dependencies.parser.parseFile.mockResolvedValue([]) - // Hash the literal expected text, not the input bytes or the production decoder's output. - const expectedHash = createHash("sha256").update(expectedContent).digest("hex") - const preparation = new FilePreparation(dependencies) - - expect(await preparation.prepareFile(filePath)).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: expectedHash, - pointsToUpsert: [], - }) - expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { - content: expectedContent, - fileHash: expectedHash, - }) - - dependencies.parser.parseFile.mockClear() - dependencies.cacheManager.getHash.mockReturnValue(expectedHash) - expect(await preparation.prepareFile(filePath)).toEqual({ - path: filePath, - status: "skipped", - reason: "File has not changed", - }) - expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) - expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() - }) - }) - - it.each(["access", "ignore", "stat", "read", "cache", "parse", "embed"])( - "returns the original %s error as a local error", - async (stage) => { - const dependencies = setup() - const error = new Error(`${stage} failed`) - const fail = () => { - throw error - } - if (stage === "access") dependencies.ignoreController.validateAccess.mockImplementation(fail) - if (stage === "ignore") dependencies.ignoreInstance.ignores.mockImplementation(fail) - if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(error) - if (stage === "read") dependencies.fileSystem.readFile.mockRejectedValue(error) - if (stage === "cache") dependencies.cacheManager.getHash.mockImplementation(fail) - if (stage === "parse") dependencies.parser.parseFile.mockRejectedValue(error) - if (stage === "embed") dependencies.embedder.createEmbeddings.mockRejectedValue(error) - const result = await new FilePreparation(dependencies).prepareFile(filePath) - expect(result).toEqual({ path: filePath, status: "local_error", error }) - expect(result.error).toBe(error) - }, - ) - - it("does not wrap non-Error rejections", async () => { - const dependencies = setup() - dependencies.fileSystem.readFile.mockRejectedValue("read failed") - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "local_error", - error: "read failed", - }) - }) -}) diff --git a/src/services/code-index/processors/__tests__/parser-constants.spec.ts b/src/services/code-index/processors/__tests__/parser-constants.spec.ts index fceec3947b..884bfd5bb6 100644 --- a/src/services/code-index/processors/__tests__/parser-constants.spec.ts +++ b/src/services/code-index/processors/__tests__/parser-constants.spec.ts @@ -1,7 +1,6 @@ import { build } from "esbuild" import * as path from "path" import * as parserConstants from "../parser-constants" -import { MAX_BLOCK_CHARS } from "../../constants" it("preserves the four foundation parser constants exactly", () => { expect({ ...parserConstants }).toEqual({ @@ -10,7 +9,6 @@ it("preserves the four foundation parser constants exactly", () => { MIN_CHUNK_REMAINDER_CHARS: 200, MAX_CHARS_TOLERANCE_FACTOR: 1.15, }) - expect(parserConstants.MAX_BLOCK_CHARS).toBe(MAX_BLOCK_CHARS) }) it("bundles the actual parser worker without host or types/provider barrel inputs", async () => { diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 656b78cfde..2ed46b50f1 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -1,6 +1,6 @@ // npx vitest services/code-index/processors/__tests__/parser.spec.ts -import { CodeParser, codeParser } from "../parser" +import { CodeParser } from "../parser" import { loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser" import { parseMarkdown } from "../../../tree-sitter/markdownParser" import { readFile } from "fs/promises" @@ -312,15 +312,6 @@ describe("CodeParser", () => { }) }) - describe("singleton instance", () => { - it("should maintain parser state across calls", async () => { - const result1 = await codeParser.parseFile("test.js", { content: "const a = 1" }) - const result2 = await codeParser.parseFile("test.js", { content: "const b = 2" }) - expect(result1).toBeDefined() - expect(result2).toBeDefined() - }) - }) - describe("Markdown Support", () => { beforeEach(() => { clearAllMocks() diff --git a/src/services/code-index/processors/__tests__/parser.txt.spec.ts b/src/services/code-index/processors/__tests__/parser.txt.spec.ts index 0fa9526916..eca53e8351 100644 --- a/src/services/code-index/processors/__tests__/parser.txt.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.txt.spec.ts @@ -1,5 +1,6 @@ import { CodeParser } from "../parser" -import { scannerExtensions, shouldUseFallbackChunking } from "../../shared/supported-extensions" +import { scannerExtensions } from "../../shared/supported-extensions" +import { isFallbackExtension as shouldUseFallbackChunking } from "../../../shared/fallback-extensions" vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ TelemetryService: { diff --git a/src/services/code-index/processors/__tests__/parser.vb.spec.ts b/src/services/code-index/processors/__tests__/parser.vb.spec.ts index 95fd73cc9a..9124a8464e 100644 --- a/src/services/code-index/processors/__tests__/parser.vb.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.vb.spec.ts @@ -9,7 +9,7 @@ vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ }, })) -import { shouldUseFallbackChunking } from "../../shared/supported-extensions" +import { isFallbackExtension as shouldUseFallbackChunking } from "../../../shared/fallback-extensions" describe("CodeParser - VB.NET and Fallback Extensions Support", () => { let parser: CodeParser diff --git a/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts b/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts index c7584022ce..c8559ba395 100644 --- a/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts @@ -3,7 +3,7 @@ import { readFile } from "fs/promises" import type { Node, QueryCapture, Tree } from "web-tree-sitter" import { CodeParser } from "../parser" import { codeIndexLog } from "../../code-index-log" -import { MAX_BLOCK_CHARS } from "../../constants" +import { MAX_BLOCK_CHARS } from "../parser-constants" import type { CodeBlock } from "../../interfaces" import { ParserFatalError } from "../../parser-errors" import { scannerExtensions } from "../../shared/supported-extensions" diff --git a/src/services/code-index/processors/file-preparation-dependencies.ts b/src/services/code-index/processors/file-preparation-dependencies.ts deleted file mode 100644 index 001da428d7..0000000000 --- a/src/services/code-index/processors/file-preparation-dependencies.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Ignore } from "ignore" -import type { FileSystem } from "vscode" -import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" -import type { CacheManager } from "../cache-manager" -import type { ICodeParser, IEmbedder } from "../interfaces" - -export interface FilePreparationDependencies { - workspacePath: string - ignoreController: Pick - ignoreInstance?: Pick - fileSystem: Pick - cacheManager: Pick - parser: ICodeParser - embedder?: Pick -} diff --git a/src/services/code-index/processors/file-preparation.ts b/src/services/code-index/processors/file-preparation.ts deleted file mode 100644 index a99ff6b56a..0000000000 --- a/src/services/code-index/processors/file-preparation.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { createHash } from "crypto" -import { Uri } from "vscode" -import { v5 as uuidv5 } from "uuid" -import type { CodeBlock, FileProcessingResult, PointStruct } from "../interfaces" -import type { FilePreparationDependencies } from "./file-preparation-dependencies" -import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" -import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" - -/** Prepares one file for batching without writing points or mutating the hash cache. */ -export class FilePreparation { - constructor(private readonly dependencies: FilePreparationDependencies) {} - - public async prepareFile(filePath: string): Promise { - const dependencies = this.dependencies - try { - const skipReason = await this.getSkipReason(filePath) - if (skipReason !== undefined) { - return this.skippedResult(filePath, skipReason) - } - - const fileContent = await dependencies.fileSystem.readFile(Uri.file(filePath)) - const content = Buffer.from(fileContent).toString("utf-8") - const newHash = createHash("sha256").update(content).digest("hex") - - if (dependencies.cacheManager.getHash(filePath) === newHash) { - return this.skippedResult(filePath, "File has not changed") - } - - const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash }) - const pointsToUpsert = await this.preparePoints(blocks) - - return { - path: filePath, - status: "processed_for_batching", - newHash, - pointsToUpsert, - } - } catch (error) { - return { - path: filePath, - status: "local_error", - error: error as Error, - } - } - } - - private async getSkipReason(filePath: string): Promise { - const dependencies = this.dependencies - // Use relative paths so ignored directories outside the workspace do not exclude the file. - const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath) - if (isPathInIgnoredDirectory(relativeFilePath)) { - return "File is in an ignored directory" - } - - if ( - !dependencies.ignoreController.validateAccess(filePath) || - dependencies.ignoreInstance?.ignores(relativeFilePath) - ) { - return "File is ignored by .rooignore or .gitignore" - } - - const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath)) - if (fileStat.size > MAX_FILE_SIZE_BYTES) { - return "File is too large" - } - return undefined - } - - private skippedResult(filePath: string, reason: string): FileProcessingResult { - return { path: filePath, status: "skipped", reason } - } - - private async preparePoints(blocks: CodeBlock[]): Promise { - const dependencies = this.dependencies - if (!dependencies.embedder || blocks.length === 0) { - return [] - } - - const texts = blocks.map((block) => block.content) - const { embeddings } = await dependencies.embedder.createEmbeddings(texts) - - return blocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, dependencies.workspacePath) - const stableName = `${normalizedAbsolutePath}:${block.start_line}` - const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - }, - } - }) - } -} diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 84eeb301c8..1efd997f78 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -588,6 +588,3 @@ export class CodeParser implements ICodeParser { return results } } - -// Export a singleton instance for convenience -export const codeParser = new CodeParser() diff --git a/src/services/code-index/shared/__tests__/get-relative-path.spec.ts b/src/services/code-index/shared/__tests__/get-relative-path.spec.ts index d5398f1332..9981e59473 100644 --- a/src/services/code-index/shared/__tests__/get-relative-path.spec.ts +++ b/src/services/code-index/shared/__tests__/get-relative-path.spec.ts @@ -1,34 +1,8 @@ import path from "path" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../get-relative-path" +import { generateRelativeFilePath } from "../get-relative-path" describe("get-relative-path", () => { - describe("generateNormalizedAbsolutePath", () => { - it("should use provided workspace root", () => { - const filePath = "src/file.ts" - const workspaceRoot = path.join(path.sep, "custom", "workspace") - const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) - // On Windows, path.resolve adds the drive letter, so we need to use path.resolve for the expected value - expect(result).toBe(path.resolve(workspaceRoot, filePath)) - }) - - it("should handle absolute paths", () => { - const filePath = path.join(path.sep, "absolute", "path", "file.ts") - const workspaceRoot = path.join(path.sep, "custom", "workspace") - const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) - // When an absolute path is provided, it should be resolved to include drive letter on Windows - expect(result).toBe(path.resolve(filePath)) - }) - - it("should normalize paths with . and .. segments", () => { - const filePath = "./src/../src/file.ts" - const workspaceRoot = path.join(path.sep, "custom", "workspace") - const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) - // Use path.resolve to get the expected normalized absolute path - expect(result).toBe(path.resolve(workspaceRoot, "src", "file.ts")) - }) - }) - describe("generateRelativeFilePath", () => { it("should use provided workspace root", () => { const workspaceRoot = path.join(path.sep, "custom", "workspace") diff --git a/src/services/code-index/shared/get-relative-path.ts b/src/services/code-index/shared/get-relative-path.ts index 642a207208..92342bd975 100644 --- a/src/services/code-index/shared/get-relative-path.ts +++ b/src/services/code-index/shared/get-relative-path.ts @@ -1,20 +1,5 @@ import path from "path" -/** - * Generates a normalized absolute path from a given file path and workspace root. - * Handles path resolution and normalization to ensure consistent absolute paths. - * - * @param filePath - The file path to normalize (can be relative or absolute) - * @param workspaceRoot - The root directory of the workspace (required) - * @returns The normalized absolute path - */ -export function generateNormalizedAbsolutePath(filePath: string, workspaceRoot: string): string { - // Resolve the path to make it absolute if it's relative - const resolvedPath = path.resolve(workspaceRoot, filePath) - // Normalize to handle any . or .. segments and duplicate slashes - return path.normalize(resolvedPath) -} - /** * Generates a relative file path from a normalized absolute path and workspace root. * Ensures consistent relative path generation across different platforms. diff --git a/src/services/code-index/shared/supported-extensions.ts b/src/services/code-index/shared/supported-extensions.ts index a5f93e161f..6966e36f70 100644 --- a/src/services/code-index/shared/supported-extensions.ts +++ b/src/services/code-index/shared/supported-extensions.ts @@ -1,31 +1,7 @@ import { extensions as allExtensions } from "../../tree-sitter" -import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions" +import { fallbackExtensions } from "../../shared/fallback-extensions" export { fallbackExtensions } // Include all extensions including markdown for the scanner export const scannerExtensions = allExtensions - -/** - * Extensions that should always use fallback chunking instead of tree-sitter parsing. - * These are typically languages that don't have a proper WASM parser available - * or where the parser doesn't work correctly. - * - * NOTE: Only extensions that are already in the supported extensions list can be added here. - * To add support for new file types, they must first be added to the tree-sitter extensions list. - * - * HOW TO ADD A NEW FALLBACK EXTENSION: - * 1. First ensure the extension is in src/services/tree-sitter/index.ts extensions array - * 2. Add the extension to the fallbackExtensions array below - * 3. The file will automatically use length-based chunking for indexing - * - * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere - */ -/** - * Check if a file extension should use fallback chunking - * @param extension File extension (including the dot) - * @returns true if the extension should use fallback chunking - */ -export function shouldUseFallbackChunking(extension: string): boolean { - return isFallbackExtension(extension) -} From b2fc597bbc97fb39fa9646628d942b91102593ff Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:16:49 -0800 Subject: [PATCH 09/42] build: skip non-numeric Retry-After in the Qdrant patch so HTTP dates reach our parser --- patches/@qdrant__js-client-rest@1.20.0.patch | 24 +++++++++++++------- pnpm-lock.yaml | 6 ++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/patches/@qdrant__js-client-rest@1.20.0.patch b/patches/@qdrant__js-client-rest@1.20.0.patch index 42d905ae87..eea88bb792 100644 --- a/patches/@qdrant__js-client-rest@1.20.0.patch +++ b/patches/@qdrant__js-client-rest@1.20.0.patch @@ -1,48 +1,56 @@ diff --git a/dist/browser/index.cjs b/dist/browser/index.cjs --- a/dist/browser/index.cjs +++ b/dist/browser/index.cjs -@@ -991,7 +991,7 @@ +@@ -991,8 +991,8 @@ } catch (error) { if (error instanceof ApiError && error.status === 429) { - const retryAfterHeader = error.headers.get('retry-after')?.[0]; +- if (retryAfterHeader) { + const retryAfterHeader = error.headers.get('retry-after'); - if (retryAfterHeader) { ++ if (retryAfterHeader && !isNaN(Number(retryAfterHeader))) { throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); } + } diff --git a/dist/browser/index.js b/dist/browser/index.js --- a/dist/browser/index.js +++ b/dist/browser/index.js -@@ -989,7 +989,7 @@ +@@ -989,8 +989,8 @@ } catch (error) { if (error instanceof ApiError && error.status === 429) { - const retryAfterHeader = error.headers.get('retry-after')?.[0]; +- if (retryAfterHeader) { + const retryAfterHeader = error.headers.get('retry-after'); - if (retryAfterHeader) { ++ if (retryAfterHeader && !isNaN(Number(retryAfterHeader))) { throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); } + } diff --git a/dist/cjs/api-client.js b/dist/cjs/api-client.js --- a/dist/cjs/api-client.js +++ b/dist/cjs/api-client.js -@@ -51,7 +51,7 @@ +@@ -51,8 +51,8 @@ } catch (error) { if (error instanceof openapi_typescript_fetch_1.ApiError && error.status === 429) { - const retryAfterHeader = error.headers.get('retry-after')?.[0]; +- if (retryAfterHeader) { + const retryAfterHeader = error.headers.get('retry-after'); - if (retryAfterHeader) { ++ if (retryAfterHeader && !isNaN(Number(retryAfterHeader))) { throw new errors_js_1.QdrantClientResourceExhaustedError(error.message, retryAfterHeader); } + } diff --git a/dist/esm/api-client.js b/dist/esm/api-client.js --- a/dist/esm/api-client.js +++ b/dist/esm/api-client.js -@@ -47,7 +47,7 @@ +@@ -47,8 +47,8 @@ } catch (error) { if (error instanceof ApiError && error.status === 429) { - const retryAfterHeader = error.headers.get('retry-after')?.[0]; +- if (retryAfterHeader) { + const retryAfterHeader = error.headers.get('retry-after'); - if (retryAfterHeader) { ++ if (retryAfterHeader && !isNaN(Number(retryAfterHeader))) { throw new QdrantClientResourceExhaustedError(error.message, retryAfterHeader); } + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f3e83f360..15d0ac316f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,7 +48,7 @@ overrides: patchedDependencies: '@qdrant/js-client-rest@1.20.0': - hash: a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd + hash: 25b0bcb3835c3b5fe58aa29056bebb895a7d48958dac9d69a33ca7c7186fadd9 path: patches/@qdrant__js-client-rest@1.20.0.patch importers: @@ -476,7 +476,7 @@ importers: version: 1.32.2(zod@4.6.7) '@qdrant/js-client-rest': specifier: 1.20.0 - version: 1.20.0(patch_hash=a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd)(typescript@5.9.3) + version: 1.20.0(patch_hash=25b0bcb3835c3b5fe58aa29056bebb895a7d48958dac9d69a33ca7c7186fadd9)(typescript@5.9.3) '@roo-code/cloud': specifier: workspace:^ version: link:../packages/cloud @@ -10691,7 +10691,7 @@ snapshots: '@protobufjs/utf8@1.1.2': {} - '@qdrant/js-client-rest@1.20.0(patch_hash=a2f576a222593d493080be95751562c88867de5bd9cf6c4ab6e37a857eb5e8fd)(typescript@5.9.3)': + '@qdrant/js-client-rest@1.20.0(patch_hash=25b0bcb3835c3b5fe58aa29056bebb895a7d48958dac9d69a33ca7c7186fadd9)(typescript@5.9.3)': dependencies: '@qdrant/openapi-typescript-fetch': 1.2.6 typescript: 5.9.3 From 92c9b71c8c5b2d4dfcebeb8accd141351baf3e77 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:22:38 -0800 Subject: [PATCH 10/42] test(code-index): await both concurrent verify counts in the Retry-After contract --- .../__tests__/qdrant-client.retry-contract.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts index af2bec52de..c3418ed20e 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts @@ -105,7 +105,8 @@ it.each([ expect(error).toMatchObject({ retryAfterMs: seconds * 1000, classification: "rate-limit", statusCode: 429 }) expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) expect(error.cause).toMatchObject({ retry_after: seconds }) - expect(requests).toHaveLength(activity === "verify" ? 2 : 1) + // Verify issues two concurrent counts; the rejection can arrive before the second reaches the server. + await vi.waitFor(() => expect(requests).toHaveLength(activity === "verify" ? 2 : 1)) }) it.each(["upsert", "delete", "verify"] as const)("%s is a single attempt without the policy", async (activity) => { From 88445751ce71e3c5b3f6ee29146842dcb9c159b2 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 10:34:23 -0800 Subject: [PATCH 11/42] fix(code-index): load web-tree-sitter's CJS build in the parse worker bundle --- src/services/code-index/parse-worker.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/parse-worker.ts b/src/services/code-index/parse-worker.ts index 67b73c7c24..f6e7d96f0c 100644 --- a/src/services/code-index/parse-worker.ts +++ b/src/services/code-index/parse-worker.ts @@ -2,7 +2,6 @@ import { codeIndexLog } from "./code-index-log" import { parentPort, workerData } from "node:worker_threads" import { readFile } from "node:fs/promises" import * as path from "node:path" -import { Parser } from "web-tree-sitter" import { CodeParser } from "./processors/parser" import { isParserAbort } from "./parser-errors" import type { ParseRequest, ParseReply } from "./parse-worker-protocol" @@ -22,6 +21,8 @@ parentPort!.on("message", async (request: ParseRequest) => { codeIndexLog.info("parse worker entering wasm fault loop") ;(instance.exports.loop as () => void)() } + // MUST use require: an ESM import bundles web-tree-sitter's ESM build, whose import.meta.url is undefined in CJS. + const { Parser } = require("web-tree-sitter") as typeof import("web-tree-sitter") await Parser.init({ locateFile: (filename: string) => path.join(wasmDirectory, filename) }) const blocks = await parser.parseFile(request.filePath, { content: request.content, From b59270269f57c151b76bfa593b37529b3e5132f8 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 12:13:20 -0800 Subject: [PATCH 12/42] fix(code-index): address CodeRabbit review feedback Continue stopping other workspaces when one stop fails, reject retry delays past the Date range, and tighten listener and replacement assertions. --- .../webview/__tests__/ClineProvider.spec.ts | 17 ++++++----------- src/core/webview/webviewMessageHandler.ts | 8 +++++++- .../embedders/__tests__/cancellation.spec.ts | 8 ++++++-- .../shared/__tests__/indexing-errors.spec.ts | 2 ++ .../code-index/shared/indexing-errors.ts | 9 +++++++++ .../__tests__/qdrant-client.http-pacing.spec.ts | 2 +- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index a639abdad7..2261ae3a28 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -6337,19 +6337,14 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.log).toHaveBeenCalledExactlyOnceWith( enabled ? "Error starting auto-enabled indexing: auto-enable failure" - : "Error setting auto-enable default: auto-enable failure", + : "Error stopping auto-disabled indexing: auto-enable failure", ) } else expect(provider.log).not.toHaveBeenCalled() - if (!enabled && outcome === "reject") { - expect(manager.getCurrentStatus).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - } else { - expect(manager.getCurrentStatus).toHaveBeenCalledOnce() - expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ - type: "indexingStatusUpdate", - values: manager.getCurrentStatus.mock.results[0].value, - }) - } + expect(manager.getCurrentStatus).toHaveBeenCalledOnce() + expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ + type: "indexingStatusUpdate", + values: manager.getCurrentStatus.mock.results[0].value, + }) } finally { getAllInstances.mockRestore() } diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 943a298339..b7e0e23a41 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3574,7 +3574,13 @@ export const webviewMessageHandler = async ( const wasEnabled = priorStates.get(m)! const isNowEnabled = m.isWorkspaceEnabled if (wasEnabled && !isNowEnabled) { - await m.stopIndexing() + try { + await m.stopIndexing() + } catch (error) { + provider.log( + `Error stopping auto-disabled indexing: ${error instanceof Error ? error.message : String(error)}`, + ) + } } else if (!wasEnabled && isNowEnabled && m.isFeatureEnabled && m.isFeatureConfigured) { try { await m.startIndexing(provider.contextProxy) diff --git a/src/services/code-index/embedders/__tests__/cancellation.spec.ts b/src/services/code-index/embedders/__tests__/cancellation.spec.ts index db907c768a..bfc718c425 100644 --- a/src/services/code-index/embedders/__tests__/cancellation.spec.ts +++ b/src/services/code-index/embedders/__tests__/cancellation.spec.ts @@ -219,11 +219,13 @@ describe("cancellable wait", () => { it("removes its abort listener when the wait completes", async () => { const controller = new AbortController() + const addListener = vi.spyOn(controller.signal, "addEventListener") const removeListener = vi.spyOn(controller.signal, "removeEventListener") const result = waitForRetry(100, controller.signal) await vi.advanceTimersByTimeAsync(100) await result - expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)) + expect(addListener).toHaveBeenCalledOnce() + expect(removeListener).toHaveBeenCalledWith("abort", addListener.mock.calls[0][1]) expect(vi.getTimerCount()).toBe(0) }) }) @@ -248,11 +250,13 @@ it("Ollama keeps its timeout when a caller signal is supplied and cleans up afte it("cancels a wait with the unchanged reason and removes its timer and listener", async () => { const controller = new AbortController() + const addListener = vi.spyOn(controller.signal, "addEventListener") const removeListener = vi.spyOn(controller.signal, "removeEventListener") const reason = Object.freeze({ stopped: true }) const pending = waitForRetry(60_000, controller.signal) controller.abort(reason) await expect(pending).rejects.toBe(reason) - expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)) + expect(addListener).toHaveBeenCalledOnce() + expect(removeListener).toHaveBeenCalledWith("abort", addListener.mock.calls[0][1]) expect(vi.getTimerCount()).toBe(0) }) diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts index eddab2f58c..d296f22c20 100644 --- a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -102,6 +102,8 @@ describe("parseRetryAfterMs", () => { [0, 0], [-1, undefined], [Infinity, undefined], + [{ "retry-after-ms": "1e308" }, undefined], + [1e300, undefined], [undefined, undefined], ] as const)("parses %o", (source, expected) => { expect(parseRetryAfterMs(source, now)).toBe(expected) diff --git a/src/services/code-index/shared/indexing-errors.ts b/src/services/code-index/shared/indexing-errors.ts index d3b09b8c3a..c893f3fb95 100644 --- a/src/services/code-index/shared/indexing-errors.ts +++ b/src/services/code-index/shared/indexing-errors.ts @@ -29,7 +29,16 @@ function nonnegativeNumber(value: unknown): number | undefined { return Number.isFinite(number) && number >= 0 ? number : undefined } +// A delay whose resume deadline falls outside the Date range would make toISOString() throw. +function validDelay(delay: number | undefined, now: number): number | undefined { + return delay !== undefined && Number.isFinite(new Date(now + delay).getTime()) ? delay : undefined +} + export function parseRetryAfterMs(source: unknown, now = Date.now()): number | undefined { + return validDelay(parseRawRetryAfterMs(source, now), now) +} + +function parseRawRetryAfterMs(source: unknown, now: number): number | undefined { if (typeof source === "number") { const seconds = nonnegativeNumber(source) return seconds !== undefined && Number.isFinite(seconds * 1000) ? seconds * 1000 : undefined diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts index 4ee365d748..cfc324a594 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts @@ -164,7 +164,7 @@ it.each(scenarios)("T26/T32 HTTP $activity $scenario", async ({ activity, header }) await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) await waitForTransport(() => expect(cache.completeFile).toHaveBeenCalledOnce()) - await pending + await expect(pending).resolves.toBeUndefined() expect(requests).toBe((limits + 1) * perAttempt) expect(failures.list()).toEqual([]) expect(cache.completeFile).toHaveBeenCalledOnce() From 4aca2e61b1741c2c18553037b27f32ecc3e25654 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 12:27:00 -0800 Subject: [PATCH 13/42] fix(code-index): address CodeRabbit round 2 feedback Keep batch partial state for watcher failures the registry does not hold, format provider resume times in the extension language, and tighten tests for listener cleanup, pacer path aggregation, scan escalation, mixed block filtering, explicit search incompleteness and multi-workspace auto-disable stops. --- .../__tests__/CodebaseSearchTool.spec.ts | 19 +++++++++++- .../webview/__tests__/ClineProvider.spec.ts | 30 +++++++++++++++++++ .../code-index-scan-executor.spec.ts | 20 +++++++++++++ .../__tests__/file-completion.spec.ts | 19 ++++++++++++ .../code-index/__tests__/orchestrator.spec.ts | 14 +++++++-- .../__tests__/provider-pacer.spec.ts | 20 +++++++++---- src/services/code-index/orchestrator.ts | 15 ++++++---- .../src/components/chat/CodeIndexPopover.tsx | 5 ++-- .../chat/__tests__/CodeIndexPopover.test.tsx | 24 ++++++++------- .../__tests__/IndexingStatusBadge.spec.tsx | 6 +++- 10 files changed, 145 insertions(+), 27 deletions(-) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts index 9d6b4ab616..257638f9d9 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -102,8 +102,9 @@ describe("CodebaseSearchTool", () => { results: VectorStoreSearchResult[] = [], failedPaths: string[] = [], waitingPaths: string[] = [], + incomplete = failedPaths.length > 0 || waitingPaths.length > 0, ): Awaited> { - return { results, incomplete: failedPaths.length > 0 || waitingPaths.length > 0, failedPaths, waitingPaths } + return { results, incomplete, failedPaths, waitingPaths } } it("exports a named tool instance", () => { @@ -624,4 +625,20 @@ describe("CodebaseSearchTool", () => { expect(output).toEqual(expect.stringContaining(hasMatches ? "Query:" : "No relevant code snippets")) }, ) + + it.each([ + { waitingPaths: [], expected: "Partial index: results may be incomplete.\n\n" }, + { + waitingPaths: ["waiting.ts"], + expected: + "Partial index: results may be incomplete.\n" + + "Waiting files (indexing resumes automatically):\n- waiting.ts\n\n", + }, + ])("follows the explicit incomplete flag with waiting paths $waitingPaths", async ({ waitingPaths, expected }) => { + vi.mocked(manager.searchIndex).mockResolvedValue(searchResponse([], [], waitingPaths, true)) + await tool.execute({ query }, task, callbacks) + const output = vi.mocked(callbacks.pushToolResult).mock.calls[0][0] + expect(output).toEqual(expect.stringContaining(expected)) + expect(output).not.toEqual(expect.stringContaining("Failed files:")) + }) }) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 2261ae3a28..d86bc1f4d7 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -6350,6 +6350,36 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { } }) + it("keeps stopping other workspaces after one auto-disabled stop fails", async () => { + const { CodeIndexManagerRegistry } = await import("../../../services/code-index/code-index-manager-registry") + let workspaceEnabled = true + const managers = [0, 1].map(() => createIndexManager()) + managers[0].setAutoEnableDefault.mockImplementation(async () => { + workspaceEnabled = false + }) + managers[0].stopIndexing.mockRejectedValue(new Error("first stop failed")) + for (const manager of managers) { + Object.defineProperty(manager, "isWorkspaceEnabled", { get: () => workspaceEnabled }) + } + // The registry double provides the enablement, stop, and status members used by this handler. + const getAllInstances = vi + .spyOn(CodeIndexManagerRegistry, "getAllInstances") + .mockReturnValue(managers as unknown as ReturnType) + const provider = createProvider({ + getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: managers[0] }), + }) + try { + await webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: false }) + expect(managers[0].stopIndexing).toHaveBeenCalledOnce() + expect(managers[1].stopIndexing).toHaveBeenCalledOnce() + expect(provider.log).toHaveBeenCalledExactlyOnceWith( + "Error stopping auto-disabled indexing: first stop failed", + ) + } finally { + getAllInstances.mockRestore() + } + }) + it("covers changed clear-index response paths", async () => { const manager = createIndexManager() const getScope = vi.fn().mockReturnValueOnce(undefined).mockReturnValue({ codeIndexManager: manager }) diff --git a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts index e69ba39579..778ead0516 100644 --- a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts +++ b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts @@ -139,4 +139,24 @@ describe("CodeIndexScanExecutor", () => { } }, ) + + function embeddingFailure(status: number): Error { + return new Error("file embedding failed", { + cause: new IndexingRetryError(new EmbeddingError("provider failed", { status }), 1, "permanent", "embed"), + }) + } + + it.each([ + { method: "runIncrementalScan" as const, statuses: [401] }, + { method: "runFullScan" as const, statuses: [401, 500] }, + ])("does not escalate $method with statuses $statuses", async ({ method, statuses }) => { + const { executor, scanner } = setup() + scanner.scanDirectory.mockImplementation(async (_directory, onError, _onIndexed, onParsed) => { + onParsed?.(statuses.length) + for (const status of statuses) onError?.(embeddingFailure(status)) + return { stats: { processed: 0, skipped: 0 }, totalBlockCount: statuses.length } + }) + await expect(executor[method](new AbortController().signal)).resolves.toBe(true) + expect(executor.partial).toBe(true) + }) }) diff --git a/src/services/code-index/__tests__/file-completion.spec.ts b/src/services/code-index/__tests__/file-completion.spec.ts index 606506ab1e..e8e0205ef6 100644 --- a/src/services/code-index/__tests__/file-completion.spec.ts +++ b/src/services/code-index/__tests__/file-completion.spec.ts @@ -558,6 +558,25 @@ describe("FileIndexWriter", () => { expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") }) + it("filters only whitespace blocks from mixed content and types", async () => { + const mixed = blocks(4) + mixed[0].type = "class" + mixed[1].content = " \n" + mixed[2].type = "method" + mixed[3].type = "interface" + mixed[3].content = "\t" + store.countPointsForFile.mockResolvedValue({ current: 2, other: 0 }) + const onComplete = vi.fn() + await writer.replace(filePath, "hash", mixed, embedder, attempt.signal, undefined, onComplete) + expect(embedder.createEmbeddings.mock.calls.flatMap(([texts]) => texts)).toEqual([" block 0 ", " block 2 "]) + const points = store.upsertPoints.mock.calls.flatMap(([batch]) => batch) + expect(points.map((point) => point.payload)).toEqual([ + expect.objectContaining({ codeChunk: " block 0 ", segmentHash: "segment-0" }), + expect.objectContaining({ codeChunk: " block 2 ", segmentHash: "segment-2" }), + ]) + expect(onComplete).toHaveBeenCalledExactlyOnceWith(2) + }) + it("does not issue cleanup or completion after a failed batch", async () => { store.upsertPoints.mockRejectedValueOnce(new Error("Upsert failed")) await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow( diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 117838f198..9dcf3fe403 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -366,8 +366,6 @@ describe("CodeIndexOrchestrator lifecycle", () => { "does not complete a scan with %s", async (reason) => { scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "verify", "save failed", 3, "exhausted") if (reason === "batch error") onError?.(new Error("save failed")) if (reason === "missing blocks") { onParsed?.(10) @@ -403,6 +401,18 @@ describe("CodeIndexOrchestrator lifecycle", () => { expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Indexed")).toBe(false) }) + it("keeps an unregistered watcher failure partial while another file is registered", async () => { + await orchestrator.startIndexing() + failures.renew("other.ts", "revision") + failures.record("other.ts", "revision", "read", "read failed", 1, "exhausted") + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ + processedFiles: [{ path: "/workspace/unregistered.ts", status: "local_error" }], + }) + failures.clear("other.ts", "revision") + await drain() + expect(state.state).toBe("Standby") + }) + it("reports partial when cancellation interrupts a watcher before durable invalidation", async () => { await orchestrator.startIndexing() watcher.isProcessing = true diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index c5c50a276c..41481eceb6 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -367,17 +367,25 @@ describe("ProviderPacerRegistry and scope", () => { embedder.reportRateLimit(limited(1000), embedder.markIssued()) store.reportRateLimit(limited(2000), store.markIssued()) const controller = new AbortController() - const outcomes = [embedder, embedder, store].map((pacer) => - pacer.run("same.ts", controller.signal, async () => undefined).catch((error) => error), + const queued: [typeof embedder, string][] = [ + [embedder, "same.ts"], + [embedder, "embed-only.ts"], + [embedder, "same.ts"], + [store, "same.ts"], + [store, "store-only.ts"], + ] + const outcomes = queued.map(([pacer, file]) => + pacer.run(file, controller.signal, async () => undefined).catch((error) => error), ) await flush() - expect(registry.waitingPaths()).toEqual(["same.ts"]) + const allPaths = ["same.ts", "embed-only.ts", "store-only.ts"] + expect(registry.waitingPaths()).toEqual(allPaths) expect(registry.providerWaits()).toEqual([ - { service: "openai", resumeAt: 1000, waitingFiles: ["same.ts"] }, - { service: "qdrant", resumeAt: 2000, waitingFiles: ["same.ts"] }, + { service: "openai", resumeAt: 1000, waitingFiles: ["same.ts", "embed-only.ts"] }, + { service: "qdrant", resumeAt: 2000, waitingFiles: ["same.ts", "store-only.ts"] }, ]) registry.providerWaits()[0].waitingFiles.push("mutated") - expect(registry.waitingPaths()).toEqual(["same.ts"]) + expect(registry.waitingPaths()).toEqual(allPaths) controller.abort() await Promise.all(outcomes) expect(registry.waitingPaths()).toEqual([]) diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 6ddabb23a5..1f070d101a 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -15,6 +15,7 @@ import { ProviderPacerRegistry } from "./provider-pacer" import { INDEX_STOP_LIMIT_MS } from "./constants" import { codeIndexLog } from "./code-index-log" import { formatProviderWaitMessage } from "./shared/provider-wait-message" +import { generateRelativeFilePath } from "./shared/get-relative-path" export class CodeIndexOrchestrator { private watcherSubscriptions: vscode.Disposable[] = [] @@ -41,7 +42,7 @@ export class CodeIndexOrchestrator { constructor( private readonly configManager: CodeIndexConfigManager, private readonly stateManager: CodeIndexStateManager, - workspacePath: string, + private readonly workspacePath: string, private readonly cacheManager: CacheManager, private readonly vectorStore: IVectorStore, private readonly scanner: IDirectoryScanner, @@ -175,11 +176,15 @@ export class CodeIndexOrchestrator { if (!this.isLive(signal)) return this.watcherBatchPending = false this.naturallySettled = true - this.batchPartial = - !!summary.batchError || - summary.processedFiles.some((file) => file.status === "error" || file.status === "local_error") // Registry-backed failures recover through the registry, not a sticky batch flag. - if (this.failures.list().length > 0) this.batchPartial = false + const registered = new Set(this.failures.list().map((failure) => failure.path)) + this.batchPartial = + (!!summary.batchError && summary.processedFiles.length === 0) || + summary.processedFiles.some( + (file) => + (file.status === "error" || file.status === "local_error") && + !registered.has(generateRelativeFilePath(file.path, this.workspacePath)), + ) this.updateRecoveryStatus(signal) }), ] diff --git a/webview-ui/src/components/chat/CodeIndexPopover.tsx b/webview-ui/src/components/chat/CodeIndexPopover.tsx index 0770a77de3..8b3b76b504 100644 --- a/webview-ui/src/components/chat/CodeIndexPopover.tsx +++ b/webview-ui/src/components/chat/CodeIndexPopover.tsx @@ -201,7 +201,8 @@ export const CodeIndexPopover: React.FC = ({ }) => { const SECRET_PLACEHOLDER = "••••••••••••••••" const { t } = useAppTranslation() - const { codebaseIndexConfig, codebaseIndexModels, cwd, apiConfiguration, platform, arch } = useExtensionState() + const { codebaseIndexConfig, codebaseIndexModels, cwd, apiConfiguration, platform, arch, language } = + useExtensionState() const [open, setOpen] = useState(false) const [isAdvancedSettingsOpen, setIsAdvancedSettingsOpen] = useState(false) const [isSetupSettingsOpen, setIsSetupSettingsOpen] = useState(false) @@ -706,7 +707,7 @@ export const CodeIndexPopover: React.FC = ({ time: wait.resumeAt === undefined ? undefined - : new Date(wait.resumeAt).toLocaleString(), + : new Date(wait.resumeAt).toLocaleString(language), }, )}

diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx index 40b4b758f0..0f45860584 100644 --- a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx +++ b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx @@ -284,13 +284,14 @@ const failedStatus = ( } } -const renderStatusPopover = (status: IndexingStatus) => +const renderStatusPopover = (status: IndexingStatus, language: ExtensionState["language"] = "en") => renderWithExtensionState( , { state: { + language, cwd: "/workspace", codebaseIndexConfig: { ...baseSettings, codebaseIndexEmbedderProvider: providerIdentifiers.openai }, codebaseIndexModels: {}, @@ -316,14 +317,17 @@ describe("CodeIndexPopover indexing reliability", () => { async (systemStatus) => { await translation.changeLanguage("de") const resumeAt = Date.parse("2026-11-11T12:02:00Z") - renderStatusPopover({ - ...indexingStatus, - systemStatus, - providerWaits: [ - { service: "openai", resumeAt, waitingFiles: ["waiting.ts"] }, - { service: "qdrant", waitingFiles: [] }, - ], - }) + renderStatusPopover( + { + ...indexingStatus, + systemStatus, + providerWaits: [ + { service: "openai", resumeAt, waitingFiles: ["waiting.ts"] }, + { service: "qdrant", waitingFiles: [] }, + ], + }, + "de", + ) const active = systemStatus === "Indexing" const timed = active ? "providerWaitingUntil" : "providerPausedUntil" const untimed = active ? "providerWaiting" : "providerPaused" @@ -331,7 +335,7 @@ describe("CodeIndexPopover indexing reliability", () => { screen.getByText( translation.t(`settings:codeIndex.${timed}`, { service: "openai", - time: new Date(resumeAt).toLocaleString(), + time: new Date(resumeAt).toLocaleString("de"), }), ), ).toBeVisible() diff --git a/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx b/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx index 2eb7f24337..0ba260b3bb 100644 --- a/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx +++ b/webview-ui/src/components/chat/__tests__/IndexingStatusBadge.spec.tsx @@ -224,10 +224,14 @@ describe("IndexingStatusBadge", () => { }) it("cleans up its event listener on unmount", () => { + const addListener = vi.spyOn(window, "addEventListener") const { unmount } = render() + const registered = addListener.mock.calls.find(([type]) => type === "message")?.[1] + expect(registered).toBeDefined() const removeListener = vi.spyOn(window, "removeEventListener") unmount() - expect(removeListener).toHaveBeenCalledWith("message", expect.any(Function)) + expect(removeListener).toHaveBeenCalledWith("message", registered) + addListener.mockRestore() removeListener.mockRestore() }) }) From bb181b2fe8f3addffb66eb435089985fd3fa41cb Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 13:42:18 -0800 Subject: [PATCH 14/42] test(code-index): match trimmed embedding input; cover registered recovery and whole-batch error --- .../__tests__/file-completion.spec.ts | 2 +- .../code-index/__tests__/orchestrator.spec.ts | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/services/code-index/__tests__/file-completion.spec.ts b/src/services/code-index/__tests__/file-completion.spec.ts index e8e0205ef6..2ad8d3bbe4 100644 --- a/src/services/code-index/__tests__/file-completion.spec.ts +++ b/src/services/code-index/__tests__/file-completion.spec.ts @@ -568,7 +568,7 @@ describe("FileIndexWriter", () => { store.countPointsForFile.mockResolvedValue({ current: 2, other: 0 }) const onComplete = vi.fn() await writer.replace(filePath, "hash", mixed, embedder, attempt.signal, undefined, onComplete) - expect(embedder.createEmbeddings.mock.calls.flatMap(([texts]) => texts)).toEqual([" block 0 ", " block 2 "]) + expect(embedder.createEmbeddings.mock.calls.flatMap(([texts]) => texts)).toEqual(["block 0", "block 2"]) const points = store.upsertPoints.mock.calls.flatMap(([batch]) => batch) expect(points.map((point) => point.payload)).toEqual([ expect.objectContaining({ codeChunk: " block 0 ", segmentHash: "segment-0" }), diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 9dcf3fe403..c633799bbd 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -396,11 +396,35 @@ describe("CodeIndexOrchestrator lifecycle", () => { expect(state.state).toBe("Indexing") failures.renew("bad.ts", "revision") failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [{ path: "bad.ts", status: "error" }] }) + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ + processedFiles: [{ path: "/workspace/bad.ts", status: "error" }], + }) expect(state.state).toBe("Standby") expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Indexed")).toBe(false) }) + it.each([ + { + name: "recovers Indexed once a registered failed file clears", + summary: { processedFiles: [{ path: "/workspace/bad.ts", status: "error" as const }] }, + expected: "Indexed", + }, + { + name: "stays partial after a whole-batch error with no processed files", + summary: { processedFiles: [], batchError: new Error("batch failed") }, + expected: "Standby", + }, + ])("watcher batch $name", async ({ summary, expected }) => { + await orchestrator.startIndexing() + watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) + failures.renew("bad.ts", "revision") + failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") + watcher.onDidFinishBatchProcessing.mock.calls[0][0](summary) + failures.clear("bad.ts", "revision") + await drain() + expect(state.state).toBe(expected) + }) + it("keeps an unregistered watcher failure partial while another file is registered", async () => { await orchestrator.startIndexing() failures.renew("other.ts", "revision") From dc8deae3b2507d1998d70edf1dba6686c28f51fa Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 14:52:14 -0800 Subject: [PATCH 15/42] fix(code-index): guard pacer deadline and use literal incomplete default in search test --- src/core/tools/__tests__/CodebaseSearchTool.spec.ts | 8 +++++--- src/services/code-index/__tests__/provider-pacer.spec.ts | 2 +- src/services/code-index/provider-pacer.ts | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts index 257638f9d9..739b4068dc 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -102,7 +102,7 @@ describe("CodebaseSearchTool", () => { results: VectorStoreSearchResult[] = [], failedPaths: string[] = [], waitingPaths: string[] = [], - incomplete = failedPaths.length > 0 || waitingPaths.length > 0, + incomplete = false, ): Awaited> { return { results, incomplete, failedPaths, waitingPaths } } @@ -299,7 +299,9 @@ describe("CodebaseSearchTool", () => { it.each([false, true])("warns about an incomplete index with matches: %s", async (hasMatches) => { const failedPaths = ["src/failed.ts", "other/unreadable.ts"] - vi.mocked(manager.searchIndex).mockResolvedValue(searchResponse(hasMatches ? [result()] : [], failedPaths)) + vi.mocked(manager.searchIndex).mockResolvedValue( + searchResponse(hasMatches ? [result()] : [], failedPaths, [], true), + ) await tool.execute({ query, path: "src" }, task, callbacks) @@ -612,7 +614,7 @@ describe("CodebaseSearchTool", () => { "prints separate waiting and failed warnings before output, matches=%s", async (hasMatches) => { vi.mocked(manager.searchIndex).mockResolvedValue( - searchResponse(hasMatches ? [result()] : [], ["failed.ts"], ["waiting.ts"]), + searchResponse(hasMatches ? [result()] : [], ["failed.ts"], ["waiting.ts"], true), ) await tool.execute({ query }, task, callbacks) const output = vi.mocked(callbacks.pushToolResult).mock.calls[0][0] diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 41481eceb6..94424ddb9e 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -203,7 +203,7 @@ describe("ProviderPacer", () => { expect(request).not.toHaveBeenCalled() }) - it.each([0, -1, NaN, Infinity, "invalid"])( + it.each([0, -1, NaN, Infinity, "invalid", 1e300, 8.64e15 + 1])( "uses fallback for invalid or elapsed provider delay %s", (retryAfterMs) => { const pacer = new ProviderPacer("openai:test") diff --git a/src/services/code-index/provider-pacer.ts b/src/services/code-index/provider-pacer.ts index d754951ad2..324d0b7c9f 100644 --- a/src/services/code-index/provider-pacer.ts +++ b/src/services/code-index/provider-pacer.ts @@ -118,8 +118,11 @@ export class ProviderPacer { const paused = now < this.pausedUntil const stale = issueSeq < this.episodeSeq const requestedDelay = new EmbeddingError("Provider rate limited", error).retryAfterMs + // An out-of-range deadline would break the ISO resume time, so back off instead. const providerDelay = - typeof requestedDelay === "number" && Number.isFinite(requestedDelay) && requestedDelay > 0 + typeof requestedDelay === "number" && + requestedDelay > 0 && + Number.isFinite(new Date(now + requestedDelay).getTime()) ? requestedDelay : undefined if (stale && providerDelay === undefined) return From bd8d4f641253a56e4180f3017796610bf180b2ee Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 16:01:11 -0800 Subject: [PATCH 16/42] test(code-index): cover provider deadline that becomes out-of-range as time advances --- .../__tests__/provider-pacer.spec.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 94424ddb9e..549c094f9c 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -204,7 +204,7 @@ describe("ProviderPacer", () => { }) it.each([0, -1, NaN, Infinity, "invalid", 1e300, 8.64e15 + 1])( - "uses fallback for invalid or elapsed provider delay %s", + "uses fallback for invalid, elapsed, or out-of-range provider delay %s", (retryAfterMs) => { const pacer = new ProviderPacer("openai:test") pacer.reportRateLimit({ status: 429, retryAfterMs }, pacer.markIssued()) @@ -212,6 +212,26 @@ describe("ProviderPacer", () => { }, ) + // The deadline limit is 8.64e15 ms (max Date), so the same delay flips from valid to invalid as time advances. + it.each([ + { advanceMs: 0, retryAfterMs: 8.64e15, pausedUntil: 8.64e15, source: "provider" }, + { advanceMs: 0, retryAfterMs: 8.64e15 - 500, pausedUntil: 8.64e15 - 500, source: "provider" }, + { advanceMs: 1000, retryAfterMs: 8.64e15 - 500, pausedUntil: 1500, source: "backoff" }, + ])( + "validates the provider deadline at report time (advance $advanceMs ms, delay $retryAfterMs)", + async ({ advanceMs, retryAfterMs, pausedUntil, source }) => { + const pacer = new ProviderPacer("openai:test") + await vi.advanceTimersByTimeAsync(advanceMs) + pacer.reportRateLimit(limited(retryAfterMs), pacer.markIssued()) + expect(pacer.state().pausedUntil).toBe(pausedUntil) + expect(codeIndexLog.info).toHaveBeenCalledWith( + "Provider pacing wait", + expect.objectContaining({ source }), + ) + expect(new Date(pausedUntil).toISOString()).toEqual(expect.any(String)) + }, + ) + it("paused search reports extend without counting episodes or shortening waits", async () => { const pacer = new ProviderPacer("openai:test") pacer.reportRateLimit(limited(1000), pacer.markIssued()) From 8e04d2882688e8e0accb4cd6ab9c8c819dfec4d4 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 16:05:48 -0800 Subject: [PATCH 17/42] style(code-index): apply prettier to provider pacer test --- src/services/code-index/__tests__/provider-pacer.spec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 549c094f9c..6b8c05c69b 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -224,10 +224,7 @@ describe("ProviderPacer", () => { await vi.advanceTimersByTimeAsync(advanceMs) pacer.reportRateLimit(limited(retryAfterMs), pacer.markIssued()) expect(pacer.state().pausedUntil).toBe(pausedUntil) - expect(codeIndexLog.info).toHaveBeenCalledWith( - "Provider pacing wait", - expect.objectContaining({ source }), - ) + expect(codeIndexLog.info).toHaveBeenCalledWith("Provider pacing wait", expect.objectContaining({ source })) expect(new Date(pausedUntil).toISOString()).toEqual(expect.any(String)) }, ) From 2c63f85755c9efa3e3a79926291a5032281049d7 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 18:01:07 -0800 Subject: [PATCH 18/42] fix(code-index): treat missing source files as removed (port fork #116) Ports simurg79/Roo-Code#116: ENOENT/FileNotFound at a file's own stat or read now routes to removal in the scanner and watcher instead of a read failure. Adds T42/T43/T44 tests adapted to Zoo harnesses. --- .../processors/__tests__/file-watcher.spec.ts | 169 +++++++++++++ .../processors/__tests__/scanner.spec.ts | 228 ++++++++++++++++++ .../code-index/processors/file-watcher.ts | 14 +- src/services/code-index/processors/scanner.ts | 19 +- .../shared/__tests__/indexing-errors.spec.ts | 34 +++ .../code-index/shared/indexing-errors.ts | 18 ++ 6 files changed, 476 insertions(+), 6 deletions(-) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 500089bb1b..4959926e14 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -22,6 +22,7 @@ import { MAX_FILE_SIZE_BYTES } from "../../constants" import { RooIgnoreController } from "../../../../core/ignore/RooIgnoreController" import { makeExtensionContext, makeUri } from "../../../../test-utils/vscode" import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" +import { ProviderPacerRegistry } from "../../provider-pacer" vi.mock("vscode", async () => { const { makeEventEmitter, makeUri } = await import("../../../../test-utils/vscode") @@ -49,6 +50,8 @@ vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ vi.mock("../../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), + debug: vi.fn(), + warn: vi.fn(), redact: (message: string) => message, createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), }, @@ -166,6 +169,7 @@ function setup() { const generations = new GenerationAllocator(cacheManager) const ignored = ignore() const access = new RooIgnoreController(workspacePath) + const pacers = new ProviderPacerRegistry() let currentAttempt = true const createWatcher = () => { const created = new FileWatcher( @@ -182,6 +186,7 @@ function setup() { generations, failures, () => currentAttempt, + pacers, ) watchers.push(created) return created @@ -198,6 +203,7 @@ function setup() { failures, gate, ignored, + pacers, createWatcher, watcher: createWatcher(), supersede: () => { @@ -723,3 +729,166 @@ describe("FileWatcher reliability", () => { expect(parser.parseFile).toHaveBeenCalledTimes(11) }) }) + +describe("T43 watcher source missing at read is a removal", () => { + const errorWithCode = (code: string) => Object.assign(new Error("gone"), { code }) + const failAt = (stage: "stat" | "read", error: Error) => { + if (stage === "stat") vi.mocked(vscode.workspace.fs.stat).mockRejectedValueOnce(error) + else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(error) + } + const pointsFor = (points: Map, filePath: string) => + [...points.values()].filter((point) => { + const payload: Record = point.payload + return payload.filePath === path.relative(workspacePath, filePath) + }) + const fileStat = (size: number): vscode.FileStat => ({ size, type: 1, ctime: 0, mtime: 0 }) + + it.each(["stat", "read"] as const)("never-indexed file missing at %s is skipped", async (stage) => { + const { watcher, store, failures } = setup() + failAt(stage, errorWithCode("FileNotFound")) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "skipped" }) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }) + + it.each(["stat", "read"] as const)("previously indexed file missing at %s is removed", async (stage) => { + const { watcher, points, hashes, failures } = setup() + expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) + expect(pointsFor(points, fileA)).not.toEqual([]) + failAt(stage, errorWithCode(stage === "stat" ? "FileNotFound" : "ENOENT")) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) + expect(pointsFor(points, fileA)).toEqual([]) + expect(hashes.get(fileA)).toBeUndefined() + expect(failures.list()).toEqual([]) + }) + + it("clears a prior failure for a path found missing", async () => { + const { watcher, parser, failures } = setup() + parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) + await watcher.processFile(fileA) + expect(failures.list()).toHaveLength(1) + failAt("stat", errorWithCode("FileNotFound")) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) + expect(failures.list()).toEqual([]) + }) + + it("keeps a cleanup failure as a delete failure", async () => { + const { watcher, store, failures } = setup() + await watcher.processFile(fileA) + failAt("read", errorWithCode("FileNotFound")) + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) + expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "delete" })]) + }) + + it.each([ + ["stat", "NoPermissions"], + ["read", "EACCES"], + ["read", "Unknown"], + ] as const)("keeps denied access at %s (%s) as a read failure", async (stage, code) => { + const { watcher, failures } = setup() + failAt(stage, errorWithCode(code)) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity: "read", state: "exhausted" }), + ]) + }) + + it("keeps a not-found error from setup, outside stat and read, as a failure", async () => { + const { watcher, access, failures } = setup() + vi.mocked(access.validateAccess).mockImplementationOnce(() => { + throw errorWithCode("ENOENT") + }) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) + expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "read" })]) + }) + + it.each( + (["stat", "read"] as const).flatMap((stage) => + (["new", "indexed", "failed"] as const).flatMap((history) => + (["during", "after"] as const).map((timing) => ({ stage, history, timing })), + ), + ), + )( + "removes $history source missing at $stage $timing a concurrent provider wait", + async ({ stage, history, timing }) => { + const { watcher, points, hashes, store, embedder, failures, pacers } = setup() + const finished = vi.fn<(summary: BatchProcessingSummary) => void>() + watcher.onDidFinishBatchProcessing(finished) + await watcher.initialize() + if (history === "indexed") await queue() + if (history === "failed") { + points.set("obsolete", { id: "obsolete", vector: [0], payload: { filePath: "a.ts", generation: 0 } }) + failures.renew("a.ts", "old") + failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") + } + const metadata = deferred() + vi.mocked(vscode.workspace.fs.stat).mockImplementation(async (uri) => + uri.fsPath === fileA ? metadata.promise : fileStat(100), + ) + const readFile = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! + const missingRead = vi.fn() + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => { + if (uri.fsPath === fileA) { + missingRead() + throw errorWithCode("FileNotFound") + } + return readFile(uri) + }) + embedder.createEmbeddings + .mockClear() + .mockRejectedValueOnce(Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 })) + finished.mockClear() + await queue([fileA, fileB]) + expect(pacers.waitingPaths()).toContain("b.ts") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(missingRead).not.toHaveBeenCalled() + expect(finished).not.toHaveBeenCalled() + const resumeAt = pacers.providerWaits()[0].resumeAt! + await vi.advanceTimersByTimeAsync(resumeAt - Date.now() - (timing === "during" ? 1 : 0)) + if (stage === "stat") metadata.reject(errorWithCode("FileNotFound")) + else metadata.resolve(fileStat(100)) + await drain() + expect(missingRead).toHaveBeenCalledTimes(stage === "read" ? 1 : 0) + if (timing === "during") { + expect(pacers.waitingPaths()).toContain("b.ts") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(finished).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + } + await drain() + expect(finished).toHaveBeenCalledExactlyOnceWith({ + processedFiles: expect.arrayContaining([ + { + path: fileA, + status: history === "new" ? "skipped" : "success", + ...(history === "new" ? { reason: "File is not tracked" } : {}), + }, + { path: fileB, status: "success" }, + ]), + batchError: undefined, + }) + expect(pointsFor(points, fileA)).toEqual([]) + expect(hashes.get(fileA)).toBeUndefined() + expect(pointsFor(points, fileB)).toHaveLength(1) + expect(hashes.get(fileB)).toBe(hash("current content")) + expect(failures.list()).toEqual([]) + expect(pacers.waitingPaths()).toEqual([]) + expect(watcher.isProcessing).toBe(false) + if (history === "new") + expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) + else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) + }, + ) + + it("indexes a recreated file normally", async () => { + const { watcher, points, hashes } = setup() + await watcher.processFile(fileA) + failAt("read", errorWithCode("FileNotFound")) + await watcher.processFile(fileA) + expect(pointsFor(points, fileA)).toEqual([]) + expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) + expect(pointsFor(points, fileA)).not.toEqual([]) + expect(hashes.get(fileA)).toBe(hash("current content")) + }) +}) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 560b5c0568..5705460092 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -17,6 +17,7 @@ import { generateRelativeFilePath } from "../../shared/get-relative-path" import { PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, MAX_FILE_SIZE_BYTES } from "../../constants" import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" +import { ProviderPacerRegistry } from "../../provider-pacer" vi.mock("fs/promises", () => ({ stat: vi.fn() })) @@ -41,6 +42,8 @@ vi.mock("../../code-index-log", () => ({ redact: (message: string) => message, createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), info: vi.fn(), + debug: vi.fn(), + warn: vi.fn(), }, })) @@ -151,6 +154,7 @@ function setup() { const cacheManager = cache as unknown as CacheManager const generations = new GenerationAllocator(cacheManager) const ignored = ignore() + const pacers = new ProviderPacerRegistry() let currentAttempt = true const createScanner = () => new DirectoryScanner( @@ -165,6 +169,7 @@ function setup() { failures, workspacePath, () => currentAttempt, + pacers, ) return { cache, @@ -177,6 +182,7 @@ function setup() { failures, gate, ignored, + pacers, createScanner, scanner: createScanner(), supersede: () => { @@ -562,3 +568,225 @@ describe("DirectoryScanner", () => { expect(parser.parseFile.mock.calls.map(([filePath]) => filePath)).toEqual(selected) }) }) + +describe("T42 source missing at read is a removal", () => { + const contentHash = hash("current content") + const errorWithCode = (code: string) => Object.assign(new Error("gone"), { code }) + const failAt = (stage: "stat" | "read", error: Error) => { + if (stage === "stat") vi.mocked(stat).mockRejectedValueOnce(error) + else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(error) + } + const listPaths = (paths: string[]) => vi.mocked(listFiles).mockResolvedValue([paths, false]) + const filePoints = (points: Map, filePath: string) => + [...points.values()].filter((point) => { + const payload: Record = point.payload + return payload.filePath === generateRelativeFilePath(filePath, workspacePath) + }) + const seed = (points: Map, filePath: string) => + points.set(`seed:${filePath}`, { + id: `seed:${filePath}`, + vector: [0], + payload: { filePath: generateRelativeFilePath(filePath, workspacePath), generation: 0 }, + }) + + it.each( + (["stat", "read", "oversized"] as const).flatMap((source) => + (["new", "indexed", "failed"] as const).map((history) => ({ source, history })), + ), + )("preserves bookkeeping and logs for $history $source source", async ({ source, history }) => { + const { scanner, parser, cache, points, failures, store } = setup() + listPaths([fileA, fileB]) + if (history === "indexed") cache.completeFile(fileA, contentHash) + if (history !== "new") seed(points, fileA) + if (history === "failed") { + failures.renew("a.ts", "old") + failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") + } + if (source === "oversized") + vi.mocked(stat).mockImplementation(async (filePath) => + statsOfSize(filePath === fileA ? MAX_FILE_SIZE_BYTES + 1 : 100), + ) + else failAt(source, errorWithCode(source === "stat" ? "ENOENT" : "FileNotFound")) + const held = deferred() + parser.parseFile.mockReturnValue(held.promise) + const onError = vi.fn() + const pending = scanner.scanDirectory(workspacePath, onError) + await vi.advanceTimersByTimeAsync(0) + try { + expect(parser.parseFile).toHaveBeenCalledExactlyOnceWith(fileB, expect.anything()) + expect(scanner["incompleteFiles"]).toEqual(new Set(history === "new" ? [fileB] : [fileA, fileB])) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + if (source === "oversized") expect(codeIndexLog.debug).not.toHaveBeenCalled() + else + expect(codeIndexLog.debug).toHaveBeenCalledExactlyOnceWith( + "Source missing at read; treating as removed", + { path: "a.ts" }, + ) + expect(codeIndexLog.warn).not.toHaveBeenCalled() + const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value + expect(batch.record).not.toHaveBeenCalled() + } finally { + held.resolve(blocks(fileB)) + await pending + } + const result = await pending + expect(result.stats).toEqual({ processed: 1, skipped: source === "oversized" ? 1 : 0 }) + expect(result.incompleteFiles).toEqual([]) + expect(scanner.hasIncompleteFiles).toBe(false) + expect(filePoints(points, fileA)).toEqual([]) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getHash(fileB)).toBe(contentHash) + expect(onError).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + if (history === "new") expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) + else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) + }) + + it.each([ + ["stat", "ENOENT"], + ["read", "FileNotFound"], + ] as const)("never-indexed file missing at %s needs no recovery", async (stage, code) => { + const { scanner, cache, store, failures } = setup() + listPaths([fileA, fileB]) + failAt(stage, errorWithCode(code)) + const onError = vi.fn() + const result = await scanner.scanDirectory(workspacePath, onError) + expect(result.incompleteFiles).toEqual([]) + expect(result.stats).toEqual({ processed: 1, skipped: 0 }) + expect(cache.getHash(fileB)).toBe(contentHash) + expect(onError).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) + expect(failures.list()).toEqual([]) + }) + + it.each(["stat", "read"] as const)( + "previously indexed file missing at %s is removed in the same scan", + async (stage) => { + const { scanner, points, cache, failures } = setup() + await scanner.scanDirectory(workspacePath) + expect(filePoints(points, fileA)).toHaveLength(1) + failAt(stage, errorWithCode(stage === "stat" ? "ENOENT" : "FileNotFound")) + const onError = vi.fn() + const result = await scanner.scanDirectory(workspacePath, onError) + expect(result.incompleteFiles).toEqual([]) + expect(filePoints(points, fileA)).toEqual([]) + expect(cache.getHash(fileA)).toBeUndefined() + expect(onError).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }, + ) + + it.each( + (["stat", "read"] as const).flatMap((stage) => + (["new", "indexed", "failed"] as const).flatMap((history) => + (["during", "after"] as const).map((timing) => ({ stage, history, timing })), + ), + ), + )( + "removes $history source missing at $stage $timing a concurrent provider wait", + async ({ stage, history, timing }) => { + const { scanner, points, cache, store, embedder, failures, pacers } = setup() + if (history === "indexed") await scanner.scanDirectory(workspacePath) + if (history === "failed") { + seed(points, fileA) + failures.renew("a.ts", "old") + failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") + } + listPaths([fileA, fileB]) + const metadata = deferred() + vi.mocked(stat).mockImplementation(async (filePath) => + filePath === fileA ? metadata.promise : statsOfSize(100), + ) + const readFile = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! + const missingRead = vi.fn() + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => { + if (uri.fsPath === fileA) { + missingRead() + throw errorWithCode("FileNotFound") + } + return readFile(uri) + }) + embedder.createEmbeddings + .mockClear() + .mockRejectedValueOnce(Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 })) + const onError = vi.fn() + const pending = scanner.scanDirectory(workspacePath, onError) + await vi.advanceTimersByTimeAsync(0) + expect(pacers.waitingPaths()).toContain("b.ts") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(missingRead).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(timing === "during" ? 29_999 : 30_000) + if (stage === "stat") metadata.reject(errorWithCode("ENOENT")) + else metadata.resolve(statsOfSize(100)) + await vi.advanceTimersByTimeAsync(0) + expect(missingRead).toHaveBeenCalledTimes(stage === "read" ? 1 : 0) + if (timing === "during") { + expect(pacers.waitingPaths()).toContain("b.ts") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + } + const result = await pending + expect(result.stats).toEqual({ processed: 1, skipped: 0 }) + expect(result.incompleteFiles).toEqual([]) + expect(scanner.hasIncompleteFiles).toBe(false) + expect(filePoints(points, fileA)).toEqual([]) + expect(cache.getHash(fileA)).toBeUndefined() + expect(filePoints(points, fileB)).toHaveLength(1) + expect(cache.getHash(fileB)).toBe(contentHash) + expect(onError).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + expect(pacers.waitingPaths()).toEqual([]) + if (history === "new") + expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) + else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) + }, + ) + + it("clears a prior failure for a path found missing", async () => { + const { scanner, parser, failures } = setup() + parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) + await scanner.scanDirectory(workspacePath, vi.fn()) + expect(failures.list()).toHaveLength(1) + failAt("read", errorWithCode("FileNotFound")) + const result = await scanner.scanDirectory(workspacePath, vi.fn()) + expect(result.incompleteFiles).toEqual([]) + expect(failures.list()).toEqual([]) + }) + + it("keeps a cleanup failure as a delete failure, not a read failure", async () => { + const { scanner, store, failures } = setup() + await scanner.scanDirectory(workspacePath) + failAt("stat", errorWithCode("ENOENT")) + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + const result = await scanner.scanDirectory(workspacePath, vi.fn()) + expect(result.incompleteFiles).toEqual([fileA]) + expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "delete" })]) + }) + + it.each([ + ["stat", "EACCES"], + ["read", "NoPermissions"], + ["read", "EISDIR"], + ] as const)("keeps denied access at %s (%s) as a read failure", async (stage, code) => { + const { scanner, failures } = setup() + failAt(stage, errorWithCode(code)) + const onError = vi.fn() + const result = await scanner.scanDirectory(workspacePath, onError) + expect(result.incompleteFiles).toEqual([fileA]) + expect(onError).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity: "read", state: "exhausted" }), + ]) + }) + + it("indexes a recreated file normally on the next scan", async () => { + const { scanner, points, cache } = setup() + await scanner.scanDirectory(workspacePath) + failAt("read", errorWithCode("FileNotFound")) + await scanner.scanDirectory(workspacePath) + expect(filePoints(points, fileA)).toEqual([]) + await scanner.scanDirectory(workspacePath) + expect(filePoints(points, fileA)).toHaveLength(1) + expect(cache.getHash(fileA)).toBe(contentHash) + }) +}) diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index f5e71aecd5..83a6eb58b6 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -25,6 +25,7 @@ import { Package } from "../../../shared/package" import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" import { codeIndexLog } from "../code-index-log" import { ProviderPacerRegistry } from "../provider-pacer" +import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" type FileEvent = { uri: vscode.Uri; type: "create" | "change" | "delete" } @@ -269,6 +270,13 @@ export class FileWatcher implements IFileWatcher { } } + private removeMissingFile(filePath: string, signal: AbortSignal): Promise { + codeIndexLog.debug("Source missing at read; treating as removed", { + path: generateRelativeFilePath(filePath, this.workspacePath), + }) + return this.removeFile(filePath, signal) + } + async processFile(filePath: string): Promise { this.checkActive() this.activeFiles++ @@ -289,11 +297,13 @@ export class FileWatcher implements IFileWatcher { ) { return await this.removeFile(filePath, signal) } - const fileStat = await vscode.workspace.fs.stat(vscode.Uri.file(filePath)) + const fileStat = await readSource(vscode.workspace.fs.stat(vscode.Uri.file(filePath))) this.checkActive(signal) + if (fileStat === SOURCE_MISSING) return await this.removeMissingFile(filePath, signal) if (fileStat.size > MAX_FILE_SIZE_BYTES) return await this.removeFile(filePath, signal) - const fileContent = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) + const fileContent = await readSource(vscode.workspace.fs.readFile(vscode.Uri.file(filePath))) this.checkActive(signal) + if (fileContent === SOURCE_MISSING) return await this.removeMissingFile(filePath, signal) const content = Buffer.from(fileContent).toString("utf8") const newHash = createHash("sha256").update(content).digest("hex") revision = newHash diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index bac59bc96c..d749f256d8 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -26,6 +26,7 @@ import { FileIndexWriter } from "../file-completion" import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" import { codeIndexLog } from "../code-index-log" import { ProviderPacerRegistry } from "../provider-pacer" +import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number @@ -142,6 +143,15 @@ export class DirectoryScanner implements IDirectoryScanner { const relativePath = generateRelativeFilePath(filePath, scanWorkspace) onError?.(new Error(`${message} (File: ${relativePath})`, { cause: error })) } + const dropSelected = (filePath: string) => { + selectedPaths.delete(filePath) + if (!trackedPaths.has(filePath)) incompleteFiles.delete(filePath) + } + // A missing source falls into the removal phase; it is neither processed nor skipped. + const dropMissing = (filePath: string, relativePath: string) => { + codeIndexLog.debug("Source missing at read; treating as removed", { path: relativePath }) + dropSelected(filePath) + } const processFile = async (filePath: string) => { checkActive() const relativePath = generateRelativeFilePath(filePath, scanWorkspace) @@ -150,16 +160,17 @@ export class DirectoryScanner implements IDirectoryScanner { let activity: FileFailureActivity = "read" let fileSignal = signal try { - const stats = await stat(filePath) + const stats = await readSource(stat(filePath)) checkActive() + if (stats === SOURCE_MISSING) return dropMissing(filePath, relativePath) if (stats.size > MAX_FILE_SIZE_BYTES) { - selectedPaths.delete(filePath) - if (!trackedPaths.has(filePath)) incompleteFiles.delete(filePath) + dropSelected(filePath) skippedCount++ return } - const buffer = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) + const buffer = await readSource(vscode.workspace.fs.readFile(vscode.Uri.file(filePath))) checkActive() + if (buffer === SOURCE_MISSING) return dropMissing(filePath, relativePath) const content = Buffer.from(buffer).toString("utf-8") const hash = createHash("sha256").update(content).digest("hex") revision = hash diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts index d296f22c20..be323c14ba 100644 --- a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -6,12 +6,46 @@ import { EmbeddingError, VectorStoreError, parseRetryAfterMs, + isSourceMissing, + readSource, + SOURCE_MISSING, } from "../indexing-errors" import { formatEmbeddingError } from "../validation-helpers" import { IndexingRetryError } from "../indexing-errors" vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +describe("T44 isSourceMissing and readSource", () => { + const withCode = (code?: string) => Object.assign(new Error("x"), code ? { code } : {}) + + it.each([ + ["ENOENT", true], + ["FileNotFound", true], + ["EACCES", false], + ["EPERM", false], + ["NoPermissions", false], + ["Unknown", false], + ["EISDIR", false], + ["ENOTDIR", false], + [undefined, false], + ])("code %s -> %s", (code, expected) => { + expect(isSourceMissing(withCode(code))).toBe(expected) + }) + + it("ignores message text and non-error values", () => { + expect(isSourceMissing(new Error("ENOENT: no such file"))).toBe(false) + expect(isSourceMissing(undefined)).toBe(false) + expect(isSourceMissing("ENOENT")).toBe(false) + }) + + it("maps only a missing source to the sentinel and rethrows anything else unchanged", async () => { + expect(await readSource(Promise.resolve(5))).toBe(5) + expect(await readSource(Promise.reject(withCode("FileNotFound")))).toBe(SOURCE_MISSING) + const denied = withCode("EACCES") + await expect(readSource(Promise.reject(denied))).rejects.toBe(denied) + }) +}) + describe("indexing error classification", () => { it.each([ [{ status: 429 }, "rate-limit"], diff --git a/src/services/code-index/shared/indexing-errors.ts b/src/services/code-index/shared/indexing-errors.ts index c893f3fb95..d7a658ccfa 100644 --- a/src/services/code-index/shared/indexing-errors.ts +++ b/src/services/code-index/shared/indexing-errors.ts @@ -22,6 +22,24 @@ type ErrorShape = ErrorDetails & { type RetryAfterHeaders = Headers | Record +// ENOENT: Node fs/promises; FileNotFound: vscode.FileSystemError. Other codes stay read failures. +export function isSourceMissing(error: unknown): boolean { + const code = (error as ErrorShape | undefined)?.code + return code === "ENOENT" || code === "FileNotFound" +} + +export const SOURCE_MISSING = Symbol("source-missing") + +/** Applies missing-source detection only to the file's own stat or read call. */ +export async function readSource(read: PromiseLike): Promise { + try { + return await read + } catch (error) { + if (isSourceMissing(error)) return SOURCE_MISSING + throw error + } +} + function nonnegativeNumber(value: unknown): number | undefined { if (typeof value !== "number" && typeof value !== "string") return undefined if (typeof value === "string" && !value.trim()) return undefined From 5393fa765ee23fdf60de55485a1d1e1ea852150a Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 21:53:04 -0800 Subject: [PATCH 19/42] refactor(code-index): remove parse worker, log module and layout-only changes --- knip.json | 3 +- pnpm-lock.yaml | 17 + src/__tests__/extension.spec.ts | 6 - src/core/task/Task.ts | 8 - .../task/__tests__/Task.run-shadow.spec.ts | 30 -- src/core/task/__tests__/Task.spec.ts | 16 - src/esbuild.mjs | 2 +- src/package.json | 1 + src/scripts/verify-coverage-contract.mjs | 1 - .../__tests__/code-index-log.spec.ts | 179 --------- .../__tests__/fixtures/fault-loop.wasm | Bin 39 -> 0 bytes .../__tests__/fixtures/fault-loop.wat | 4 - .../__tests__/mock-code-index-log.ts | 7 - .../parse-worker-client.real.spec.ts | 159 -------- .../__tests__/parse-worker-client.spec.ts | 255 ------------ .../__tests__/parse-worker-imports.spec.ts | 56 --- .../__tests__/parser-errors.spec.ts | 28 -- .../code-index/code-index-log-host.ts | 8 - src/services/code-index/code-index-log.ts | 138 ------- src/services/code-index/interfaces/cache.ts | 12 - .../code-index/interfaces/file-processor.ts | 51 ++- .../code-index/parse-worker-client.ts | 189 --------- .../code-index/parse-worker-protocol.ts | 17 - src/services/code-index/parse-worker.ts | 39 -- src/services/code-index/parser-errors.ts | 22 -- src/services/code-index/parser-types.ts | 25 -- .../__tests__/file-preparation.spec.ts | 327 ++++++++++++++++ .../__tests__/parser-constants.spec.ts | 32 -- .../__tests__/parser.worker-contract.spec.ts | 368 ------------------ .../__tests__/processor-factories.spec.ts | 131 +++---- .../file-preparation-dependencies.ts | 15 + .../code-index/processors/file-preparation.ts | 100 +++++ src/services/code-index/processors/index.ts | 3 + .../code-index/processors/parser-constants.ts | 5 - .../__tests__/get-relative-path.spec.ts | 28 +- .../code-index/shared/get-relative-path.ts | 15 + .../code-index/shared/supported-extensions.ts | 26 +- src/vitest.dist.config.ts | 7 +- src/vitest.services.config.ts | 1 - src/vitest.unit.config.ts | 1 - src/workers/codeIndexParse.ts | 1 - 41 files changed, 626 insertions(+), 1707 deletions(-) delete mode 100644 src/services/code-index/__tests__/code-index-log.spec.ts delete mode 100644 src/services/code-index/__tests__/fixtures/fault-loop.wasm delete mode 100644 src/services/code-index/__tests__/fixtures/fault-loop.wat delete mode 100644 src/services/code-index/__tests__/mock-code-index-log.ts delete mode 100644 src/services/code-index/__tests__/parse-worker-client.real.spec.ts delete mode 100644 src/services/code-index/__tests__/parse-worker-client.spec.ts delete mode 100644 src/services/code-index/__tests__/parse-worker-imports.spec.ts delete mode 100644 src/services/code-index/__tests__/parser-errors.spec.ts delete mode 100644 src/services/code-index/code-index-log-host.ts delete mode 100644 src/services/code-index/code-index-log.ts delete mode 100644 src/services/code-index/parse-worker-client.ts delete mode 100644 src/services/code-index/parse-worker-protocol.ts delete mode 100644 src/services/code-index/parse-worker.ts delete mode 100644 src/services/code-index/parser-errors.ts delete mode 100644 src/services/code-index/parser-types.ts create mode 100644 src/services/code-index/processors/__tests__/file-preparation.spec.ts delete mode 100644 src/services/code-index/processors/__tests__/parser-constants.spec.ts delete mode 100644 src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts create mode 100644 src/services/code-index/processors/file-preparation-dependencies.ts create mode 100644 src/services/code-index/processors/file-preparation.ts create mode 100644 src/services/code-index/processors/index.ts delete mode 100644 src/services/code-index/processors/parser-constants.ts delete mode 100644 src/workers/codeIndexParse.ts diff --git a/knip.json b/knip.json index e0ab62eb9d..1f4956faa6 100644 --- a/knip.json +++ b/knip.json @@ -16,12 +16,13 @@ "playwright-ct": false, "workspaces": { "src": { - "entry": ["extension.ts!", "extension/api.ts!", "workers/countTokens.ts!", "workers/codeIndexParse.ts!"], + "entry": ["extension.ts!", "extension/api.ts!", "workers/countTokens.ts!"], "project": [ "**/*.ts!", "!**/__tests__/**!", "!vitest.setup.ts!", "!utils/vitest-verbosity.ts!", + "!utils/TaskSemaphore.ts!", "!utils/WorkspacePathResolver.ts!", "!test-utils/**!", "!utils/versionToken.ts!", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 15d0ac316f..6f1a2491f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -582,6 +582,9 @@ importers: os-name: specifier: ^6.0.0 version: 6.1.0 + p-limit: + specifier: ^6.2.0 + version: 6.2.0 p-wait-for: specifier: ^5.0.2 version: 5.0.2 @@ -7120,6 +7123,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -9045,6 +9052,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + yoctocolors@2.2.0: resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} engines: {node: '>=18'} @@ -16142,6 +16153,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.2 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -18235,6 +18250,8 @@ snapshots: yocto-queue@0.1.0: {} + yocto-queue@1.2.2: {} + yoctocolors@2.2.0: {} yoga-layout@3.2.1: {} diff --git a/src/__tests__/extension.spec.ts b/src/__tests__/extension.spec.ts index 28dba2649e..abcd3c28a0 100644 --- a/src/__tests__/extension.spec.ts +++ b/src/__tests__/extension.spec.ts @@ -388,9 +388,6 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, - failedFiles: [], - failedFileSummary: undefined, - providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() @@ -437,9 +434,6 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, - failedFiles: [], - failedFileSummary: undefined, - providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 950801a0c1..77a1333b39 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -402,12 +402,6 @@ export class Task extends EventEmitter implements TaskLike { // write was queued with the value at publish time, so a queued write cannot publish // after the task was cancelled or disposed. cancellationGeneration: number = 0 - private readonly lifecycleAbortController = new AbortController() - - public get lifecycleSignal(): AbortSignal { - return this.lifecycleAbortController.signal - } - currentRequestAbortController?: AbortController /** * Controller for the waiter on an in-flight `ensureModelFetched()` call (see @@ -3619,7 +3613,6 @@ export class Task extends EventEmitter implements TaskLike { // the resumed run's, and only if it advances monotonically. this.cancellationGeneration++ this.driveRunEvent({ tag: "abortRequested", abandoned: isAbandoned }) - this.lifecycleAbortController.abort() } requestAbandon(): void { @@ -3633,7 +3626,6 @@ export class Task extends EventEmitter implements TaskLike { // task is gone. this.cancellationGeneration++ this.driveRunEvent({ tag: "disposeRequested" }) - this.lifecycleAbortController.abort() } private driveRunEvent(event: RunEvent): void { diff --git a/src/core/task/__tests__/Task.run-shadow.spec.ts b/src/core/task/__tests__/Task.run-shadow.spec.ts index b57688b404..5ddc4261a8 100644 --- a/src/core/task/__tests__/Task.run-shadow.spec.ts +++ b/src/core/task/__tests__/Task.run-shadow.spec.ts @@ -97,36 +97,6 @@ describe("Task run-state write helpers", () => { expect(task.runState.stream).toEqual({ tag: "live", generation: 2, cleanupFinished: false }) }) - it.each(["abort", "abandon", "dispose"] as const)("keeps one lifetime signal after %s", (operation) => { - const signal = task.lifecycleSignal - const onAbort = vi.fn() - signal.addEventListener("abort", onAbort) - expect(signal.aborted).toBe(false) - expect(task.lifecycleSignal).toBe(signal) - - if (operation === "dispose") task.requestDispose() - else task.requestAbort(operation === "abandon") - - const reason: unknown = signal.reason - expect(reason).toBeInstanceOf(DOMException) - expect(reason).toHaveProperty("name", "AbortError") - expect(signal.aborted).toBe(true) - task.requestAbort(false) - task.requestDispose() - expect(task.lifecycleSignal).toBe(signal) - expect(task.lifecycleSignal.reason).toBe(reason) - expect(onAbort).toHaveBeenCalledOnce() - const resumed = makeTask() - expect(resumed.lifecycleSignal).not.toBe(signal) - expect(resumed.lifecycleSignal.aborted).toBe(false) - }) - - it("does not end the lifetime signal when cancelling an individual request", () => { - task.currentRequestAbortController = new AbortController() - task.cancelCurrentRequest() - expect(task.lifecycleSignal.aborted).toBe(false) - }) - it("requestAbort(false) sets abort only", () => { task.requestAbort(false) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 3f79bc5223..3fea8b61bc 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -4784,15 +4784,9 @@ describe("Cline", () => { }) vi.spyOn(task, "dispose").mockResolvedValue(undefined) - const lifetimeSignal = task.lifecycleSignal - expect(lifetimeSignal.aborted).toBe(false) expect(task.cancellationGeneration).toBe(0) await task.abortTask() - const lifetimeReason: unknown = lifetimeSignal.reason - expect(lifetimeSignal.aborted).toBe(true) - expect(lifetimeReason).toBeInstanceOf(DOMException) - expect(lifetimeReason).toHaveProperty("name", "AbortError") expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(1) @@ -4801,14 +4795,10 @@ describe("Cline", () => { // on resume: that is exactly why the generation exists. Only the // generation keeps the cancelled run and the resumed run apart. expect(task.cancellationGeneration).toBe(1) - expect(task.lifecycleSignal).toBe(lifetimeSignal) - expect(task.lifecycleSignal.aborted).toBe(true) - expect(task.lifecycleSignal.reason).toBe(lifetimeReason) await task.abortTask() expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(2) - expect(task.lifecycleSignal.reason).toBe(lifetimeReason) }) it("advances the cancellation generation on disposal, without any explicit cancel", async () => { @@ -4822,16 +4812,10 @@ describe("Cline", () => { startTask: false, }) - const lifetimeSignal = task.lifecycleSignal - expect(lifetimeSignal.aborted).toBe(false) expect(task.cancellationGeneration).toBe(0) await task.dispose() - expect(task.lifecycleSignal).toBe(lifetimeSignal) - expect(lifetimeSignal.aborted).toBe(true) - expect(lifetimeSignal.reason).toBeInstanceOf(DOMException) - expect(lifetimeSignal.reason).toHaveProperty("name", "AbortError") expect(task.abort).toBe(true) expect(task.cancellationGeneration).toBe(1) }) diff --git a/src/esbuild.mjs b/src/esbuild.mjs index c824ef9c2d..8159581f36 100644 --- a/src/esbuild.mjs +++ b/src/esbuild.mjs @@ -134,7 +134,7 @@ async function main() { */ const workerConfig = { ...buildOptions, - entryPoints: ["workers/countTokens.ts", "workers/codeIndexParse.ts"], + entryPoints: ["workers/countTokens.ts"], outdir: "dist/workers", } diff --git a/src/package.json b/src/package.json index 37f9dd2f12..98578ff92a 100644 --- a/src/package.json +++ b/src/package.json @@ -528,6 +528,7 @@ "ollama": "^0.6.0", "openai": "^5.12.2", "os-name": "^6.0.0", + "p-limit": "^6.2.0", "p-wait-for": "^5.0.2", "partial-json": "^0.1.7", "pdf-parse": "^1.1.1", diff --git a/src/scripts/verify-coverage-contract.mjs b/src/scripts/verify-coverage-contract.mjs index 91cedf90b6..985e514cb6 100644 --- a/src/scripts/verify-coverage-contract.mjs +++ b/src/scripts/verify-coverage-contract.mjs @@ -103,7 +103,6 @@ collectTests(root) const laneForTest = (file) => { if (file === "__tests__/dist_assets.spec.ts") return "dist" - if (file === "services/code-index/__tests__/parse-worker-client.real.spec.ts") return "dist" // Reads root-level config; runs outside Turbo in a dedicated CI step. if (file === "services/__tests__/pr-review-state-workflow.test.ts") return "workflow" if (file.startsWith("services/tree-sitter/")) return "tree-sitter" diff --git a/src/services/code-index/__tests__/code-index-log.spec.ts b/src/services/code-index/__tests__/code-index-log.spec.ts deleted file mode 100644 index 78466c37d6..0000000000 --- a/src/services/code-index/__tests__/code-index-log.spec.ts +++ /dev/null @@ -1,179 +0,0 @@ -const mocks = vi.hoisted(() => ({ - channel: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), dispose: vi.fn() }, - createOutputChannel: vi.fn(), - postMessage: vi.fn(), - thread: { isMainThread: true }, -})) -vi.mock("vscode", () => ({ window: { createOutputChannel: mocks.createOutputChannel } })) -vi.mock("node:worker_threads", () => ({ - get isMainThread() { - return mocks.thread.isMainThread - }, - parentPort: { postMessage: mocks.postMessage }, -})) - -import { codeIndexLog } from "../code-index-log" -import { initializeCodeIndexHostLogging } from "../code-index-log-host" -import { clearAllMocks } from "../../../test-utils/reset" - -beforeEach(() => { - codeIndexLog.dispose() - clearAllMocks() - mocks.thread.isMainThread = true - mocks.createOutputChannel.mockReturnValue(mocks.channel) - codeIndexLog.configure({ workspacePaths: [], secrets: [] }) - initializeCodeIndexHostLogging() -}) - -afterEach(() => codeIndexLog.dispose()) - -it("lazily uses one named log channel and prefixes every severity", () => { - expect(mocks.createOutputChannel).not.toHaveBeenCalled() - for (const level of ["info", "warn", "error", "debug"] as const) { - codeIndexLog[level]("message", { source: "scan", attempt: 2 }) - expect(mocks.channel[level]).toHaveBeenCalledWith('[CodeIndex] message {"source":"scan","attempt":2}') - } - expect(mocks.createOutputChannel).toHaveBeenCalledExactlyOnceWith("Zoo Code Indexing", { log: true }) -}) - -it("redacts configured secrets, headers, URL credentials and workspace roots in all fields", () => { - codeIndexLog.configure({ - workspacePaths: ["C:\\work\\project", "/home/user/project"], - secrets: ["private-key", "a+b"], - }) - codeIndexLog.error("https://user:password@host/test?api-key=private-key", { - path: "c:/work/project/src/file.ts", - other: "C:\\work\\project\\src\\other.ts", - error: new Error('/home/user/project/src/file.ts Authorization: Bearer private-key\napi-key="a+b"'), - nested: { Authorization: "secret", apiKey: "secret", content: "never log file contents" }, - }) - const output = mocks.channel.error.mock.calls[0][0] - for (const secret of [ - "user:password", - "private-key", - "a+b", - "home/user", - "work", - "never log file contents", - '"secret"', - ]) { - expect(output).not.toContain(secret) - } - expect(output).toContain("src/file.ts") - expect(output).toContain("[REDACTED]") -}) - -it("does not log SDK request bodies or arbitrary Error properties", () => { - const error = Object.assign(new Error("request failed"), { request: { body: "file contents" }, status: 503 }) - codeIndexLog.error("failed", error) - expect(mocks.channel.error.mock.calls[0][0]).toContain('"status":503') - expect(mocks.channel.error.mock.calls[0][0]).not.toContain("file contents") -}) - -it("keeps every retry and final failure in one warning per explicit flush", () => { - const batch = codeIndexLog.createFailureBatch({ source: "watcher", batch: 7 }) - batch.flush() - for (const attempt of [1, 2, 3]) - batch.record({ path: "file.ts", activity: "embed", reason: "503", attempt, willRetry: attempt < 3 }) - expect(mocks.channel.warn).not.toHaveBeenCalled() - batch.flush() - batch.flush() - expect(mocks.channel.warn).toHaveBeenCalledOnce() - const output = mocks.channel.warn.mock.calls[0][0] - const fields = JSON.parse(output.slice(output.indexOf("{"))) - expect(fields.failureCount).toBe(3) - expect(fields.failures.map((entry: { attempt: number }) => entry.attempt)).toEqual([1, 2, 3]) - expect(fields.failures[2].willRetry).toBe(false) - batch.record({ path: "other.ts", activity: "verify", reason: "timeout", attempt: 1, willRetry: true }) - batch.flush() - expect(mocks.channel.warn).toHaveBeenCalledTimes(2) -}) - -it.each(["info", "warn", "error", "debug"] as const)( - "forwards redacted worker %s diagnostics without invoking the host sink", - (level) => { - mocks.thread.isMainThread = false - codeIndexLog.configure({ workspacePaths: ["/workspace"], secrets: ["configured-secret"] }) - codeIndexLog[level]("api-key=hidden", { - path: "/workspace/file.ts", - reason: "configured-secret", - content: "file contents", - nested: { token: "hidden" }, - }) - expect(mocks.createOutputChannel).not.toHaveBeenCalled() - expect(mocks.channel[level]).not.toHaveBeenCalled() - expect(mocks.postMessage).toHaveBeenCalledExactlyOnceWith({ - id: 0, - diagnostic: { - level, - message: "api-key=[REDACTED]", - fields: { - path: "file.ts", - reason: "[REDACTED]", - content: "[REDACTED]", - nested: { token: "[REDACTED]" }, - }, - }, - }) - }, -) - -it("initializes idempotently and recreates the channel after disposal", () => { - codeIndexLog.info("first") - initializeCodeIndexHostLogging() - codeIndexLog.info("second") - expect(mocks.createOutputChannel).toHaveBeenCalledTimes(1) - expect(mocks.channel.dispose).not.toHaveBeenCalled() - codeIndexLog.dispose() - expect(mocks.channel.dispose).toHaveBeenCalledTimes(1) - codeIndexLog.info("third") - expect(mocks.createOutputChannel).toHaveBeenCalledTimes(2) -}) - -it("disposes the previous channel when replacing the host sink", () => { - codeIndexLog.info("first") - const replacement = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), dispose: vi.fn() } - codeIndexLog.setHostSinkFactory(() => replacement) - expect(mocks.channel.dispose).toHaveBeenCalledOnce() - codeIndexLog.info("second") - expect(replacement.info).toHaveBeenCalledExactlyOnceWith("[CodeIndex] second") -}) - -it("handles cycles and configuration replacement", () => { - const fields: Record = {} - fields.circular = fields - codeIndexLog.info("cycle", fields) - expect(mocks.channel.info.mock.calls[0][0]).toContain("[Circular]") - codeIndexLog.configure({ secrets: ["old"] }) - codeIndexLog.configure({ secrets: ["new"] }) - codeIndexLog.info("old new") - expect(mocks.channel.info).toHaveBeenLastCalledWith("[CodeIndex] old [REDACTED]") -}) - -it("matches a reference redactor and scales across a 1/2/4/8 input ladder", () => { - codeIndexLog.configure({ workspacePaths: ["/workspace/project"], secrets: ["configured-secret"] }) - const unit = - 'failed /workspace/project/src/file.ts https://user:password@host/path api-key="token" configured-secret\n' - const expectedUnit = 'failed src/file.ts https://[REDACTED]@host/path api-key="[REDACTED]" [REDACTED]\n' - let captured = "" - mocks.channel.debug.mockImplementation((message: string) => { - captured = message - }) - const samples: number[] = [] - for (const multiplier of [1, 2, 4, 8]) { - const input = unit.repeat(500 * multiplier) - const expected = `[CodeIndex] ${expectedUnit.repeat(500 * multiplier)}` - codeIndexLog.debug(input) - expect(captured).toBe(expected) - const timings: number[] = [] - for (let sampleNumber = 0; sampleNumber < 5; sampleNumber++) { - const started = performance.now() - for (let iteration = 0; iteration < 15; iteration++) codeIndexLog.debug(input) - timings.push(performance.now() - started) - } - samples.push(timings.sort((left, right) => left - right)[2]) - } - console.log(`codeIndexLog redaction ladder ms: ${samples.map((sample) => sample.toFixed(2)).join(" / ")}`) - expect(samples[2] / samples[0]).toBeLessThan(8) - expect(samples[3] / samples[0]).toBeLessThan(16) -}) diff --git a/src/services/code-index/__tests__/fixtures/fault-loop.wasm b/src/services/code-index/__tests__/fixtures/fault-loop.wasm deleted file mode 100644 index 146022e9e9b2bf6dadf01099b0967ec3e4f4257b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39 ucmZQbEY4+QU|?WmVN76PU}j=uVCP_D$;r ({ - codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, -})) -// Embedder failure paths report telemetry; an uninitialized singleton would turn a 429 into a permanent error. -vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { instance: { captureEvent: vi.fn() } }, -})) diff --git a/src/services/code-index/__tests__/parse-worker-client.real.spec.ts b/src/services/code-index/__tests__/parse-worker-client.real.spec.ts deleted file mode 100644 index 672886e773..0000000000 --- a/src/services/code-index/__tests__/parse-worker-client.real.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -vi.mock("../code-index-log", () => ({ - codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, -})) -import { copyFile, mkdtemp, readFile, readdir, rm } from "node:fs/promises" -import { tmpdir } from "node:os" -import * as path from "node:path" -import { Worker } from "node:worker_threads" -import { ParseWorkerClient, type ParseReply } from "../parse-worker-client" -import { INDEX_STOP_LIMIT_MS, PARSE_TIMEOUT_MS } from "../constants" - -const options = { - content: "function example() { return 'a sufficiently long function body for indexing'; }", - fileHash: "hash", -} -const distribution = path.resolve("dist") -const builtWorker = path.join(distribution, "workers", "codeIndexParse.js") - -function within(promise: Promise, milliseconds: number): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`Did not settle within ${milliseconds} ms`)), milliseconds) - promise.then( - (value) => { - clearTimeout(timer) - resolve(value) - }, - (error: unknown) => { - clearTimeout(timer) - reject(error) - }, - ) - }) -} - -async function probe(): Promise { - const started = performance.now() - await new Promise((resolve) => setTimeout(resolve, 20)) - const elapsed = performance.now() - started - expect(elapsed).toBeLessThan(500) - return elapsed -} - -describe("ParseWorkerClient built worker", () => { - let directory: string - const workers: Worker[] = [] - - beforeAll(async () => { - directory = await mkdtemp(path.join(tmpdir(), "zoo-parse-worker-")) - await copyFile(builtWorker, path.join(directory, "codeIndexParse.js")) - await copyFile( - path.resolve("services/code-index/__tests__/fixtures/fault-loop.wasm"), - path.join(directory, "fault-loop.wasm"), - ) - }) - - afterEach(async () => { - await Promise.all(workers.splice(0).map((worker) => worker.terminate())) - }) - - afterAll(async () => { - if (directory) await rm(directory, { recursive: true, force: true }) - }) - - function createWorker(fault?: string): Worker { - const environment = { ...process.env } - delete environment.ROO_PARSE_WORKER_FAULT - if (fault) environment.ROO_PARSE_WORKER_FAULT = fault - const worker = new Worker(fault ? path.join(directory, "codeIndexParse.js") : builtWorker, { - env: environment, - workerData: { wasmDirectory: distribution }, - }) - workers.push(worker) - return worker - } - - it("preserves both worker outputs without packaging the test-only fault fixture or VS Code imports", async () => { - const filenames = await readdir(path.join(distribution, "workers")) - expect(filenames).toContain("countTokens.js") - expect(filenames).toContain("codeIndexParse.js") - expect(filenames).not.toContain("fault-loop.wasm") - expect(await readdir(distribution)).not.toContain("fault-loop.wasm") - const bundle = await readFile(builtWorker, "utf8") - expect(bundle).not.toMatch(/require\(["']vscode["']\)/) - }) - - it("loads the runtime and TypeScript grammar from extension dist rather than the worker directory", async () => { - const client = new ParseWorkerClient(() => createWorker()) - try { - const blocks = await within(client.parseFile("normal.ts", options), 5000) - expect(blocks).toEqual([ - expect.objectContaining({ - content: options.content, - type: "function_declaration", - fileHash: options.fileHash, - }), - ]) - } finally { - client.stop() - await within(client.stopped, INDEX_STOP_LIMIT_MS) - } - }) - - it.each(["timeout", "stop"])( - "physically exits a real WASM hang on %s without blocking the main loop", - async (action) => { - const worker = createWorker("wasm-hang") - const entered = new Promise((resolve, reject) => { - worker.once("error", reject) - worker.on("message", (reply: ParseReply) => { - if (reply.diagnostic?.message === "parse worker entering wasm fault loop") resolve() - }) - }) - const client = new ParseWorkerClient(() => worker) - let exitObserved = false - worker.once("exit", () => { - exitObserved = true - }) - const pending = client.parseFile("hang.ts", options).catch((error: unknown) => error) - await within(entered, 5000) - const hangProbe = await probe() - expect(exitObserved).toBe(false) - let stoppingAt: number - if (action === "stop") { - stoppingAt = performance.now() - expect(client.stop()).toBeUndefined() - } else { - expect(await within(pending, PARSE_TIMEOUT_MS)).toMatchObject({ category: "parse-timeout" }) - stoppingAt = performance.now() - } - const stoppingProbe = probe() - await within(client.stopped, INDEX_STOP_LIMIT_MS) - const exitElapsed = performance.now() - stoppingAt - expect(exitElapsed).toBeLessThan(INDEX_STOP_LIMIT_MS) - expect(exitObserved).toBe(true) - expect(await pending).toMatchObject( - action === "stop" ? { name: "AbortError" } : { category: "parse-timeout" }, - ) - expect(await stoppingProbe).toBeLessThan(500) - expect(hangProbe).toBeLessThan(500) - }, - ) - - it("contains a fatal abort and permits explicit recovery using a new worker", async () => { - const failed = new ParseWorkerClient(() => createWorker("abort")) - await expect(failed.parseFile("abort.ts", options)).rejects.toMatchObject({ - name: "ParserFatalError", - category: "parser-abort", - message: expect.stringContaining("Aborted()"), - }) - await within(failed.stopped, INDEX_STOP_LIMIT_MS) - const replacement = new ParseWorkerClient(() => createWorker()) - try { - const blocks = await within(replacement.parseFile("normal.ts", options), 5000) - expect(blocks[0].content).toBe(options.content) - } finally { - replacement.stop() - await within(replacement.stopped, INDEX_STOP_LIMIT_MS) - } - }) -}) diff --git a/src/services/code-index/__tests__/parse-worker-client.spec.ts b/src/services/code-index/__tests__/parse-worker-client.spec.ts deleted file mode 100644 index 67484bd4d2..0000000000 --- a/src/services/code-index/__tests__/parse-worker-client.spec.ts +++ /dev/null @@ -1,255 +0,0 @@ -vi.mock("../code-index-log", () => ({ - codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, -})) -import { codeIndexLog } from "../code-index-log" -import { EventEmitter } from "node:events" -import { ParseWorkerClient, type ParseRequest, type ParseWorker } from "../parse-worker-client" - -class MockWorker extends EventEmitter { - requests: ParseRequest[] = [] - terminate = vi.fn().mockResolvedValue(0) - postMessage(request: ParseRequest): void { - this.requests.push(request) - } -} - -const options = { - content: "function example() { return 'a sufficiently long function body for indexing'; }", - fileHash: "hash", -} - -async function drain(): Promise { - for (let turn = 0; turn < 10; turn++) await Promise.resolve() -} - -describe("ParseWorkerClient", () => { - afterEach(() => vi.useRealTimers()) - - it("settles stopped immediately when stopped before creating a worker", async () => { - const factory = vi.fn<() => ParseWorker>() - const client = new ParseWorkerClient(factory) - expect(client.stop()).toBeUndefined() - await client.stopped - expect(factory).not.toHaveBeenCalled() - }) - - it.each(["resolved", "rejected", "pending"])( - "waits for physical exit with a %s termination promise and logs the parent's timeout once", - async (termination) => { - vi.useFakeTimers() - const log = vi.spyOn(codeIndexLog, "info").mockImplementation(() => {}) - try { - const worker = new MockWorker() - if (termination === "rejected") worker.terminate.mockRejectedValue(new Error("termination failed")) - if (termination === "pending") worker.terminate.mockReturnValue(new Promise(() => {})) - const client = new ParseWorkerClient(() => worker) - let exited = false - void client.stopped.then(() => { - exited = true - }) - const pending = client.parseFile("active.ts", options).catch((error: unknown) => error) - await drain() - expect(client.stop()).toBeUndefined() - expect(await pending).toMatchObject({ name: "AbortError" }) - await vi.advanceTimersByTimeAsync(2000) - expect(exited).toBe(false) - client.logStopTimeout() - client.logStopTimeout() - expect(log).toHaveBeenCalledExactlyOnceWith("parse worker exit not observed within stop limit") - worker.emit("exit", 1) - await client.stopped - expect(exited).toBe(true) - expect(log).toHaveBeenLastCalledWith(expect.stringMatching(/^parse worker exited after \d+ ms$/)) - client.logStopTimeout() - expect(log).toHaveBeenCalledTimes(2) - } finally { - log.mockRestore() - } - }, - ) - - it("observes an unexpected physical exit even when terminate never settles", async () => { - const worker = new MockWorker() - worker.terminate.mockReturnValue(new Promise(() => {})) - const client = new ParseWorkerClient(() => worker) - const pending = client.parseFile("active.ts", options).catch((error: unknown) => error) - await drain() - worker.emit("exit", 1) - expect(await pending).toMatchObject({ name: "ParserFatalError" }) - await client.stopped - expect(worker.terminate).not.toHaveBeenCalled() - }) - - it("forwards diagnostics without completing or interrupting the active parse", async () => { - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const pending = client.parseFile("active.ts", options) - await drain() - worker.emit("message", { - id: 0, - diagnostic: { - level: "warn", - message: "parser warning", - fields: { path: "active.ts" }, - }, - }) - expect(codeIndexLog.warn).toHaveBeenLastCalledWith("parser warning", { path: "active.ts" }) - expect(worker.terminate).not.toHaveBeenCalled() - worker.emit("message", { id: worker.requests[0].id, blocks: [] }) - await expect(pending).resolves.toEqual([]) - client.stop() - }) - - it("dispatches FIFO and starts each deadline at dispatch, not enqueue", async () => { - vi.useFakeTimers() - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker, 100) - const first = client.parseFile("first.ts", options) - const second = client.parseFile("second.ts", options) - await drain() - expect(worker.requests.map((request) => request.filePath)).toEqual(["first.ts"]) - await vi.advanceTimersByTimeAsync(90) - worker.emit("message", { id: worker.requests[0].id, blocks: [] }) - await first - await vi.advanceTimersByTimeAsync(90) - expect(worker.terminate).not.toHaveBeenCalled() - worker.emit("message", { id: worker.requests[1].id, blocks: [] }) - await second - client.stop() - }) - - it("uses the ten-second watchdog without granting a queued request its own failed-worker attempt", async () => { - vi.useFakeTimers() - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const pending = client.parseFile("hung.ts", options).catch((error: unknown) => error) - await drain() - await vi.advanceTimersByTimeAsync(9999) - expect(worker.terminate).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(await pending).toMatchObject({ category: "parse-timeout", filePath: "hung.ts" }) - expect(worker.terminate).toHaveBeenCalledOnce() - }) - - it("rejects an ordinary parse error without preventing the next file from completing", async () => { - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const failed = client.parseFile("bad.ts", options).catch((error: unknown) => error) - const next = client.parseFile("good.ts", options) - await drain() - worker.emit("message", { id: worker.requests[0].id, error: { message: "invalid source", fatal: false } }) - expect(await failed).toMatchObject({ message: "invalid source", name: "Error" }) - expect(worker.terminate).not.toHaveBeenCalled() - worker.emit("message", { id: worker.requests[1].id, blocks: [] }) - await expect(next).resolves.toEqual([]) - client.stop() - }) - - it("ignores mismatched replies and performs fallback chunking only when requested", async () => { - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const pending = client.parseFile("source.ts", options) - await drain() - worker.emit("message", { id: -1, error: { message: "Aborted()", fatal: true } }) - expect(worker.terminate).not.toHaveBeenCalled() - worker.emit("message", { id: worker.requests[0].id, fallback: true }) - await expect(pending).resolves.toEqual([ - expect.objectContaining({ file_path: "source.ts", content: options.content, fileHash: options.fileHash }), - ]) - client.stop() - }) - - it.each(["Aborted()", "memory access out of bounds", "error", "exit", "hang"])( - "contains %s and terminally rejects queued work", - async (failure) => { - vi.useFakeTimers() - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker, 100) - const active = client.parseFile("bad.ts", options).catch((error: unknown) => error) - const queued = client.parseFile("queued.ts", options).catch((error: unknown) => error) - await drain() - if (failure === "hang") await vi.advanceTimersByTimeAsync(100) - else if (failure === "error") worker.emit("error", new Error("crash")) - else if (failure === "exit") worker.emit("exit", 1) - else worker.emit("message", { id: worker.requests[0].id, error: { message: failure, fatal: false } }) - expect(await active).toMatchObject({ - name: "ParserFatalError", - filePath: "bad.ts", - category: failure === "hang" ? "parse-timeout" : "parser-abort", - }) - expect(await queued).toMatchObject({ name: "AbortError" }) - await expect(client.parseFile("later.md", options)).rejects.toMatchObject({ name: "AbortError" }) - expect(worker.requests).toHaveLength(1) - expect(worker.terminate).toHaveBeenCalledTimes(failure === "exit" ? 0 : 1) - }, - ) - - it("removes a cancelled queue entry without stopping another caller", async () => { - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const controller = new AbortController() - const first = client.parseFile("first.ts", options) - const cancelled = client - .parseFile("cancelled.ts", { ...options, signal: controller.signal }) - .catch((error: unknown) => error) - const last = client.parseFile("last.ts", options) - await drain() - controller.abort() - expect(await cancelled).toMatchObject({ name: "AbortError" }) - worker.emit("message", { id: worker.requests[0].id, blocks: [] }) - await first - expect(worker.requests[1].filePath).toBe("last.ts") - worker.emit("message", { id: worker.requests[1].id, blocks: [] }) - await last - client.stop() - }) - - it.each(["stop", "abort"])( - "%s synchronously settles active and queued requests without waiting for termination", - async (action) => { - const worker = new MockWorker() - worker.terminate.mockReturnValue(new Promise(() => {})) - const client = new ParseWorkerClient(() => worker) - const controller = new AbortController() - const active = client - .parseFile("active.ts", { ...options, signal: controller.signal }) - .catch((error: unknown) => error) - const queued = client.parseFile("queued.ts", options).catch((error: unknown) => error) - await drain() - if (action === "stop") client.stop() - else controller.abort() - expect(await active).toMatchObject({ name: "AbortError" }) - expect(await queued).toMatchObject({ name: "AbortError" }) - client.stop() - expect(worker.terminate).toHaveBeenCalledTimes(1) - }, - ) - - it.each(["error", "exit"])("reports an idle worker %s as a parser failure on the next parse", async (failure) => { - const worker = new MockWorker() - const client = new ParseWorkerClient(() => worker) - const first = client.parseFile("first.ts", options) - await drain() - worker.emit("message", { id: worker.requests[0].id, blocks: [] }) - await first - if (failure === "error") worker.emit("error", new Error("crash")) - else worker.emit("exit", 1) - await expect(client.parseFile("next.ts", options)).rejects.toMatchObject({ name: "ParserFatalError" }) - }) - - it("keeps independent workspaces isolated and a replacement usable", async () => { - const failedWorker = new MockWorker() - const otherWorker = new MockWorker() - const failed = new ParseWorkerClient(() => failedWorker) - const other = new ParseWorkerClient(() => otherWorker) - const failure = failed.parseFile("bad.ts", options).catch((error: unknown) => error) - const success = other.parseFile("good.ts", options) - await drain() - failedWorker.emit("exit", 1) - await failure - otherWorker.emit("message", { id: otherWorker.requests[0].id, blocks: [] }) - await expect(success).resolves.toEqual([]) - expect(otherWorker.terminate).not.toHaveBeenCalled() - other.stop() - }) -}) diff --git a/src/services/code-index/__tests__/parse-worker-imports.spec.ts b/src/services/code-index/__tests__/parse-worker-imports.spec.ts deleted file mode 100644 index 7f8ed47057..0000000000 --- a/src/services/code-index/__tests__/parse-worker-imports.spec.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { existsSync, readFileSync } from "node:fs" -import * as path from "node:path" -import * as ts from "typescript" - -function collectWorkerImports(entryPath: string) { - const visited = new Set() - const imports: string[] = [] - const pending = [entryPath] - while (pending.length > 0) { - const filePath = pending.pop()! - if (visited.has(filePath)) continue - visited.add(filePath) - const source = ts.createSourceFile(filePath, readFileSync(filePath, "utf8"), ts.ScriptTarget.Latest, true) - function visit(node: ts.Node): void { - let modulePath: string | undefined - if ((ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) && node.moduleSpecifier) { - if (ts.isStringLiteral(node.moduleSpecifier)) modulePath = node.moduleSpecifier.text - } else if (ts.isImportTypeNode(node) && ts.isLiteralTypeNode(node.argument)) { - if (ts.isStringLiteral(node.argument.literal)) modulePath = node.argument.literal.text - } else if ( - ts.isCallExpression(node) && - (node.expression.kind === ts.SyntaxKind.ImportKeyword || - (ts.isIdentifier(node.expression) && node.expression.text === "require")) - ) { - const argument = node.arguments[0] - if (argument && ts.isStringLiteral(argument)) modulePath = argument.text - } - if (ts.isIdentifier(node)) expect(node.text).not.toBe("createRequire") - if (modulePath !== undefined) { - imports.push(modulePath) - if (modulePath.startsWith(".")) { - const resolved = path.resolve(path.dirname(filePath), modulePath) - const dependency = [resolved, `${resolved}.ts`, path.join(resolved, "index.ts")].find( - (candidate) => candidate.endsWith(".ts") && existsSync(candidate), - ) - expect(dependency, `${filePath}: ${modulePath}`).toBeDefined() - if (dependency) pending.push(dependency) - } - } - ts.forEachChild(node, visit) - } - visit(source) - } - return { visited, imports } -} - -it("keeps host imports out of the entire worker source graph, including type-only and dynamic imports", () => { - const { visited, imports } = collectWorkerImports(path.resolve("workers/codeIndexParse.ts")) - expect(imports.filter((modulePath) => /^(?:vscode(?:\/|$)|@roo-code\/)/.test(modulePath))).toEqual([]) - const files = [...visited].map((filePath) => filePath.replaceAll("\\", "/")) - expect(files.some((filePath) => filePath.endsWith("/code-index-log.ts"))).toBe(true) - expect(files.some((filePath) => filePath.endsWith("/parse-worker-protocol.ts"))).toBe(true) - expect( - files.filter((filePath) => /\/(?:code-index-log-host|parse-worker-client|manager)\.ts$/.test(filePath)), - ).toEqual([]) -}) diff --git a/src/services/code-index/__tests__/parser-errors.spec.ts b/src/services/code-index/__tests__/parser-errors.spec.ts deleted file mode 100644 index 11d3a6b6c9..0000000000 --- a/src/services/code-index/__tests__/parser-errors.spec.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { abortError, isParserAbort, ParserFatalError, throwIfAborted } from "../parser-errors" - -describe("parser errors", () => { - it.each(["parser-abort", "parse-timeout"] as const)("preserves the %s category and file", (category) => { - const error = new ParserFatalError(category, "source.ts") - expect(error).toBeInstanceOf(Error) - expect(error).toMatchObject({ name: "ParserFatalError", category, filePath: "source.ts", message: category }) - expect(new ParserFatalError(category, "source.ts", "detail").message).toBe("detail") - }) - - it.each([ - ["Aborted()", true], - [new Error("MEMORY ACCESS OUT OF BOUNDS"), true], - [new Error("ordinary parse failure"), false], - [undefined, false], - ])("classifies %s without treating ordinary failures as fatal", (error, expected) => { - expect(isParserAbort(error)).toBe(expected) - }) - - it("only throws a cancellation for an aborted signal", () => { - expect(abortError()).toMatchObject({ name: "AbortError", message: "Indexing aborted" }) - expect(() => throwIfAborted()).not.toThrow() - const controller = new AbortController() - expect(() => throwIfAborted(controller.signal)).not.toThrow() - controller.abort() - expect(() => throwIfAborted(controller.signal)).toThrow("Indexing aborted") - }) -}) diff --git a/src/services/code-index/code-index-log-host.ts b/src/services/code-index/code-index-log-host.ts deleted file mode 100644 index 3bab4a12f3..0000000000 --- a/src/services/code-index/code-index-log-host.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { window } from "vscode" -import { codeIndexLog, type CodeIndexLogSink } from "./code-index-log" - -const createHostSink = (): CodeIndexLogSink => window.createOutputChannel("Zoo Code Indexing", { log: true }) - -export function initializeCodeIndexHostLogging(): void { - codeIndexLog.setHostSinkFactory(createHostSink) -} diff --git a/src/services/code-index/code-index-log.ts b/src/services/code-index/code-index-log.ts deleted file mode 100644 index 0a66d8ea54..0000000000 --- a/src/services/code-index/code-index-log.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { isMainThread, parentPort } from "node:worker_threads" - -export type CodeIndexLogLevel = "info" | "warn" | "error" | "debug" -export interface CodeIndexLogConfiguration { - workspacePaths?: readonly string[] - secrets?: readonly string[] -} -export interface CodeIndexFailureDiagnostic { - path: string - activity: string - reason: string - attempt: number - willRetry: boolean - [key: string]: unknown -} - -export type CodeIndexLogSink = Record void> & { dispose(): void } - -const redacted = "[REDACTED]" -const sensitiveField = /^(?:authorization|.*api[-_]?key|.*secret|.*password|.*token)$/i -const contentField = - /^(?:content|contents|text|texts|input|inputs|body|request|response|data|embedding|embeddings|vectors)$/i -const escapePattern = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") -let workspacePattern: RegExp | undefined -let secretPattern: RegExp | undefined -let channel: CodeIndexLogSink | undefined -let hostSinkFactory: (() => CodeIndexLogSink) | undefined - -function configure(configuration: CodeIndexLogConfiguration): void { - if (configuration.workspacePaths !== undefined) { - const roots = configuration.workspacePaths - .filter(Boolean) - .map((root) => root.replace(/[\\/]+$/, "")) - .filter(Boolean) - .sort((left, right) => right.length - left.length) - workspacePattern = roots.length - ? new RegExp( - `(?:${roots.map((root) => root.split(/[\\/]/).map(escapePattern).join("[\\\\/]")).join("|")})(?:[\\\\/]|(?=$|[\\s"',:)]))`, - "gi", - ) - : undefined - } - if (configuration.secrets !== undefined) { - const secrets = [...new Set(configuration.secrets.filter(Boolean))].sort( - (left, right) => right.length - left.length, - ) - secretPattern = secrets.length ? new RegExp(secrets.map(escapePattern).join("|"), "g") : undefined - } -} - -function redact(value: string): string { - let result = value.replace(/\b([a-z][a-z\d+.-]*:\/\/)[^\s/?#@]*@/gi, `$1${redacted}@`) - result = result.replace(/\b(authorization|api[-_]?key)(["']?\s*[:=]\s*["']?)[^\r\n"',;}]+/gi, `$1$2${redacted}`) - if (workspacePattern) result = result.replace(workspacePattern, "") - if (secretPattern) result = result.replace(secretPattern, redacted) - return result -} - -function sanitize(value: unknown, seen = new WeakSet()): unknown { - if (typeof value === "string") return redact(value) - if (value === null || typeof value !== "object") return value - if (seen.has(value)) return "[Circular]" - seen.add(value) - if (value instanceof Error) { - // SDK errors can hold request bodies and credentials outside the message. - const error = value as Error & { - code?: unknown - status?: unknown - statusCode?: unknown - classification?: unknown - } - return sanitize( - { - name: error.name, - message: error.message, - code: error.code, - status: error.status, - statusCode: error.statusCode, - classification: error.classification, - }, - seen, - ) - } - if (Array.isArray(value)) return value.map((entry) => sanitize(entry, seen)) - return Object.fromEntries( - Object.entries(value).map(([key, entry]) => [ - redact(key), - sensitiveField.test(key) || contentField.test(key) ? redacted : sanitize(entry, seen), - ]), - ) -} - -function write(level: CodeIndexLogLevel, message: string, fields?: unknown): void { - const safeMessage = redact(message) - const safeFields = sanitize(fields) - if (!isMainThread) { - parentPort?.postMessage({ id: 0, diagnostic: { level, message: safeMessage, fields: safeFields } }) - return - } - channel ??= hostSinkFactory?.() - if (!channel) return - const suffix = safeFields === undefined ? "" : ` ${JSON.stringify(safeFields)}` - channel[level](`[CodeIndex] ${safeMessage}${suffix}`) -} - -function createFailureBatch(fields?: Record) { - let failures: CodeIndexFailureDiagnostic[] = [] - return { - record(failure: CodeIndexFailureDiagnostic): void { - failures.push({ ...failure }) - }, - flush(): void { - if (failures.length === 0) return - const completed = failures - failures = [] - write("warn", "File operation failures", { ...fields, failureCount: completed.length, failures: completed }) - }, - } -} - -export const codeIndexLog = { - configure, - setHostSinkFactory(factory: () => CodeIndexLogSink): void { - if (hostSinkFactory === factory) return - codeIndexLog.dispose() - hostSinkFactory = factory - }, - redact, - createFailureBatch, - info: (message: string, fields?: unknown) => write("info", message, fields), - warn: (message: string, fields?: unknown) => write("warn", message, fields), - error: (message: string, fields?: unknown) => write("error", message, fields), - debug: (message: string, fields?: unknown) => write("debug", message, fields), - dispose(): void { - channel?.dispose() - channel = undefined - }, -} diff --git a/src/services/code-index/interfaces/cache.ts b/src/services/code-index/interfaces/cache.ts index 5bfaab5b50..01931a3a8b 100644 --- a/src/services/code-index/interfaces/cache.ts +++ b/src/services/code-index/interfaces/cache.ts @@ -4,16 +4,4 @@ export interface ICacheManager { deleteHash(filePath: string): void flush(): Promise getAllHashes(): Record - isTracked(filePath: string): boolean - getOpenFiles(): Record - getTrackedPaths(): string[] - invalidateFile(filePath: string): Promise - completeFile(filePath: string, hash: string): void - completeRemoval(filePath: string): void - saveNow(): Promise - clearCacheFile(): Promise - readonly loadSucceeded: boolean - readonly loadedAt: number - readonly generationWatermark: number | undefined - reserveGenerations(watermark: number): Promise } diff --git a/src/services/code-index/interfaces/file-processor.ts b/src/services/code-index/interfaces/file-processor.ts index 81fc92902d..96ef3fd962 100644 --- a/src/services/code-index/interfaces/file-processor.ts +++ b/src/services/code-index/interfaces/file-processor.ts @@ -1,15 +1,31 @@ -import type * as vscode from "vscode" -import type { PointStruct } from "./vector-store" -import type { ParserFatalError } from "../parser-errors" +import * as vscode from "vscode" +import type { FileFailureActivity } from "../file-failure-registry" -export type { ICodeParser, CodeBlock } from "../parser-types" +/** + * Interface for code file parser + */ +export interface ICodeParser { + /** + * Parses a code file into code blocks + * @param filePath Path to the file to parse + * @param options Optional parsing options + * @returns Promise resolving to array of code blocks + */ + parseFile( + filePath: string, + options?: { + minBlockLines?: number + maxBlockLines?: number + content?: string + fileHash?: string + }, + ): Promise +} /** * Interface for directory scanner */ export interface IDirectoryScanner { - readonly hasIncompleteFiles?: boolean - /** * Scans a directory for code blocks * @param directoryPath Path to the directory to scan @@ -28,7 +44,6 @@ export interface IDirectoryScanner { skipped: number } totalBlockCount: number - incompleteFiles?: string[] }> } @@ -40,8 +55,6 @@ export interface IFileWatcher extends vscode.Disposable { * Initializes the file watcher */ initialize(signal?: AbortSignal): Promise - readonly onParserFatal?: vscode.Event - readonly isProcessing?: boolean /** * Event emitted when a batch of files begins processing. @@ -81,9 +94,25 @@ export interface BatchProcessingSummary { export interface FileProcessingResult { path: string - status: "success" | "skipped" | "error" | "processed_for_batching" | "local_error" + status: "success" | "skipped" | "error" | "processed_for_batching" | "local_error" | "missing" error?: Error reason?: string newHash?: string - pointsToUpsert?: PointStruct[] + blocks?: CodeBlock[] + activity?: FileFailureActivity +} + +/** + * Common types used across the code-index service + */ + +export interface CodeBlock { + file_path: string + identifier: string | null + type: string + start_line: number + end_line: number + content: string + fileHash: string + segmentHash: string } diff --git a/src/services/code-index/parse-worker-client.ts b/src/services/code-index/parse-worker-client.ts deleted file mode 100644 index 4a9d129808..0000000000 --- a/src/services/code-index/parse-worker-client.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { codeIndexLog } from "./code-index-log" -import { Worker } from "node:worker_threads" -import * as path from "node:path" -import { CodeParser } from "./processors/parser" -import type { CodeBlock, ICodeParser } from "./parser-types" -import type { ParseRequest, ParseReply } from "./parse-worker-protocol" -import { abortError, ParserFatalError, isParserAbort } from "./parser-errors" -import { PARSE_TIMEOUT_MS } from "./constants" - -export type { ParseRequest, ParseReply } from "./parse-worker-protocol" - -export interface ParseWorker { - postMessage(request: ParseRequest): void - on(event: "message", listener: (reply: ParseReply) => void): unknown - on(event: "error", listener: (error: Error) => void): unknown - on(event: "exit", listener: (code: number) => void): unknown - terminate(): Promise -} - -interface PendingParse { - request: ParseRequest - resolve: (blocks: CodeBlock[]) => void - reject: (error: Error) => void - signal?: AbortSignal - cancel: () => void -} - -export class ParseWorkerClient implements ICodeParser { - private worker?: ParseWorker - private queue: PendingParse[] = [] - private active?: PendingParse - private timer?: NodeJS.Timeout - private nextId = 0 - private isStopped = false - private resolveStopped!: () => void - readonly stopped = new Promise((resolve) => { - this.resolveStopped = resolve - }) - private stopStartedAt?: number - private exitObserved = false - private stopTimeoutLogged = false - private readonly parser: CodeParser - private idleFailure?: Pick - - constructor( - private readonly workerFactory: () => ParseWorker = () => - new Worker(workerPath, { workerData: { wasmDirectory: __dirname } }), - private readonly timeoutMs = PARSE_TIMEOUT_MS, - workerPath = path.join(__dirname, "workers", "codeIndexParse.js"), - ) { - this.parser = new CodeParser((filePath, content, fileHash, signal) => - this.enqueue(filePath, content, fileHash, signal), - ) - } - - parseFile( - filePath: string, - options?: { content?: string; fileHash?: string; signal?: AbortSignal }, - ): Promise { - const stopError = this.getStopError(filePath, options?.signal) - if (stopError) return Promise.reject(stopError) - return this.parser.parseFile(filePath, options) - } - - private getStopError(filePath: string, signal?: AbortSignal): Error | undefined { - if (signal?.aborted) return abortError() - if (this.idleFailure) { - const { category, message } = this.idleFailure - this.idleFailure = undefined - return new ParserFatalError(category, filePath, message) - } - if (this.isStopped) return abortError() - return undefined - } - - private enqueue(filePath: string, content: string, fileHash: string, signal?: AbortSignal): Promise { - const stopError = this.getStopError(filePath, signal) - if (stopError) return Promise.reject(stopError) - return new Promise((resolve, reject) => { - const pending: PendingParse = { - request: { id: ++this.nextId, filePath, content, fileHash }, - resolve, - reject, - signal, - cancel: () => { - if (this.active === pending) this.stop() - else { - this.queue = this.queue.filter((entry) => entry !== pending) - this.settle(pending, abortError()) - } - }, - } - signal?.addEventListener("abort", pending.cancel, { once: true }) - this.queue.push(pending) - this.dispatch() - }) - } - - private settle(pending: PendingParse, error?: Error, blocks: CodeBlock[] = []): void { - pending.signal?.removeEventListener("abort", pending.cancel) - if (error) pending.reject(error) - else pending.resolve(blocks) - } - - private dispatch(): void { - if (this.isStopped || this.active || this.queue.length === 0) return - this.active = this.queue.shift()! - try { - if (!this.worker) { - this.worker = this.workerFactory() - this.worker.on("message", (reply) => { - if (reply.diagnostic) { - const { level, message, fields } = reply.diagnostic - codeIndexLog[level](message, fields) - return - } - if (this.isStopped || reply.id !== this.active?.request.id) return - if (reply.error?.fatal || (reply.error && isParserAbort(reply.error.message))) { - this.fail("parser-abort", reply.error?.message) - return - } - clearTimeout(this.timer) - const completed = this.active - this.active = undefined - const blocks = !reply.fallback - ? reply.blocks - : this.parser.fallbackContent( - completed.request.filePath, - completed.request.content, - completed.request.fileHash, - ) - this.settle(completed, reply.error ? new Error(reply.error.message) : undefined, blocks) - this.dispatch() - }) - this.worker.on("error", (error) => this.fail("parser-abort", error.message)) - this.worker.on("exit", (code) => { - this.exitObserved = true - this.fail("parser-abort", `Parse worker exited (${code})`) - codeIndexLog.info( - `parse worker exited after ${Math.round(performance.now() - this.stopStartedAt!)} ms`, - ) - this.worker = undefined - this.resolveStopped() - }) - } - this.timer = setTimeout(() => this.fail("parse-timeout"), this.timeoutMs) - this.worker.postMessage(this.active.request) - } catch (error) { - this.fail("parser-abort", String(error)) - } - } - - private fail(category: ParserFatalError["category"], message?: string): void { - if (this.isStopped) return - const failed = this.active - this.active = undefined - if (failed) this.settle(failed, new ParserFatalError(category, failed.request.filePath, message)) - else this.idleFailure = { category, message: message ?? category } - this.stop() - } - - stop(): void { - if (this.isStopped) return - this.isStopped = true - this.stopStartedAt = performance.now() - clearTimeout(this.timer) - if (this.active) this.settle(this.active, abortError()) - this.active = undefined - for (const pending of this.queue) this.settle(pending, abortError()) - this.queue = [] - if (!this.worker) { - this.exitObserved = true - this.resolveStopped() - } else if (!this.exitObserved) { - // Termination acknowledgement alone is not evidence of physical exit. - void this.worker.terminate().catch(() => undefined) - } - } - - logStopTimeout(): void { - if (!this.isStopped || this.exitObserved || this.stopTimeoutLogged) return - this.stopTimeoutLogged = true - codeIndexLog.info("parse worker exit not observed within stop limit") - } - - dispose(): void { - this.stop() - } -} diff --git a/src/services/code-index/parse-worker-protocol.ts b/src/services/code-index/parse-worker-protocol.ts deleted file mode 100644 index aa795e9f75..0000000000 --- a/src/services/code-index/parse-worker-protocol.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { CodeIndexLogLevel } from "./code-index-log" -import type { CodeBlock } from "./parser-types" - -export interface ParseRequest { - id: number - filePath: string - content: string - fileHash: string -} - -export interface ParseReply { - id: number - blocks?: CodeBlock[] - fallback?: boolean - error?: { message: string; fatal: boolean } - diagnostic?: { level: CodeIndexLogLevel; message: string; fields?: unknown } -} diff --git a/src/services/code-index/parse-worker.ts b/src/services/code-index/parse-worker.ts deleted file mode 100644 index f6e7d96f0c..0000000000 --- a/src/services/code-index/parse-worker.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { codeIndexLog } from "./code-index-log" -import { parentPort, workerData } from "node:worker_threads" -import { readFile } from "node:fs/promises" -import * as path from "node:path" -import { CodeParser } from "./processors/parser" -import { isParserAbort } from "./parser-errors" -import type { ParseRequest, ParseReply } from "./parse-worker-protocol" - -let fault = process.env.ROO_PARSE_WORKER_FAULT -const { wasmDirectory } = workerData as { wasmDirectory: string } -const parser = new CodeParser(undefined, wasmDirectory) -parentPort!.on("message", async (request: ParseRequest) => { - let reply: ParseReply - try { - const nextFault = fault - fault = undefined - if (nextFault === "abort") throw new Error("Aborted() (injected parse worker fault)") - if (nextFault === "wasm-hang") { - const bytes = new Uint8Array(await readFile(path.join(__dirname, "fault-loop.wasm"))) - const { instance } = await WebAssembly.instantiate(bytes) - codeIndexLog.info("parse worker entering wasm fault loop") - ;(instance.exports.loop as () => void)() - } - // MUST use require: an ESM import bundles web-tree-sitter's ESM build, whose import.meta.url is undefined in CJS. - const { Parser } = require("web-tree-sitter") as typeof import("web-tree-sitter") - await Parser.init({ locateFile: (filename: string) => path.join(wasmDirectory, filename) }) - const blocks = await parser.parseFile(request.filePath, { - content: request.content, - fileHash: request.fileHash, - }) - reply = { id: request.id, blocks, fallback: parser.fallbackNeeded } - } catch (error) { - reply = { - id: request.id, - error: { message: error instanceof Error ? error.message : String(error), fatal: isParserAbort(error) }, - } - } - parentPort!.postMessage(reply) -}) diff --git a/src/services/code-index/parser-errors.ts b/src/services/code-index/parser-errors.ts deleted file mode 100644 index 74d454242e..0000000000 --- a/src/services/code-index/parser-errors.ts +++ /dev/null @@ -1,22 +0,0 @@ -export class ParserFatalError extends Error { - constructor( - public readonly category: "parser-abort" | "parse-timeout", - public readonly filePath: string, - message?: string, - ) { - super(message ?? category) - this.name = "ParserFatalError" - } -} - -export function abortError(): DOMException { - return new DOMException("Indexing aborted", "AbortError") -} - -export function throwIfAborted(signal?: AbortSignal): void { - if (signal?.aborted) throw abortError() -} - -export function isParserAbort(error: unknown): boolean { - return /Aborted\(|memory access out of bounds/i.test(error instanceof Error ? error.message : String(error)) -} diff --git a/src/services/code-index/parser-types.ts b/src/services/code-index/parser-types.ts deleted file mode 100644 index 25c5062b25..0000000000 --- a/src/services/code-index/parser-types.ts +++ /dev/null @@ -1,25 +0,0 @@ -export interface ICodeParser { - readonly fallbackNeeded?: boolean - fallbackContent?(filePath: string, content: string, fileHash: string): CodeBlock[] - parseFile( - filePath: string, - options?: { - minBlockLines?: number - maxBlockLines?: number - content?: string - fileHash?: string - signal?: AbortSignal - }, - ): Promise -} - -export interface CodeBlock { - file_path: string - identifier: string | null - type: string - start_line: number - end_line: number - content: string - fileHash: string - segmentHash: string -} diff --git a/src/services/code-index/processors/__tests__/file-preparation.spec.ts b/src/services/code-index/processors/__tests__/file-preparation.spec.ts new file mode 100644 index 0000000000..22020a9fdd --- /dev/null +++ b/src/services/code-index/processors/__tests__/file-preparation.spec.ts @@ -0,0 +1,327 @@ +import { createHash } from "crypto" +import path from "path" +import { FileType, Uri } from "vscode" +import { v5 as uuidv5 } from "uuid" +import type { CodeBlock, ICodeParser, IEmbedder } from "../../interfaces" +import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../../constants" +import { FilePreparation } from "../file-preparation" +import type { FilePreparationDependencies } from "../file-preparation-dependencies" + +// Keep path spies local to the modules under test, not Node or the test runner. +vi.mock("path", async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, default: { ...actual } } +}) + +describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platform) => { + beforeEach(() => { + vi.spyOn(path, "resolve").mockImplementation(path[platform].resolve) + vi.spyOn(path, "relative").mockImplementation(path[platform].relative) + vi.spyOn(path, "normalize").mockImplementation(path[platform].normalize) + }) + + afterEach(() => vi.restoreAllMocks()) + + const filePath = "/workspace/src/file.ts" + const relativeFilePath = path[platform].join("src", "file.ts") + const normalizedFilePath = path[platform].resolve(filePath) + const content = "test content" + const hash = createHash("sha256").update(content).digest("hex") + const block: CodeBlock = { + file_path: filePath, + identifier: null, + type: "function", + start_line: 2, + end_line: 5, + content, + fileHash: hash, + segmentHash: "segment", + } + + function setup() { + return { + workspacePath: "/workspace", + ignoreController: { + validateAccess: vi + .fn() + .mockReturnValue(true), + }, + ignoreInstance: { ignores: vi.fn<(path: string) => boolean>().mockReturnValue(false) }, + fileSystem: { + stat: vi + .fn() + .mockResolvedValue({ type: FileType.File, ctime: 0, mtime: 0, size: 100 }), + readFile: vi + .fn() + .mockResolvedValue(Buffer.from(content)), + }, + cacheManager: { getHash: vi.fn() }, + parser: { parseFile: vi.fn().mockResolvedValue([{ ...block }]) }, + embedder: { + createEmbeddings: vi + .fn() + .mockResolvedValue({ embeddings: [[0.1, 0.2]] }), + }, + } satisfies FilePreparationDependencies + } + + it.each([".git/config", ".hidden/file.ts", "node_modules/pkg/file.ts", "dist/file.js"])( + "skips excluded directory %s before access or file reads", + async (relativePath) => { + const dependencies = setup() + const path = `/workspace/${relativePath}` + expect(await new FilePreparation(dependencies).prepareFile(path)).toEqual({ + path, + status: "skipped", + reason: "File is in an ignored directory", + }) + expect(dependencies.ignoreController.validateAccess).not.toHaveBeenCalled() + expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() + expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) + + it.each(["access", "gitignore"])("skips %s exclusions before reading", async (source) => { + const dependencies = setup() + dependencies.ignoreController.validateAccess.mockReturnValue(source !== "access") + dependencies.ignoreInstance.ignores.mockReturnValue(source === "gitignore") + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File is ignored by .rooignore or .gitignore", + }) + expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledWith(filePath) + if (source === "access") { + expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() + } else { + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) + } + expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("skips oversized files without reading them", async () => { + const dependencies = setup() + dependencies.fileSystem.stat.mockResolvedValue({ + type: FileType.File, + ctime: 0, + mtime: 0, + size: MAX_FILE_SIZE_BYTES + 1, + }) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File is too large", + }) + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("skips unchanged content before parsing or embedding", async () => { + const dependencies = setup() + dependencies.cacheManager.getHash.mockReturnValue(hash) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File has not changed", + }) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("returns the hash and empty points when parsing produces no blocks", async () => { + const dependencies = setup() + dependencies.parser.parseFile.mockResolvedValue([]) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [], + }) + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("still parses and returns the hash without an embedder or gitignore", async () => { + const dependencies = setup() + expect( + await new FilePreparation({ ...dependencies, embedder: undefined, ignoreInstance: undefined }).prepareFile( + filePath, + ), + ).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [], + }) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) + }) + + it("preserves normalized paths, stable IDs and embedding order at the size limit", async () => { + const dependencies = setup() + dependencies.fileSystem.stat.mockResolvedValue({ + type: FileType.File, + ctime: 0, + mtime: 0, + size: MAX_FILE_SIZE_BYTES, + }) + dependencies.parser.parseFile.mockResolvedValue([ + { ...block, file_path: "src/../src/file.ts" }, + { ...block, start_line: 8, end_line: 10, content: "second" }, + ]) + dependencies.embedder.createEmbeddings.mockResolvedValue({ + embeddings: [ + [0.1, 0.2], + [0.3, 0.4], + ], + }) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [ + { + id: uuidv5(`${normalizedFilePath}:2`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: [0.1, 0.2], + payload: { filePath: relativeFilePath, codeChunk: content, startLine: 2, endLine: 5 }, + }, + { + id: uuidv5(`${normalizedFilePath}:8`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: [0.3, 0.4], + payload: { filePath: relativeFilePath, codeChunk: "second", startLine: 8, endLine: 10 }, + }, + ], + }) + expect(dependencies.fileSystem.stat).toHaveBeenCalledWith(Uri.file(filePath)) + expect(dependencies.fileSystem.readFile).toHaveBeenCalledWith(Uri.file(filePath)) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) + expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledWith([content, "second"]) + }) + + it("calls service methods with their original receivers", async () => { + const dependencies = setup() + expect((await new FilePreparation(dependencies).prepareFile(filePath)).status).toBe("processed_for_batching") + expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledOnce() + expect(dependencies.ignoreController.validateAccess.mock.contexts[0]).toBe(dependencies.ignoreController) + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledOnce() + expect(dependencies.ignoreInstance.ignores.mock.contexts[0]).toBe(dependencies.ignoreInstance) + expect(dependencies.fileSystem.stat).toHaveBeenCalledOnce() + expect(dependencies.fileSystem.stat.mock.contexts[0]).toBe(dependencies.fileSystem) + expect(dependencies.fileSystem.readFile).toHaveBeenCalledOnce() + expect(dependencies.fileSystem.readFile.mock.contexts[0]).toBe(dependencies.fileSystem) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledOnce() + expect(dependencies.cacheManager.getHash.mock.contexts[0]).toBe(dependencies.cacheManager) + expect(dependencies.parser.parseFile).toHaveBeenCalledOnce() + expect(dependencies.parser.parseFile.mock.contexts[0]).toBe(dependencies.parser) + expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(dependencies.embedder.createEmbeddings.mock.contexts[0]).toBe(dependencies.embedder) + }) + + it("does not treat a hidden workspace ancestor as an excluded directory", async () => { + const dependencies = setup() + dependencies.workspacePath = "/.hidden/workspace" + expect((await new FilePreparation(dependencies).prepareFile("/.hidden/workspace/src/file.ts")).status).toBe( + "processed_for_batching", + ) + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) + }) + + describe.each(["Buffer", "Uint8Array"] as const)("UTF-8 decoding from %s", (representation) => { + it.each([ + { name: "ASCII", bytes: [65, 66], expectedContent: "AB" }, + { + name: "Cyrillic, Spanish, CJK and emoji", + bytes: [...Buffer.from("Привет, español 中文 😀", "utf-8")], + expectedContent: "Привет, español 中文 😀", + }, + { name: "preserved BOM", bytes: [0xef, 0xbb, 0xbf, 65, 66], expectedContent: "\uFEFFAB" }, + { name: "empty content", bytes: [], expectedContent: "" }, + { + name: "invalid and truncated UTF-8 sequences", + bytes: [65, 0xc3, 0x28, 0xff, 0xe2, 0x82], + expectedContent: "A\uFFFD(\uFFFD\uFFFD", + }, + { + name: "sliced view with nonzero byteOffset", + bytes: [0x58, 65, 0xc3, 0xb1, 0x59], + expectedContent: "Añ", + sliced: true, + }, + ])("decodes and hashes $name, then skips the cached content", async ({ bytes, expectedContent, sliced }) => { + const dependencies = setup() + const backing = representation === "Buffer" ? Buffer.from(bytes) : new Uint8Array(bytes) + const fileContent = sliced ? backing.subarray(1, backing.length - 1) : backing + if (sliced) { + expect(fileContent.byteOffset).toBeGreaterThan(0) + expect(fileContent.byteLength).toBeLessThan(backing.byteLength) + } + dependencies.fileSystem.readFile.mockResolvedValue(fileContent) + dependencies.parser.parseFile.mockResolvedValue([]) + // Hash the literal expected text, not the input bytes or the production decoder's output. + const expectedHash = createHash("sha256").update(expectedContent).digest("hex") + const preparation = new FilePreparation(dependencies) + + expect(await preparation.prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: expectedHash, + pointsToUpsert: [], + }) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { + content: expectedContent, + fileHash: expectedHash, + }) + + dependencies.parser.parseFile.mockClear() + dependencies.cacheManager.getHash.mockReturnValue(expectedHash) + expect(await preparation.prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File has not changed", + }) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + }) + + it.each(["access", "ignore", "stat", "read", "cache", "parse", "embed"])( + "returns the original %s error as a local error", + async (stage) => { + const dependencies = setup() + const error = new Error(`${stage} failed`) + const fail = () => { + throw error + } + if (stage === "access") dependencies.ignoreController.validateAccess.mockImplementation(fail) + if (stage === "ignore") dependencies.ignoreInstance.ignores.mockImplementation(fail) + if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(error) + if (stage === "read") dependencies.fileSystem.readFile.mockRejectedValue(error) + if (stage === "cache") dependencies.cacheManager.getHash.mockImplementation(fail) + if (stage === "parse") dependencies.parser.parseFile.mockRejectedValue(error) + if (stage === "embed") dependencies.embedder.createEmbeddings.mockRejectedValue(error) + const result = await new FilePreparation(dependencies).prepareFile(filePath) + expect(result).toEqual({ path: filePath, status: "local_error", error }) + expect(result.error).toBe(error) + }, + ) + + it("does not wrap non-Error rejections", async () => { + const dependencies = setup() + dependencies.fileSystem.readFile.mockRejectedValue("read failed") + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "local_error", + error: "read failed", + }) + }) +}) diff --git a/src/services/code-index/processors/__tests__/parser-constants.spec.ts b/src/services/code-index/processors/__tests__/parser-constants.spec.ts deleted file mode 100644 index 884bfd5bb6..0000000000 --- a/src/services/code-index/processors/__tests__/parser-constants.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { build } from "esbuild" -import * as path from "path" -import * as parserConstants from "../parser-constants" - -it("preserves the four foundation parser constants exactly", () => { - expect({ ...parserConstants }).toEqual({ - MAX_BLOCK_CHARS: 1000, - MIN_BLOCK_CHARS: 50, - MIN_CHUNK_REMAINDER_CHARS: 200, - MAX_CHARS_TOLERANCE_FACTOR: 1.15, - }) -}) - -it("bundles the actual parser worker without host or types/provider barrel inputs", async () => { - const result = await build({ - entryPoints: [path.resolve("workers/codeIndexParse.ts")], - bundle: true, - platform: "node", - format: "cjs", - write: false, - metafile: true, - }) - const inputs = Object.keys(result.metafile.inputs).map((input) => input.replaceAll("\\", "/")) - expect(inputs).toContain("services/code-index/processors/parser-constants.ts") - expect( - inputs.filter((input) => - /(?:code-index\/constants\/|packages\/types\/|\/providers\/|telemetry|RooIgnoreController)/.test(input), - ), - ).toEqual([]) - const imports = Object.values(result.metafile.outputs).flatMap((output) => output.imports) - expect(imports.filter((entry) => entry.path === "vscode" || entry.path === "@roo-code/types")).toEqual([]) -}) diff --git a/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts b/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts deleted file mode 100644 index c8559ba395..0000000000 --- a/src/services/code-index/processors/__tests__/parser.worker-contract.spec.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { createHash } from "crypto" -import { readFile } from "fs/promises" -import type { Node, QueryCapture, Tree } from "web-tree-sitter" -import { CodeParser } from "../parser" -import { codeIndexLog } from "../../code-index-log" -import { MAX_BLOCK_CHARS } from "../parser-constants" -import type { CodeBlock } from "../../interfaces" -import { ParserFatalError } from "../../parser-errors" -import { scannerExtensions } from "../../shared/supported-extensions" -import { type LanguageParser, loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser" -import { parseMarkdown } from "../../../tree-sitter/markdownParser" - -vi.mock("fs/promises", () => ({ readFile: vi.fn() })) -vi.mock("../../../tree-sitter/languageParser", () => ({ loadRequiredLanguageParsers: vi.fn() })) -vi.mock("../../../tree-sitter/markdownParser", () => ({ parseMarkdown: vi.fn() })) -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) - -type ParseDelegate = NonNullable[0]> - -function createNode(content: string, row = 0, children: Node[] = []): Node { - // The parser only reads text, positions, type, children and the name field from these nodes. - return { - text: content, - startPosition: { row, column: 0 }, - endPosition: { row: row + content.split("\n").length - 1, column: 0 }, - type: "function_declaration", - children, - childForFieldName: () => null, - } as unknown as Node -} - -function createLanguage() { - const release = vi.fn<() => void>() - const rootNode = createNode("source") - // Only rootNode and delete are consumed by CodeParser. - const tree = { rootNode, delete: release } as unknown as Tree - const parse = vi.fn().mockReturnValue(tree) - const captures = vi.fn().mockReturnValue([]) - // These structural doubles supply the only parser/query methods CodeParser invokes. - const language = { js: { parser: { parse }, query: { captures } } } as unknown as LanguageParser - return { language, parse, captures, release } -} - -function expectUnicodePreserved(blocks: CodeBlock[], content: string): void { - expect(blocks.length).toBeGreaterThan(1) - expect(blocks.map((block) => block.content).join("")).toBe(content) - for (const block of blocks) { - expect(Buffer.from(block.content, "utf8").toString("utf8")).toBe(block.content) - expect(block.content.length).toBeLessThanOrEqual(MAX_BLOCK_CHARS) - } - expect(new Set(blocks.map((block) => block.segmentHash)).size).toBe(blocks.length) -} - -beforeEach(() => { - vi.resetAllMocks() - vi.mocked(parseMarkdown).mockReturnValue([]) -}) - -describe("CodeParser worker contract", () => { - it("keeps worker-safe extension support identical to the scanner", async () => { - const delegate = vi.fn().mockResolvedValue([]) - const parser = new CodeParser(delegate) - const supported = scannerExtensions.filter((extension) => parser["isSupportedLanguage"](extension)) - expect(supported).toEqual(scannerExtensions) - expect(parser["isSupportedLanguage"](".unsupported")).toBe(false) - await expect(parser.parseFile("file.unsupported")).resolves.toEqual([]) - expect(delegate).not.toHaveBeenCalled() - expect(readFile).not.toHaveBeenCalled() - }) - - it.each(["file.js", "file.TS", "file.dart", "file.rdl", "file.tla"])( - "delegates structural parsing for %s with content, hash and signal", - async (filePath) => { - const blocks: CodeBlock[] = [] - const delegate = vi.fn().mockResolvedValue(blocks) - const signal = new AbortController().signal - const result = await new CodeParser(delegate).parseFile(filePath, { - content: "source", - fileHash: "hash", - signal, - }) - expect(result).toBe(blocks) - expect(delegate).toHaveBeenCalledExactlyOnceWith(filePath, "source", "hash", signal) - expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() - expect(readFile).not.toHaveBeenCalled() - }, - ) - - it.each(["file.md", "file.markdown", "file.txt", "file.TXT", "file.vb", "file.scala", "file.swift"])( - "keeps %s on the local Markdown/fallback path", - async (filePath) => { - const delegate = vi.fn() - const content = "Local parsing preserves Zoo language support and searchable file content." - const blocks = await new CodeParser(delegate).parseFile(filePath, { content, fileHash: "hash" }) - expect(blocks).toHaveLength(1) - expect(blocks[0]).toMatchObject({ content, fileHash: "hash", start_line: 1, end_line: 1 }) - expect(delegate).not.toHaveBeenCalled() - expect(loadRequiredLanguageParsers).not.toHaveBeenCalled() - }, - ) - - it.each(["", "const value = 42"])("hashes supplied content without reading a file: %j", async (content) => { - const delegate = vi.fn().mockResolvedValue([]) - await new CodeParser(delegate).parseFile("file.js", { content }) - expect(delegate).toHaveBeenCalledWith( - "file.js", - content, - createHash("sha256").update(content).digest("hex"), - undefined, - ) - expect(readFile).not.toHaveBeenCalled() - }) - - it("reads and hashes content before delegation", async () => { - vi.mocked(readFile).mockResolvedValue("file contents") - const delegate = vi.fn().mockResolvedValue([]) - await new CodeParser(delegate).parseFile("file.js") - expect(readFile).toHaveBeenCalledWith("file.js", "utf8") - expect(delegate).toHaveBeenCalledWith( - "file.js", - "file contents", - createHash("sha256").update("file contents").digest("hex"), - undefined, - ) - }) - - it("logs ordinary read failure without delegating", async () => { - const error = new Error("read failure") - vi.mocked(readFile).mockRejectedValue(error) - const delegate = vi.fn() - await expect(new CodeParser(delegate).parseFile("file.js")).resolves.toEqual([]) - expect(codeIndexLog.error).toHaveBeenCalledWith("Error reading file file.js:", error) - expect(delegate).not.toHaveBeenCalled() - }) - - it.each(["file.js", "file.md", "file.txt", "file.unsupported"])( - "rejects pre-cancelled work before reading or parsing %s", - async (filePath) => { - const controller = new AbortController() - controller.abort() - const delegate = vi.fn() - await expect( - new CodeParser(delegate).parseFile(filePath, { signal: controller.signal }), - ).rejects.toMatchObject({ - name: "AbortError", - }) - expect(delegate).not.toHaveBeenCalled() - expect(readFile).not.toHaveBeenCalled() - expect(parseMarkdown).not.toHaveBeenCalled() - }, - ) - - it.each([false, true])("cancels after an in-flight read settles (read failure: %s)", async (rejectRead) => { - const controller = new AbortController() - vi.mocked(readFile).mockImplementation(async () => { - controller.abort() - if (rejectRead) throw new Error("late read failure") - return "source" - }) - const delegate = vi.fn() - await expect( - new CodeParser(delegate).parseFile("file.js", { signal: controller.signal }), - ).rejects.toMatchObject({ - name: "AbortError", - }) - expect(delegate).not.toHaveBeenCalled() - expect(codeIndexLog.error).not.toHaveBeenCalled() - }) - - it.each([ - new ParserFatalError("parse-timeout", "file.js"), - new ParserFatalError("parser-abort", "file.js"), - new Error("ordinary worker failure"), - ])("propagates delegate failure unchanged: %s", async (error) => { - const delegate = vi.fn().mockRejectedValue(error) - await expect(new CodeParser(delegate).parseFile("file.js", { content: "source" })).rejects.toBe(error) - }) - - it("passes the worker WASM directory and requests host fallback only when captures are absent", async () => { - const { language, captures, release } = createLanguage() - vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) - const parser = new CodeParser(undefined, "extension-dist") - const content = "Uncaptured structural content still needs to be indexed through host fallback." - await expect(parser.parseFile("file.js", { content })).resolves.toEqual([]) - expect(loadRequiredLanguageParsers).toHaveBeenCalledExactlyOnceWith(["file.js"], "extension-dist") - expect(parser.fallbackNeeded).toBe(true) - expect(release).toHaveBeenCalledOnce() - expect(parser.fallbackContent("file.js", content, "hash")[0]).toMatchObject({ content, fileHash: "hash" }) - - captures.mockReturnValue([{ node: createNode(content), name: "definition.function", patternIndex: 0 }]) - const blocks = await parser.parseFile("file.js", { content }) - expect(blocks).toHaveLength(1) - expect(parser.fallbackNeeded).toBe(false) - expect(release).toHaveBeenCalledTimes(2) - expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() - }) - - it.each(["file.txt", "file.md", "file.unsupported"])("resets stale fallback state for %s", async (filePath) => { - const { language } = createLanguage() - vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) - const parser = new CodeParser(undefined, "extension-dist") - await parser.parseFile("file.js", { content: "source" }) - expect(parser.fallbackNeeded).toBe(true) - await parser.parseFile(filePath, { content: "Enough local content to create a full indexable fallback block." }) - expect(parser.fallbackNeeded).toBe(false) - }) - - it.each(["Aborted()", "memory access out of bounds", "ordinary load failure"])( - "handles a fresh and shared parser-load failure: %s", - async (message) => { - const error = new Error(message) - let rejectLoad!: (reason: Error) => void - const pending = new Promise((_resolve, reject) => { - rejectLoad = reject - }) - vi.mocked(loadRequiredLanguageParsers).mockReturnValue(pending) - const parser = new CodeParser() - const first = parser.parseFile("first.js", { content: "source" }).catch((failure: unknown) => failure) - const second = parser.parseFile("second.js", { content: "source" }).catch((failure: unknown) => failure) - rejectLoad(error) - const results = await Promise.all([first, second]) - if (message === "ordinary load failure") expect(results).toEqual([[], []]) - else for (const result of results) expect(result).toBe(error) - expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() - expect(parser["pendingLoads"].size).toBe(0) - }, - ) - - it("shares a successful parser load between concurrent files", async () => { - const { language, parse, release } = createLanguage() - let resolveLoad!: (loaded: LanguageParser) => void - vi.mocked(loadRequiredLanguageParsers).mockReturnValue( - new Promise((resolve) => { - resolveLoad = resolve - }), - ) - const parser = new CodeParser() - const first = parser.parseFile("first.js", { content: "source" }) - const second = parser.parseFile("second.js", { content: "source" }) - resolveLoad(language) - await Promise.all([first, second]) - expect(loadRequiredLanguageParsers).toHaveBeenCalledOnce() - expect(parse).toHaveBeenCalledTimes(2) - expect(release).toHaveBeenCalledTimes(2) - }) - - it.each(["success", "query failure", "fallback", "null tree"])("releases tree on %s", async (outcome) => { - const { language, parse, captures, release } = createLanguage() - vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) - const content = "A structural function with sufficient content for indexing and lifecycle checks." - const error = new Error("memory access out of bounds") - if (outcome === "query failure") - captures.mockImplementation(() => { - throw error - }) - if (outcome === "success") - captures.mockReturnValue([{ node: createNode(content), name: "definition", patternIndex: 0 }]) - if (outcome === "null tree") parse.mockReturnValue(null) - const result = new CodeParser().parseFile("file.js", { content }) - if (outcome === "query failure") await expect(result).rejects.toBe(error) - else expect((await result)[0].content).toBe(content) - expect(release).toHaveBeenCalledTimes(outcome === "null tree" ? 0 : 1) - }) - - it("walks oversized children in capture order and deduplicates captured nodes", async () => { - const { language, captures, release } = createLanguage() - const child = createNode("A child declaration with enough meaningful content to create a code block.", 4) - const leaf = createNode("z".repeat(2001), 8) - const parent = createNode("p".repeat(4000), 0, [child, leaf]) - const capture = (node: Node): QueryCapture => ({ node, name: "definition", patternIndex: 0 }) - captures.mockReturnValue([capture(parent), capture(child)]) - vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(language) - const blocks = await new CodeParser().parseFile("file.js", { content: "source" }) - expect(blocks.map((block) => block.content)).toEqual([child.text, "z".repeat(1000), "z".repeat(1000), "z"]) - expect(blocks.map((block) => block.start_line)).toEqual([5, 9, 9, 9]) - expect(release).toHaveBeenCalledOnce() - }) -}) - -describe("CodeParser Unicode segmentation", () => { - it.each(["lines", "supplementary long line"])("keeps fallback scaling proportional for %s", (shape) => { - const parser = new CodeParser() - const source = - shape === "lines" - ? "const example = value => value.toUpperCase();\n".repeat(750) - : ("a".repeat(999) + "\u{1f600}").repeat(32) - const inputs = [1, 2, 4, 8].map((scale) => source.repeat(scale)) - const measure = (content: string) => { - const samples: number[] = [] - for (let sample = 0; sample < 5; sample++) { - const started = performance.now() - for (let repeat = 0; repeat < 40; repeat++) parser.fallbackContent("file.txt", content, "hash") - samples.push(performance.now() - started) - } - return Math.min(...samples) - } - for (let warmup = 0; warmup < 40; warmup++) parser.fallbackContent("file.txt", source, "hash") - const timings = inputs.map(measure) - expect(timings[2] / timings[0]).toBeLessThan(9) - expect(timings[3] / timings[0]).toBeLessThan(18) - }) - - it.each([MAX_BLOCK_CHARS - 2, MAX_BLOCK_CHARS - 1, MAX_BLOCK_CHARS])( - "keeps a supplementary code point whole at offset %s and hashes actual offsets", - (pairOffset) => { - const content = "a".repeat(pairOffset) + "\u{1f600}" + "b".repeat(MAX_BLOCK_CHARS + 17) - const blocks = new CodeParser()["_chunkTextByLines"]([content], "file.js", "hash", "code", new Set(), 42) - expectUnicodePreserved(blocks, content) - expect(blocks[0].content.length).toBe( - pairOffset === MAX_BLOCK_CHARS - 1 ? MAX_BLOCK_CHARS - 1 : MAX_BLOCK_CHARS, - ) - let offset = 0 - for (const block of blocks) { - expect(block).toMatchObject({ start_line: 42, end_line: 42, type: "code_segment" }) - expect(block.segmentHash).toBe( - createHash("sha256") - .update(`file.js-42-42-${offset}-${block.content.length}-${block.content.slice(0, 100)}`) - .digest("hex"), - ) - offset += block.content.length - } - }, - ) - - it("uses adjusted cuts for consecutive pairs and preserves a short tail", () => { - const content = "a".repeat(999) + ("\u{1f600}" + "a".repeat(997)).repeat(3) + "\u{1f680}" - const blocks = new CodeParser().fallbackContent("file.txt", content, "hash") - expectUnicodePreserved(blocks, content) - expect(blocks.map((block) => block.content.length)).toEqual([999, 999, 999, 999, 2]) - }) - - it.each(["file.md", "file.markdown", "file.txt", "file.scala", "file.vb", "file.js"])( - "preserves Unicode through parseFile for %s", - async (filePath) => { - vi.mocked(loadRequiredLanguageParsers).mockResolvedValue(createLanguage().language) - const content = "a".repeat(999) + "\u{1f600}" + "漢é".repeat(1000) - const blocks = await new CodeParser().parseFile(filePath, { content }) - expectUnicodePreserved(blocks, content) - }, - ) - - it.each([ - "ascii ", - "café Ελληνικά Кириллица ", - "漢字かなカナ한국어 ", - "e\u0301 a\u0308 \u200d \ufe0f ", - "\ud7ff\ue000\ufffd\uffff ", - ])("retains BMP segment contents and hashes: %j", (seed) => { - const content = seed.repeat(500) - const expected: CodeBlock[] = [] - for (let offset = 0; offset < content.length; offset += MAX_BLOCK_CHARS) { - const segment = content.substring(offset, offset + MAX_BLOCK_CHARS) - expected.push({ - file_path: "file.txt", - identifier: null, - type: "fallback_chunk_segment", - start_line: 1, - end_line: 1, - content: segment, - fileHash: "hash", - segmentHash: createHash("sha256") - .update(`file.txt-1-1-${offset}-${segment.length}-${segment.slice(0, 100)}`) - .digest("hex"), - }) - } - expect(new CodeParser().fallbackContent("file.txt", content, "hash")).toEqual(expected) - }) -}) diff --git a/src/services/code-index/processors/__tests__/processor-factories.spec.ts b/src/services/code-index/processors/__tests__/processor-factories.spec.ts index 57718968ac..1b222fb997 100644 --- a/src/services/code-index/processors/__tests__/processor-factories.spec.ts +++ b/src/services/code-index/processors/__tests__/processor-factories.spec.ts @@ -7,14 +7,15 @@ import { CacheManager } from "../../cache-manager" import { BATCH_SEGMENT_THRESHOLD } from "../../constants" import { OpenAiEmbedder } from "../../embedders/openai" import { QdrantVectorStore } from "../../vector-store/qdrant-client" -import { ParseWorkerClient } from "../../parse-worker-client" -import { IndexWriteGate } from "../../index-write-gate" -import { GenerationAllocator } from "../../generation-allocator" -import { FileFailureRegistry } from "../../file-failure-registry" -import { ProviderPacerRegistry } from "../../provider-pacer" +import { CodeIndexServiceFactory } from "../../service-factory" +import { CodeIndexConfigManager } from "../../config-manager" +import { EmbedderFactory } from "../../embedders/embedder-factory" +import { VectorStoreFactory } from "../../vector-store/vector-store-factory" +import type { CodeIndexConfig } from "../../interfaces/config" import { DirectoryScannerFactory } from "../directory-scanner-factory" import { FileWatcherFactory } from "../file-watcher-factory" import { getEmbeddingBatchSize } from "../get-embedding-batch-size" +import { codeParser } from "../parser" import { DirectoryScanner } from "../scanner" import { FileWatcher } from "../file-watcher" @@ -24,7 +25,8 @@ vi.mock("vscode", () => ({ vi.mock("../../cache-manager") vi.mock("../../embedders/openai") vi.mock("../../vector-store/qdrant-client") -vi.mock("../../parse-worker-client") +vi.mock("../../config-manager") +vi.mock("../parser", () => ({ codeParser: { parseFile: vi.fn() } })) vi.mock("../scanner") vi.mock("../file-watcher") @@ -34,20 +36,14 @@ describe("processor factories", () => { function dependencies() { const context = makeExtensionContext() - const cacheManager = new CacheManager(context, "/workspace") return { context, workspacePath: "/workspace", - cacheManager, + cacheManager: new CacheManager(context, "/workspace"), embedder: new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), vectorStore: new QdrantVectorStore("/workspace", "http://localhost:6333", 512), - parser: new ParseWorkerClient(), + parser: codeParser, ignoreInstance: ignore(), - writeGate: new IndexWriteGate(), - generations: new GenerationAllocator(cacheManager), - failures: new FileFailureRegistry(), - isCurrentAttempt: vi.fn(() => true), - providerPacers: new ProviderPacerRegistry(), } } @@ -71,12 +67,6 @@ describe("processor factories", () => { options.cacheManager, options.ignoreInstance, batchSize, - options.writeGate, - options.generations, - options.failures, - options.workspacePath, - options.isCurrentAttempt, - options.providerPacers, ) expect(FileWatcher).toHaveBeenCalledWith( options.workspacePath, @@ -87,12 +77,6 @@ describe("processor factories", () => { options.ignoreInstance, undefined, batchSize, - options.parser, - options.writeGate, - options.generations, - options.failures, - options.isCurrentAttempt, - options.providerPacers, ) expect(vscode.workspace.getConfiguration).toHaveBeenCalledWith(Package.name) expect(configuration.get).toHaveBeenCalledWith("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) @@ -155,62 +139,49 @@ describe("processor factories", () => { expect(warn).not.toHaveBeenCalled() }) - it("keeps shared retry budgets and current-attempt callbacks across processor recreation", () => { + it("preserves service factory wiring, including the explicitly supplied watcher cache", () => { const options = dependencies() - options.failures.renew("file.ts", "revision") - const budget = options.failures.getRetryBudget("file.ts", "revision", "embed") - budget.retriesUsed = 2 - const reset = vi.spyOn(options.failures, "resetAllBudgets") - const scanners = new DirectoryScannerFactory() - const watchers = new FileWatcherFactory() - - for (const current of [true, false]) { - options.isCurrentAttempt.mockReturnValue(current) - scanners.create(options) - watchers.create(options) - const scannerArguments = vi.mocked(DirectoryScanner).mock.lastCall! - const watcherArguments = vi.mocked(FileWatcher).mock.lastCall! - expect(scannerArguments[2]).toBe(options.parser) - expect(watcherArguments[8]).toBe(options.parser) - expect(scannerArguments[6]).toBe(options.writeGate) - expect(watcherArguments[9]).toBe(options.writeGate) - expect(scannerArguments[7]).toBe(options.generations) - expect(watcherArguments[10]).toBe(options.generations) - expect(scannerArguments[8]).toBe(options.failures) - expect(watcherArguments[11]).toBe(options.failures) - expect(scannerArguments[10]).toBe(options.isCurrentAttempt) - expect(watcherArguments[12]).toBe(options.isCurrentAttempt) - expect(scannerArguments[11]).toBe(options.providerPacers) - expect(watcherArguments[13]).toBe(options.providerPacers) - expect(scannerArguments[10]?.()).toBe(current) - expect(watcherArguments[12]?.()).toBe(current) - } - - expect(options.failures.getRetryBudget("file.ts", "revision", "embed")).toBe(budget) - expect(budget.retriesUsed).toBe(2) - expect(reset).not.toHaveBeenCalled() + const watcherCache = new CacheManager(options.context, "/workspace") + const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager + const config: CodeIndexConfig = { isConfigured: true, embedderProvider: "openai-compatible" } + Object.defineProperty(configManager, "isFeatureConfigured", { value: true }) + vi.mocked(configManager.getConfig).mockReturnValue(config) + const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create").mockReturnValue(options.embedder) + const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create").mockReturnValue(options.vectorStore) + const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) + + const services = factory.createServices(options.context, watcherCache, options.ignoreInstance) + + expect(configManager.getConfig).toHaveBeenCalledTimes(1) + expect(createEmbedder).toHaveBeenCalledExactlyOnceWith(config) + expect(createVectorStore).toHaveBeenCalledExactlyOnceWith(config, options.workspacePath) + expect(services).toEqual({ + embedder: options.embedder, + vectorStore: options.vectorStore, + parser: codeParser, + scanner: vi.mocked(DirectoryScanner).mock.instances[0], + fileWatcher: vi.mocked(FileWatcher).mock.instances[0], + }) + expect(vi.mocked(DirectoryScanner).mock.calls[0][3]).toBe(options.cacheManager) + expect(vi.mocked(FileWatcher).mock.calls[0][2]).toBe(watcherCache) + expect(vi.mocked(FileWatcher).mock.calls[0][0]).toBe(options.workspacePath) }) - it("preserves optional constructor defaults for standalone factory callers", () => { - const { embedder, vectorStore, parser, cacheManager, ignoreInstance, context, workspacePath } = dependencies() - new DirectoryScannerFactory().create({ embedder, vectorStore, parser, cacheManager, ignoreInstance }) - new FileWatcherFactory().create({ workspacePath, context, embedder, vectorStore, cacheManager, ignoreInstance }) - - expect(vi.mocked(DirectoryScanner).mock.lastCall?.slice(6)).toEqual([ - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - ]) - expect(vi.mocked(FileWatcher).mock.lastCall?.slice(8)).toEqual([ - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - ]) + it("does not create services when indexing is not configured", () => { + const options = dependencies() + const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager + Object.defineProperty(configManager, "isFeatureConfigured", { value: false }) + const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create") + const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create") + const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) + + expect(() => factory.createServices(options.context, options.cacheManager, options.ignoreInstance)).toThrow( + "serviceFactory.codeIndexingNotConfigured", + ) + expect(configManager.getConfig).not.toHaveBeenCalled() + expect(createEmbedder).not.toHaveBeenCalled() + expect(createVectorStore).not.toHaveBeenCalled() + expect(DirectoryScanner).not.toHaveBeenCalled() + expect(FileWatcher).not.toHaveBeenCalled() }) }) diff --git a/src/services/code-index/processors/file-preparation-dependencies.ts b/src/services/code-index/processors/file-preparation-dependencies.ts new file mode 100644 index 0000000000..001da428d7 --- /dev/null +++ b/src/services/code-index/processors/file-preparation-dependencies.ts @@ -0,0 +1,15 @@ +import type { Ignore } from "ignore" +import type { FileSystem } from "vscode" +import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" +import type { CacheManager } from "../cache-manager" +import type { ICodeParser, IEmbedder } from "../interfaces" + +export interface FilePreparationDependencies { + workspacePath: string + ignoreController: Pick + ignoreInstance?: Pick + fileSystem: Pick + cacheManager: Pick + parser: ICodeParser + embedder?: Pick +} diff --git a/src/services/code-index/processors/file-preparation.ts b/src/services/code-index/processors/file-preparation.ts new file mode 100644 index 0000000000..a99ff6b56a --- /dev/null +++ b/src/services/code-index/processors/file-preparation.ts @@ -0,0 +1,100 @@ +import { createHash } from "crypto" +import { Uri } from "vscode" +import { v5 as uuidv5 } from "uuid" +import type { CodeBlock, FileProcessingResult, PointStruct } from "../interfaces" +import type { FilePreparationDependencies } from "./file-preparation-dependencies" +import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" +import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" + +/** Prepares one file for batching without writing points or mutating the hash cache. */ +export class FilePreparation { + constructor(private readonly dependencies: FilePreparationDependencies) {} + + public async prepareFile(filePath: string): Promise { + const dependencies = this.dependencies + try { + const skipReason = await this.getSkipReason(filePath) + if (skipReason !== undefined) { + return this.skippedResult(filePath, skipReason) + } + + const fileContent = await dependencies.fileSystem.readFile(Uri.file(filePath)) + const content = Buffer.from(fileContent).toString("utf-8") + const newHash = createHash("sha256").update(content).digest("hex") + + if (dependencies.cacheManager.getHash(filePath) === newHash) { + return this.skippedResult(filePath, "File has not changed") + } + + const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash }) + const pointsToUpsert = await this.preparePoints(blocks) + + return { + path: filePath, + status: "processed_for_batching", + newHash, + pointsToUpsert, + } + } catch (error) { + return { + path: filePath, + status: "local_error", + error: error as Error, + } + } + } + + private async getSkipReason(filePath: string): Promise { + const dependencies = this.dependencies + // Use relative paths so ignored directories outside the workspace do not exclude the file. + const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath) + if (isPathInIgnoredDirectory(relativeFilePath)) { + return "File is in an ignored directory" + } + + if ( + !dependencies.ignoreController.validateAccess(filePath) || + dependencies.ignoreInstance?.ignores(relativeFilePath) + ) { + return "File is ignored by .rooignore or .gitignore" + } + + const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath)) + if (fileStat.size > MAX_FILE_SIZE_BYTES) { + return "File is too large" + } + return undefined + } + + private skippedResult(filePath: string, reason: string): FileProcessingResult { + return { path: filePath, status: "skipped", reason } + } + + private async preparePoints(blocks: CodeBlock[]): Promise { + const dependencies = this.dependencies + if (!dependencies.embedder || blocks.length === 0) { + return [] + } + + const texts = blocks.map((block) => block.content) + const { embeddings } = await dependencies.embedder.createEmbeddings(texts) + + return blocks.map((block, index) => { + const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, dependencies.workspacePath) + const stableName = `${normalizedAbsolutePath}:${block.start_line}` + const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) + + return { + id: pointId, + vector: embeddings[index], + payload: { + filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath), + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + }, + } + }) + } +} diff --git a/src/services/code-index/processors/index.ts b/src/services/code-index/processors/index.ts new file mode 100644 index 0000000000..c244d9b875 --- /dev/null +++ b/src/services/code-index/processors/index.ts @@ -0,0 +1,3 @@ +export * from "./parser" +export * from "./scanner" +export * from "./file-watcher" diff --git a/src/services/code-index/processors/parser-constants.ts b/src/services/code-index/processors/parser-constants.ts deleted file mode 100644 index ec82fee326..0000000000 --- a/src/services/code-index/processors/parser-constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Keep worker parsing independent of the host/types barrel in ../constants. -export const MAX_BLOCK_CHARS = 1000 -export const MIN_BLOCK_CHARS = 50 -export const MIN_CHUNK_REMAINDER_CHARS = 200 -export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 diff --git a/src/services/code-index/shared/__tests__/get-relative-path.spec.ts b/src/services/code-index/shared/__tests__/get-relative-path.spec.ts index 9981e59473..d5398f1332 100644 --- a/src/services/code-index/shared/__tests__/get-relative-path.spec.ts +++ b/src/services/code-index/shared/__tests__/get-relative-path.spec.ts @@ -1,8 +1,34 @@ import path from "path" -import { generateRelativeFilePath } from "../get-relative-path" +import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../get-relative-path" describe("get-relative-path", () => { + describe("generateNormalizedAbsolutePath", () => { + it("should use provided workspace root", () => { + const filePath = "src/file.ts" + const workspaceRoot = path.join(path.sep, "custom", "workspace") + const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) + // On Windows, path.resolve adds the drive letter, so we need to use path.resolve for the expected value + expect(result).toBe(path.resolve(workspaceRoot, filePath)) + }) + + it("should handle absolute paths", () => { + const filePath = path.join(path.sep, "absolute", "path", "file.ts") + const workspaceRoot = path.join(path.sep, "custom", "workspace") + const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) + // When an absolute path is provided, it should be resolved to include drive letter on Windows + expect(result).toBe(path.resolve(filePath)) + }) + + it("should normalize paths with . and .. segments", () => { + const filePath = "./src/../src/file.ts" + const workspaceRoot = path.join(path.sep, "custom", "workspace") + const result = generateNormalizedAbsolutePath(filePath, workspaceRoot) + // Use path.resolve to get the expected normalized absolute path + expect(result).toBe(path.resolve(workspaceRoot, "src", "file.ts")) + }) + }) + describe("generateRelativeFilePath", () => { it("should use provided workspace root", () => { const workspaceRoot = path.join(path.sep, "custom", "workspace") diff --git a/src/services/code-index/shared/get-relative-path.ts b/src/services/code-index/shared/get-relative-path.ts index 92342bd975..642a207208 100644 --- a/src/services/code-index/shared/get-relative-path.ts +++ b/src/services/code-index/shared/get-relative-path.ts @@ -1,5 +1,20 @@ import path from "path" +/** + * Generates a normalized absolute path from a given file path and workspace root. + * Handles path resolution and normalization to ensure consistent absolute paths. + * + * @param filePath - The file path to normalize (can be relative or absolute) + * @param workspaceRoot - The root directory of the workspace (required) + * @returns The normalized absolute path + */ +export function generateNormalizedAbsolutePath(filePath: string, workspaceRoot: string): string { + // Resolve the path to make it absolute if it's relative + const resolvedPath = path.resolve(workspaceRoot, filePath) + // Normalize to handle any . or .. segments and duplicate slashes + return path.normalize(resolvedPath) +} + /** * Generates a relative file path from a normalized absolute path and workspace root. * Ensures consistent relative path generation across different platforms. diff --git a/src/services/code-index/shared/supported-extensions.ts b/src/services/code-index/shared/supported-extensions.ts index 6966e36f70..a5f93e161f 100644 --- a/src/services/code-index/shared/supported-extensions.ts +++ b/src/services/code-index/shared/supported-extensions.ts @@ -1,7 +1,31 @@ import { extensions as allExtensions } from "../../tree-sitter" -import { fallbackExtensions } from "../../shared/fallback-extensions" +import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions" export { fallbackExtensions } // Include all extensions including markdown for the scanner export const scannerExtensions = allExtensions + +/** + * Extensions that should always use fallback chunking instead of tree-sitter parsing. + * These are typically languages that don't have a proper WASM parser available + * or where the parser doesn't work correctly. + * + * NOTE: Only extensions that are already in the supported extensions list can be added here. + * To add support for new file types, they must first be added to the tree-sitter extensions list. + * + * HOW TO ADD A NEW FALLBACK EXTENSION: + * 1. First ensure the extension is in src/services/tree-sitter/index.ts extensions array + * 2. Add the extension to the fallbackExtensions array below + * 3. The file will automatically use length-based chunking for indexing + * + * Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere + */ +/** + * Check if a file extension should use fallback chunking + * @param extension File extension (including the dot) + * @returns true if the extension should use fallback chunking + */ +export function shouldUseFallbackChunking(extension: string): boolean { + return isFallbackExtension(extension) +} diff --git a/src/vitest.dist.config.ts b/src/vitest.dist.config.ts index b47de6c460..0eae23958f 100644 --- a/src/vitest.dist.config.ts +++ b/src/vitest.dist.config.ts @@ -6,11 +6,8 @@ export default mergeConfig( baseConfig, defineConfig({ test: { - // These smoke tests validate emitted artifacts; source coverage remains in the unit lane. - include: [ - "__tests__/dist_assets.spec.ts", - "services/code-index/__tests__/parse-worker-client.real.spec.ts", - ], + // This smoke test validates emitted artifacts; source coverage remains in the unit lane. + include: ["__tests__/dist_assets.spec.ts"], }, }), ) diff --git a/src/vitest.services.config.ts b/src/vitest.services.config.ts index c9ff735baa..eb3d53d0b0 100644 --- a/src/vitest.services.config.ts +++ b/src/vitest.services.config.ts @@ -9,7 +9,6 @@ export default mergeConfig( include: ["services/**/*.{test,spec}.{ts,tsx}"], exclude: [ "services/tree-sitter/**/*.{test,spec}.{ts,tsx}", - "services/code-index/__tests__/parse-worker-client.real.spec.ts", "services/__tests__/pr-review-state-workflow.test.ts", "services/__tests__/stale-pr-clock-workflow.test.ts", ], diff --git a/src/vitest.unit.config.ts b/src/vitest.unit.config.ts index ed59587db8..f3a20e86b4 100644 --- a/src/vitest.unit.config.ts +++ b/src/vitest.unit.config.ts @@ -9,7 +9,6 @@ export default mergeConfig( exclude: [ ...configDefaults.exclude, "__tests__/dist_assets.spec.ts", - "services/code-index/__tests__/parse-worker-client.real.spec.ts", "services/__tests__/pr-review-state-workflow.test.ts", "services/__tests__/stale-pr-clock-workflow.test.ts", ], diff --git a/src/workers/codeIndexParse.ts b/src/workers/codeIndexParse.ts deleted file mode 100644 index a660fc2a3e..0000000000 --- a/src/workers/codeIndexParse.ts +++ /dev/null @@ -1 +0,0 @@ -import "../services/code-index/parse-worker" From 26721b4547787f6072efc5ded2b9e33b52269189 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 21:58:33 -0800 Subject: [PATCH 20/42] refactor(code-index): single file writer, pause-gate pacer, counter-map rate-limit scopes --- .../__tests__/file-completion.spec.ts | 691 ------------------ .../__tests__/file-failure-registry.spec.ts | 17 +- .../__tests__/file-index-writer.spec.ts | 118 +++ .../__tests__/generation-allocator.spec.ts | 101 --- .../__tests__/index-write-gate.spec.ts | 320 -------- .../__tests__/provider-pacer.spec.ts | 497 +++---------- .../__tests__/state-manager.spec.ts | 114 --- src/services/code-index/constants/index.ts | 9 +- .../embedders/__tests__/cancellation.spec.ts | 1 - src/services/code-index/file-completion.ts | 311 -------- .../code-index/file-failure-registry.ts | 2 +- src/services/code-index/file-index-writer.ts | 117 +++ .../code-index/generation-allocator.ts | 58 -- src/services/code-index/index-write-gate.ts | 176 ----- src/services/code-index/provider-pacer.ts | 266 ++----- src/services/code-index/search-service.ts | 78 +- .../shared/__tests__/indexing-errors.spec.ts | 25 + .../shared/__tests__/retry-policy.spec.ts | 13 +- .../code-index/shared/indexing-errors.ts | 15 +- src/services/code-index/state-manager.ts | 40 +- 20 files changed, 516 insertions(+), 2453 deletions(-) delete mode 100644 src/services/code-index/__tests__/file-completion.spec.ts create mode 100644 src/services/code-index/__tests__/file-index-writer.spec.ts delete mode 100644 src/services/code-index/__tests__/generation-allocator.spec.ts delete mode 100644 src/services/code-index/__tests__/index-write-gate.spec.ts delete mode 100644 src/services/code-index/__tests__/state-manager.spec.ts delete mode 100644 src/services/code-index/file-completion.ts create mode 100644 src/services/code-index/file-index-writer.ts delete mode 100644 src/services/code-index/generation-allocator.ts delete mode 100644 src/services/code-index/index-write-gate.ts diff --git a/src/services/code-index/__tests__/file-completion.spec.ts b/src/services/code-index/__tests__/file-completion.spec.ts deleted file mode 100644 index 2ad8d3bbe4..0000000000 --- a/src/services/code-index/__tests__/file-completion.spec.ts +++ /dev/null @@ -1,691 +0,0 @@ -import "./mock-code-index-log" -import { v5 as uuidv5 } from "uuid" -import { FileCompletion, FileIndexWriter } from "../file-completion" -import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" -import { QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" -import type { CodeBlock, IVectorStore } from "../interfaces" -import type { CacheManager } from "../cache-manager" -import type { GenerationAllocator } from "../generation-allocator" -import { makeRateLimitScope, ProviderPacerRegistry } from "../provider-pacer" -import { FileFailureRegistry } from "../file-failure-registry" - -function deferred() { - let resolve!: (value: Value) => void - let reject!: (error: Error) => void - const promise = new Promise((complete, fail) => { - resolve = complete - reject = fail - }) - return { promise, resolve, reject } -} - -describe("FileCompletion", () => { - it("does not finish when an early batch settles before the final batch is registered", async () => { - const finish = vi.fn().mockResolvedValue(undefined) - const completion = new FileCompletion(finish) - completion.register() - await completion.settled(true) - expect(completion.outstanding).toBe(0) - expect(finish).not.toHaveBeenCalled() - completion.register() - await completion.registered() - expect(finish).not.toHaveBeenCalled() - await completion.settled(true) - expect(finish).toHaveBeenCalledOnce() - }) - - it("finishes an empty file once and rejects registration after the file is closed", async () => { - const finish = vi.fn().mockResolvedValue(undefined) - const completion = new FileCompletion(finish) - await completion.registered() - await completion.registered() - expect(finish).toHaveBeenCalledOnce() - expect(() => completion.register()).toThrow("registration already complete") - }) - - it.each(["before registration", "after registration"])("keeps a failed file incomplete %s", async (timing) => { - const finish = vi.fn() - const completion = new FileCompletion(finish) - completion.register() - completion.register() - if (timing === "after registration") await completion.registered() - await completion.settled(false) - if (timing === "before registration") await completion.registered() - await completion.settled(true) - expect(completion.failed).toBe(true) - expect(completion.outstanding).toBe(0) - expect(finish).not.toHaveBeenCalled() - }) - - it("shares a pending finish and never retries a failed finish", async () => { - const held = deferred() - const finish = vi.fn(() => held.promise) - const completion = new FileCompletion(finish) - const first = completion.registered() - const second = completion.registered() - const firstRejected = expect(first).rejects.toThrow("Verification failed") - const secondRejected = expect(second).rejects.toThrow("Verification failed") - expect(finish).toHaveBeenCalledOnce() - held.reject(new Error("Verification failed")) - await Promise.all([firstRejected, secondRejected]) - expect(completion.failed).toBe(true) - await completion.registered() - expect(finish).toHaveBeenCalledOnce() - }) -}) - -function createWriterDoubles() { - return { - cache: { - invalidateFile: vi.fn<(filePath: string) => Promise>().mockResolvedValue(undefined), - completeFile: vi.fn<(filePath: string, hash: string) => void>(), - completeRemoval: vi.fn<(filePath: string) => void>(), - }, - store: { - rateLimitScope: "qdrant:writer-test", - upsertPoints: vi.fn().mockResolvedValue(undefined), - deletePointsByFilePath: vi - .fn<(filePath: string, generation: number) => Promise>() - .mockResolvedValue(undefined), - countPointsForFile: vi - .fn<(filePath: string, generation: number) => Promise<{ current: number; other: number }>>() - .mockResolvedValue({ current: 2, other: 0 }), - }, - generations: { allocate: vi.fn().mockResolvedValue(17) }, - embedder: { - createEmbeddings: vi.fn(async (texts: string[]) => ({ embeddings: texts.map(() => [0.1, 0.2]) })), - rateLimitScope: "openai:writer-test", - validateConfiguration: vi.fn(async () => ({ valid: true })), - embedderInfo: { name: "openai" as const }, - }, - } -} - -function createWriter( - doubles: Pick, "cache" | "store" | "generations">, - gate: IndexWriteGate, - failures: FileFailureRegistry, -) { - // Doubles expose invalidation/completion, vector mutations/counts, and allocation without unrelated dependencies. - return new FileIndexWriter( - doubles.cache as unknown as CacheManager, - doubles.store as unknown as IVectorStore, - gate, - doubles.generations as unknown as GenerationAllocator, - "/workspace", - failures, - ) -} - -describe("FileIndexWriter", () => { - const filePath = "/workspace/file.ts" - let cache: ReturnType["cache"] - let store: ReturnType["store"] - let generations: ReturnType["generations"] - let embedder: ReturnType["embedder"] - let gate: IndexWriteGate - let attempt: AbortController - let writer: FileIndexWriter - let failures: FileFailureRegistry - - function blocks(count: number): CodeBlock[] { - return Array.from({ length: count }, (_, offset) => ({ - file_path: filePath, - identifier: null, - type: "function", - start_line: 1, - end_line: 1, - content: ` block ${offset} `, - fileHash: "hash", - segmentHash: `segment-${offset}`, - })) - } - - beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(100_000) - ;({ cache, store, generations, embedder } = createWriterDoubles()) - gate = new IndexWriteGate() - attempt = new AbortController() - failures = new FileFailureRegistry() - writer = createWriter({ cache, store, generations }, gate, failures) - }) - - afterEach(() => { - vi.clearAllTimers() - vi.useRealTimers() - }) - - it.each(["embed", "upsert", "delete", "verify", "remove"])( - "retries only %s at 500/1000 ms and preserves its generation", - async (activity) => { - const request = - activity === "embed" - ? embedder.createEmbeddings - : activity === "upsert" - ? store.upsertPoints - : activity === "verify" - ? store.countPointsForFile - : store.deletePointsByFilePath - request.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) - request.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) - - const records = vi.fn() - failures.onDidRecord(records) - const pending = - activity === "remove" - ? writer.remove(filePath, attempt.signal) - : writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - await vi.advanceTimersByTimeAsync(0) - expect(request).toHaveBeenCalledTimes(1) - expect(failures.list()[0]).toMatchObject({ attempt: 1, state: "retrying", classification: "server" }) - await vi.advanceTimersByTimeAsync(499) - expect(request).toHaveBeenCalledTimes(1) - await vi.advanceTimersByTimeAsync(1) - expect(request).toHaveBeenCalledTimes(2) - await vi.advanceTimersByTimeAsync(999) - expect(request).toHaveBeenCalledTimes(2) - await vi.advanceTimersByTimeAsync(1) - expect(request).toHaveBeenCalledTimes(3) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - await pending - expect(generations.allocate).toHaveBeenCalledOnce() - expect(embedder.createEmbeddings).toHaveBeenCalledTimes( - activity === "embed" ? 3 : activity === "remove" ? 0 : 1, - ) - expect(records).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - }, - ) - - it.each(["embed", "upsert"])("shares exactly two %s retries across batches", async (activity) => { - const request = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints - request.mockRejectedValueOnce({ status: 503 }).mockRejectedValueOnce({ status: 503 }) - if (activity === "embed") embedder.createEmbeddings.mockResolvedValueOnce({ embeddings: [[0.1]] }) - else store.upsertPoints.mockResolvedValueOnce(undefined) - request.mockRejectedValue({ status: 503 }) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1).catch((error) => error) - await vi.advanceTimersByTimeAsync(1500) - expect(await pending).toMatchObject({ activity, attempts: 3 }) - expect(request).toHaveBeenCalledTimes(4) - expect(failures.list()[0]).toMatchObject({ activity, attempt: 3, state: "exhausted" }) - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow() - expect(request).toHaveBeenCalledTimes(4) - }) - - it("does not charge successful batch first calls against retries", async () => { - await writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) - expect(store.upsertPoints).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - }) - - it("keeps exhausted store budgets across writer recreation and unrelated renewal", async () => { - store.upsertPoints.mockRejectedValue(Object.assign(new Error("unavailable"), { status: 503 })) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(1500) - expect(await pending).toMatchObject({ activity: "upsert", attempts: 3, classification: "server" }) - failures.renew("other.ts", "new") - const recreated = createWriter({ cache, store, generations }, gate, failures) - await expect(recreated.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( - "unavailable", - ) - expect(store.upsertPoints).toHaveBeenCalledTimes(3) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(failures.list()[0]).toMatchObject({ attempt: 3, state: "exhausted" }) - }) - - it.each(["allocation", "invalidation"])( - "exhausts a new revision after %s fails despite an old entry", - async (stage) => { - store.upsertPoints.mockRejectedValue(Object.assign(new Error("old server failure"), { status: 503 })) - const old = writer.replace(filePath, "r1", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(1500) - expect(await old).toMatchObject({ attempts: 3 }) - const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile - failingOperation.mockRejectedValue(new Error(`new ${stage} failure`)) - const records = vi.fn() - failures.onDidRecord(records) - - await expect(writer.replace(filePath, "r2", blocks(2), embedder, attempt.signal)).rejects.toThrow( - `new ${stage} failure`, - ) - const calls = failingOperation.mock.calls.length - await expect(writer.replace(filePath, "r2", blocks(2), embedder, attempt.signal)).rejects.toThrow( - `new ${stage} failure`, - ) - expect(failingOperation).toHaveBeenCalledTimes(calls) - expect(records).toHaveBeenCalledOnce() - expect(failures.list()[0]).toMatchObject({ - revision: "r2", - reason: `new ${stage} failure`, - attempt: 1, - state: "exhausted", - }) - expect(store.upsertPoints).toHaveBeenCalledTimes(3) - }, - ) - - it.each(["allocation", "invalidation"])( - "exhausts a retained successful retry when the same revision later fails %s", - async (stage) => { - failures.renew("file.ts", "hash") - const request = vi - .fn() - .mockRejectedValueOnce(Object.assign(new Error("temporary failure"), { status: 503 })) - const retry = writer.retryTransient(request, { - signal: attempt.signal, - file: filePath, - revision: "hash", - activity: "embed", - }) - await vi.advanceTimersByTimeAsync(500) - await retry - expect(failures.list()[0]).toMatchObject({ state: "retrying", attempt: 1 }) - const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile - failingOperation.mockRejectedValue(new Error(`${stage} failed`)) - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( - `${stage} failed`, - ) - expect(failures.list()[0]).toMatchObject({ - activity: "embed", - reason: "temporary failure", - attempt: 1, - classification: "server", - state: "exhausted", - }) - expect(cache.completeFile).not.toHaveBeenCalled() - }, - ) - - it.each(["allocation", "invalidation"])( - "exhausts recovered retries when completion fails before a later %s failure", - async (stage) => { - embedder.createEmbeddings.mockRejectedValueOnce( - Object.assign(new Error("temporary embedding failure"), { status: 503 }), - ) - cache.completeFile.mockImplementation(() => { - throw new Error("completion failed") - }) - const pending = writer - .replace(filePath, "hash", blocks(2), embedder, attempt.signal) - .catch((error) => error) - await vi.advanceTimersByTimeAsync(500) - expect(await pending).toMatchObject({ message: "completion failed" }) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) - expect(store.countPointsForFile).toHaveBeenCalledOnce() - const failure = { - activity: "embed", - reason: "temporary embedding failure", - attempt: 1, - classification: "server", - state: "exhausted", - } - expect(failures.list()[0]).toMatchObject(failure) - expect(failures.getFailure("file.ts", "hash", "embed")).toMatchObject(failure) - - const failingOperation = stage === "allocation" ? generations.allocate : cache.invalidateFile - failingOperation.mockRejectedValue(new Error(`later ${stage} failure`)) - const calls = failingOperation.mock.calls.length - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( - "temporary embedding failure", - ) - expect(failingOperation).toHaveBeenCalledTimes(calls) - expect(failures.list()[0]).toMatchObject(failure) - }, - ) - - it("T35 a different API key completes through the writer while the original allowance remains paused", async () => { - const pacers = new ProviderPacerRegistry() - const originalScope = makeRateLimitScope("openai", "https://api.example.test/v1", "first-test-key") - const independentScope = makeRateLimitScope("openai", "https://api.example.test/v1", "second-test-key") - const paused = pacers.get(originalScope) - paused.reportRateLimit({ status: 429, retryAfterMs: 30_000 }, paused.markIssued()) - const resumeAt = paused.state().pausedUntil - embedder.rateLimitScope = independentScope - // These doubles provide cache completion, store writes/counts, and generation allocation. - writer = new FileIndexWriter( - cache as unknown as CacheManager, - store as unknown as IVectorStore, - gate, - generations as unknown as GenerationAllocator, - "/workspace", - failures, - pacers, - ) - await writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(store.upsertPoints).toHaveBeenCalledOnce() - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - expect(paused.state().pausedUntil).toBe(resumeAt) - expect(pacers.get(independentScope).state().pausedUntil).toBeUndefined() - expect(failures.list()).toEqual([]) - }) - - it("T41 an upsert aborted inside a paused pacer never enters the gate or incurs grace", async () => { - const pacers = new ProviderPacerRegistry() - // These doubles provide cache completion, store writes/counts, and generation allocation. - writer = new FileIndexWriter( - cache as unknown as CacheManager, - store as unknown as IVectorStore, - gate, - generations as unknown as GenerationAllocator, - "/workspace", - failures, - pacers, - ) - const pacer = pacers.get(store.rateLimitScope) - pacer.reportRateLimit({ status: 429, retryAfterMs: 30_000 }, pacer.markIssued()) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(0) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(pacer.state().waitingPaths).toEqual(["file.ts"]) - expect(gate.hasPending(attempt.signal)).toBe(false) - expect(store.upsertPoints).not.toHaveBeenCalled() - attempt.abort() - expect(await pending).toBe(attempt.signal.reason) - expect(gate.uncertainFiles(attempt.signal)).toEqual([]) - expect(gate.graceDeadline(filePath)).toBe(0) - expect(gate.needsReconciliation(filePath)).toBe(false) - await gate.settled(attempt.signal) - await vi.advanceTimersByTimeAsync(60_000) - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }) - - it.each(["permanent", "abort", "revision"])("ends automatic attempts for %s", async (ending) => { - embedder.createEmbeddings.mockRejectedValue( - Object.assign(new Error("failed"), { - status: ending === "permanent" ? 401 : 503, - }), - ) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(0) - if (ending === "abort") attempt.abort() - if (ending === "revision") { - failures.renew("file.ts", "new") - failures.record("file.ts", "new", "read", "new failure", 1, "exhausted") - } - await vi.advanceTimersByTimeAsync(1500) - await pending - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(failures.list()[0].state).toBe("exhausted") - if (ending === "revision") expect(failures.list()[0].reason).toBe("new failure") - }) - - it("discards issued old-revision embeddings without mutating the new failure", async () => { - const held = deferred<{ embeddings: number[][] }>() - embedder.createEmbeddings.mockReturnValueOnce(held.promise) - const pending = writer.replace(filePath, "old", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(0) - failures.renew("file.ts", "new") - failures.record("file.ts", "new", "parse", "new failure", 1, "exhausted") - held.resolve({ embeddings: [[1], [1]] }) - expect(await pending).toMatchObject({ name: "AbortError" }) - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(failures.list()[0].reason).toBe("new failure") - }) - - it("waits for an issued superseded revision save before completing the newer revision", async () => { - const held = deferred() - store.upsertPoints.mockReturnValueOnce(held.promise) - const old = writer.replace(filePath, "old", blocks(2), embedder, attempt.signal).catch((error) => error) - await vi.advanceTimersByTimeAsync(0) - const current = writer.replace(filePath, "new", blocks(2), embedder, attempt.signal) - await vi.advanceTimersByTimeAsync(0) - expect(store.upsertPoints).toHaveBeenCalledTimes(2) - expect(cache.completeFile).not.toHaveBeenCalled() - held.resolve() - await current - expect(await old).toMatchObject({ name: "AbortError" }) - expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(filePath, "new") - expect(store.countPointsForFile).toHaveBeenCalledTimes(2) - }) - - it("awaits durable invalidation before any embedding or vector mutation", async () => { - const held = deferred() - cache.invalidateFile.mockReturnValue(held.promise) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - await vi.advanceTimersByTimeAsync(0) - expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - held.resolve() - await pending - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - }) - - it.each(["replace", "remove"] as const)( - "issues no vector mutation when %s invalidation fails", - async (operation) => { - cache.invalidateFile.mockRejectedValue(new Error("Disk full")) - const pending = - operation === "replace" - ? writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - : writer.remove(filePath, attempt.signal) - await expect(pending).rejects.toThrow("Disk full") - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(cache.completeRemoval).not.toHaveBeenCalled() - expect(gate.needsReconciliation(filePath)).toBe(false) - expect(failures.list()).toEqual([ - expect.objectContaining({ - path: "file.ts", - revision: operation === "replace" ? "hash" : "removed", - activity: operation === "replace" ? "upsert" : "delete", - reason: "Disk full", - state: "exhausted", - }), - ]) - }, - ) - - it.each(["replace", "remove"] as const)("records %s allocation failures before any mutation", async (operation) => { - generations.allocate.mockRejectedValue(new Error("watermark save failed")) - const pending = - operation === "replace" - ? writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - : writer.remove(filePath, attempt.signal) - await expect(pending).rejects.toThrow("watermark save failed") - expect(cache.invalidateFile).not.toHaveBeenCalled() - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(failures.list()[0]).toMatchObject({ - path: "file.ts", - activity: operation === "replace" ? "upsert" : "delete", - reason: "watermark save failed", - state: "exhausted", - classification: "permanent", - }) - }) - - it("upserts every registered batch before fenced cleanup, verification, hash and progress", async () => { - const finalBatch = deferred() - store.upsertPoints.mockResolvedValueOnce(undefined).mockReturnValueOnce(finalBatch.promise) - const onComplete = vi.fn() - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) - await vi.advanceTimersByTimeAsync(0) - expect(store.upsertPoints).toHaveBeenCalledTimes(2) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - finalBatch.resolve() - await pending - expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) - expect(store.countPointsForFile).toHaveBeenCalledWith(filePath, 17) - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - expect(onComplete).toHaveBeenCalledWith(2) - expect(store.upsertPoints.mock.invocationCallOrder[1]).toBeLessThan( - store.deletePointsByFilePath.mock.invocationCallOrder[0], - ) - expect(store.countPointsForFile.mock.invocationCallOrder[0]).toBeLessThan( - cache.completeFile.mock.invocationCallOrder[0], - ) - expect(embedder.createEmbeddings).toHaveBeenNthCalledWith(1, ["block 0"], undefined, expect.any(AbortSignal)) - const points = store.upsertPoints.mock.calls.flatMap(([batch]) => batch) - expect(points.map((point) => point.id)).toEqual([ - uuidv5("segment-0:17", QDRANT_CODE_BLOCK_NAMESPACE), - uuidv5("segment-1:17", QDRANT_CODE_BLOCK_NAMESPACE), - ]) - expect(points[0].payload).toMatchObject({ filePath: "file.ts", generation: 17, segmentHash: "segment-0" }) - expect(new Set(points.map((point) => point.id)).size).toBe(2) - }) - - it.each([ - { current: 1, other: 0 }, - { current: 2, other: 1 }, - ])("does not complete a file with missing or stale points (%j)", async (counts) => { - store.countPointsForFile.mockResolvedValue(counts) - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( - "verification failed", - ) - expect(cache.completeFile).not.toHaveBeenCalled() - }) - - it("cleans and verifies a zero-content file without embedding", async () => { - store.countPointsForFile.mockResolvedValue({ current: 0, other: 0 }) - const whitespace = blocks(1) - whitespace[0].content = " \n\t" - await writer.replace(filePath, "hash", whitespace, embedder, attempt.signal) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - }) - - it("filters only whitespace blocks from mixed content and types", async () => { - const mixed = blocks(4) - mixed[0].type = "class" - mixed[1].content = " \n" - mixed[2].type = "method" - mixed[3].type = "interface" - mixed[3].content = "\t" - store.countPointsForFile.mockResolvedValue({ current: 2, other: 0 }) - const onComplete = vi.fn() - await writer.replace(filePath, "hash", mixed, embedder, attempt.signal, undefined, onComplete) - expect(embedder.createEmbeddings.mock.calls.flatMap(([texts]) => texts)).toEqual(["block 0", "block 2"]) - const points = store.upsertPoints.mock.calls.flatMap(([batch]) => batch) - expect(points.map((point) => point.payload)).toEqual([ - expect.objectContaining({ codeChunk: " block 0 ", segmentHash: "segment-0" }), - expect.objectContaining({ codeChunk: " block 2 ", segmentHash: "segment-2" }), - ]) - expect(onComplete).toHaveBeenCalledExactlyOnceWith(2) - }) - - it("does not issue cleanup or completion after a failed batch", async () => { - store.upsertPoints.mockRejectedValueOnce(new Error("Upsert failed")) - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow( - "Upsert failed", - ) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(1) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(gate.needsReconciliation(filePath)).toBe(true) - }) - - it("rejects incomplete embeddings without issuing an upsert", async () => { - embedder.createEmbeddings.mockResolvedValue({ embeddings: [[0.1]] }) - await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( - "Incomplete embeddings", - ) - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - }) - - it("discards late embedding results after cancellation without issuing subsequent batches", async () => { - const held = deferred<{ embeddings: number[][] }>() - embedder.createEmbeddings.mockReturnValueOnce(held.promise) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1) - const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) - await vi.advanceTimersByTimeAsync(0) - attempt.abort() - held.resolve({ embeddings: [[0.1]] }) - await rejected - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - }) - - it.each(["complete", "stale", "cancel"])( - "keeps an uncertain file incomplete through grace (%s)", - async (outcome) => { - gate.recover([filePath]) - const onComplete = vi.fn() - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) - const observed = pending.catch((error) => error) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(store.countPointsForFile).toHaveBeenCalledTimes(1) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(cache.completeRemoval).not.toHaveBeenCalled() - expect(onComplete).not.toHaveBeenCalled() - if (outcome === "cancel") attempt.abort() - if (outcome === "stale") store.countPointsForFile.mockResolvedValue({ current: 2, other: 1 }) - await vi.advanceTimersByTimeAsync(1) - const result = await observed - if (outcome === "complete") { - expect(result).toBeUndefined() - expect(store.countPointsForFile).toHaveBeenCalledTimes(2) - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - expect(onComplete).toHaveBeenCalledOnce() - expect(gate.needsReconciliation(filePath)).toBe(false) - } else { - expect(result).toMatchObject( - outcome === "cancel" - ? { name: "AbortError" } - : { message: expect.stringContaining("verification failed") }, - ) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(onComplete).not.toHaveBeenCalled() - expect(gate.needsReconciliation(filePath)).toBe(true) - } - }, - ) - - it("extends grace when another uncertainty arrives during final verification", async () => { - gate.recover([filePath]) - const heldCounts = deferred<{ current: number; other: number }>() - store.countPointsForFile.mockResolvedValueOnce({ current: 2, other: 0 }).mockReturnValueOnce(heldCounts.promise) - const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - expect(store.countPointsForFile).toHaveBeenCalledTimes(2) - gate.recover([filePath]) - heldCounts.resolve({ current: 2, other: 0 }) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(cache.completeFile).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await pending - expect(store.countPointsForFile).toHaveBeenCalledTimes(3) - expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") - }) - - it("invalidates a completed file before removal and drops its open record only after success", async () => { - const invalidation = deferred() - const deletion = deferred() - cache.invalidateFile.mockReturnValue(invalidation.promise) - store.deletePointsByFilePath.mockReturnValue(deletion.promise) - const pending = writer.remove(filePath, attempt.signal) - await vi.advanceTimersByTimeAsync(0) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - invalidation.resolve() - await vi.advanceTimersByTimeAsync(0) - expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) - expect(cache.completeRemoval).not.toHaveBeenCalled() - deletion.resolve() - await pending - expect(cache.completeRemoval).toHaveBeenCalledWith(filePath) - }) - - it("retains the open record after an uncertain removal failure", async () => { - store.deletePointsByFilePath.mockRejectedValue(new DOMException("Lost result", "AbortError")) - await expect(writer.remove(filePath, attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) - expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) - expect(cache.completeRemoval).not.toHaveBeenCalled() - expect(gate.needsReconciliation(filePath)).toBe(true) - }) -}) diff --git a/src/services/code-index/__tests__/file-failure-registry.spec.ts b/src/services/code-index/__tests__/file-failure-registry.spec.ts index 8081684e5d..736ee3425b 100644 --- a/src/services/code-index/__tests__/file-failure-registry.spec.ts +++ b/src/services/code-index/__tests__/file-failure-registry.spec.ts @@ -1,7 +1,7 @@ import { FileFailureRegistry } from "../file-failure-registry" describe("FileFailureRegistry", () => { - it("shares retry counters only within one path, revision and operation", () => { + it("CI-FR-01/02: batches of one revision share a budget; a new revision or reset refills it", () => { const registry = new FileFailureRegistry() registry.renew("a.ts", "one") registry.renew("b.ts", "one") @@ -52,7 +52,7 @@ describe("FileFailureRegistry", () => { registry.resetAllBudgets() expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) expect(registry.has("a.ts")).toBe(true) - registry.record("a.ts", "two", "verify", "failed again", 1, "exhausted") + registry.record("a.ts", "two", "delete", "failed again", 1, "exhausted") registry.clear("a.ts", "two") expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) }) @@ -78,8 +78,8 @@ describe("FileFailureRegistry", () => { const subscription = registry.onDidRecord(records) registry.onDidChange(changed) registry.renew("a.ts", "hash") - registry.record("a.ts", "hash", "verify", "busy", 1, "retrying", "rate-limit") - registry.record("a.ts", "hash", "verify", "busy", 2, "retrying", "rate-limit") + registry.record("a.ts", "hash", "upsert", "busy", 1, "retrying", "server") + registry.record("a.ts", "hash", "upsert", "busy", 2, "retrying", "server") expect(records).toHaveBeenCalledTimes(2) expect(changed).toHaveBeenCalledTimes(2) registry.list()[0].reason = "mutated" @@ -88,4 +88,13 @@ describe("FileFailureRegistry", () => { registry.clear("a.ts") expect(records).toHaveBeenCalledTimes(2) }) + it("CI-FR-03: a success clears the entry for that path only", () => { + const registry = new FileFailureRegistry() + registry.renew("a.ts", "one") + registry.renew("b.ts", "one") + registry.record("a.ts", "one", "read", "denied", 1, "exhausted") + registry.record("b.ts", "one", "parse", "bad", 1, "exhausted") + registry.clear("a.ts", "one") + expect(registry.list().map((failure) => failure.path)).toEqual(["b.ts"]) + }) }) diff --git a/src/services/code-index/__tests__/file-index-writer.spec.ts b/src/services/code-index/__tests__/file-index-writer.spec.ts new file mode 100644 index 0000000000..17e4d510bf --- /dev/null +++ b/src/services/code-index/__tests__/file-index-writer.spec.ts @@ -0,0 +1,118 @@ +import { FileIndexWriter } from "../file-index-writer" +import { FileFailureRegistry } from "../file-failure-registry" +import { ProviderPacerRegistry } from "../provider-pacer" +import type { CodeBlock, IEmbedder, IVectorStore } from "../interfaces" + +vi.mock("../embedders/cancellation", () => ({ waitForRetry: vi.fn(() => Promise.resolve()) })) + +const workspace = "/ws" +const block = (line: number): CodeBlock => ({ + file_path: "/ws/a.ts", + identifier: null, + type: "function", + start_line: line, + end_line: line + 1, + content: `code ${line}`, + fileHash: "h1", + segmentHash: `s${line}`, +}) +const serverError = () => Object.assign(new Error("server down"), { status: 503 }) + +function setup() { + const embedder = { + rateLimitScope: "openai:1", + createEmbeddings: vi.fn(async (texts: string[]) => ({ embeddings: texts.map(() => [0.1]) })), + } + const store = { + rateLimitScope: "qdrant:1", + upsertPoints: vi.fn(async () => {}), + deletePointsByFilePath: vi.fn(async () => {}), + } + const cache = { updateHash: vi.fn(), deleteHash: vi.fn() } + const failures = new FileFailureRegistry() + const pacers = new ProviderPacerRegistry() + const run = vi.spyOn(pacers.get("openai:1"), "run") + // The doubles provide only the embedder and store members the writer calls. + const writer = new FileIndexWriter( + embedder as unknown as IEmbedder, + store as unknown as IVectorStore, + cache, + failures, + pacers, + workspace, + 1, + ) + return { writer, embedder, store, cache, failures, run } +} + +describe("FileIndexWriter", () => { + it("CI-WR-01: success updates the hash and clears the entry", async () => { + const { writer, cache, failures, store } = setup() + failures.renew("a.ts", "h1") + failures.record("a.ts", "h1", "read", "old", 1, "exhausted") + await expect(writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(true) + expect(cache.updateHash).toHaveBeenCalledWith("/ws/a.ts", "h1") + expect(failures.list()).toEqual([]) + const [[points]] = store.upsertPoints.mock.calls as unknown as [[Array<{ payload: { filePath: string } }>]] + expect(points[0].payload.filePath).toBe("a.ts") + }) + + it("uses the same point id for the same path and start line", async () => { + const { writer, store } = setup() + await writer.replace("/ws/a.ts", "h1", [block(3)]) + await writer.replace("/ws/a.ts", "h2", [{ ...block(3), content: "changed", segmentHash: "other" }]) + const ids = (store.upsertPoints.mock.calls as unknown as Array<[Array<{ id: string }>]>).map( + ([points]) => points[0].id, + ) + expect(ids[0]).toBe(ids[1]) + }) + + it.each(["embed", "upsert"] as const)( + "CI-WR-02: exhausted %s records the activity and keeps the hash", + async (activity) => { + const context = setup() + if (activity === "embed") context.embedder.createEmbeddings.mockRejectedValue(serverError()) + else context.store.upsertPoints.mockRejectedValue(serverError()) + await expect(context.writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(false) + expect(context.cache.updateHash).not.toHaveBeenCalled() + expect(context.failures.get("a.ts")).toMatchObject({ activity, state: "exhausted", attempt: 3 }) + }, + ) + + it("CI-WR-02: removal failure is recorded as delete and keeps the hash", async () => { + const { writer, store, cache, failures } = setup() + store.deletePointsByFilePath.mockRejectedValue(serverError()) + await expect(writer.remove("/ws/a.ts")).resolves.toBe(false) + expect(cache.deleteHash).not.toHaveBeenCalled() + expect(failures.get("a.ts")).toMatchObject({ activity: "delete", state: "exhausted" }) + }) + + it("CI-WR-03: each embed attempt goes through the pacer exactly once", async () => { + const { writer, embedder, run } = setup() + embedder.createEmbeddings.mockRejectedValueOnce(serverError()) + await writer.replace("/ws/a.ts", "h1", [block(1), block(5)]) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(3) + expect(run).toHaveBeenCalledTimes(3) + }) + + it("CI-WR-04: writes nothing to the registry or cache after the signal aborts", async () => { + const { writer, embedder, cache, failures } = setup() + const controller = new AbortController() + embedder.createEmbeddings.mockImplementation(async () => { + controller.abort() + throw serverError() + }) + await expect(writer.replace("/ws/a.ts", "h1", [block(1)], controller.signal)).rejects.toBeDefined() + expect(cache.updateHash).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }) + + it("removal success deletes the hash and clears prior failures", async () => { + const { writer, cache, failures } = setup() + failures.renew("a.ts", "h1") + failures.record("a.ts", "h1", "embed", "old", 3, "exhausted") + await expect(writer.remove("/ws/a.ts")).resolves.toBe(true) + expect(cache.deleteHash).toHaveBeenCalledWith("/ws/a.ts") + expect(failures.list()).toEqual([]) + }) +}) diff --git a/src/services/code-index/__tests__/generation-allocator.spec.ts b/src/services/code-index/__tests__/generation-allocator.spec.ts deleted file mode 100644 index a83cd8d1d9..0000000000 --- a/src/services/code-index/__tests__/generation-allocator.spec.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { GenerationAllocator } from "../generation-allocator" -import { IndexWriteGate } from "../index-write-gate" -import type { CacheManager } from "../cache-manager" -import type { IVectorStore } from "../interfaces" - -function setup(watermark: number | undefined = 0) { - const cache = { - generationWatermark: watermark as number | undefined, - clearCacheFile: vi.fn<() => Promise>().mockResolvedValue(undefined), - reserveGenerations: vi.fn(async (nextWatermark: number) => { - cache.generationWatermark = nextWatermark - }), - } - const store = { - hasIndexedData: vi.fn<() => Promise>().mockResolvedValue(false), - clearCollection: vi.fn<() => Promise>().mockResolvedValue(undefined), - } - // Doubles supply only durable generation reservation/reset and collection inspection/reset. - const allocator = new GenerationAllocator(cache as unknown as CacheManager) - const vectorStore = store as unknown as IVectorStore - const signal = new AbortController().signal - const gate = new IndexWriteGate() - return { cache, store, allocator, vectorStore, signal, gate } -} - -describe("GenerationAllocator", () => { - it("awaits durable reservation before issuing a generation", async () => { - const { cache, allocator, vectorStore, signal, gate } = setup() - let release!: () => void - const reservation = new Promise((resolve) => { - release = resolve - }) - cache.reserveGenerations.mockReturnValueOnce(reservation) - const initialized = vi.fn() - const pending = allocator.initialize(vectorStore, signal, gate).then(initialized) - await Promise.resolve() - expect(cache.reserveGenerations).toHaveBeenCalledWith(1024) - expect(initialized).not.toHaveBeenCalled() - release() - await pending - await expect(allocator.allocate()).resolves.toBe(1) - }) - - it("serializes allocations and reserves only when the 1024-value block runs out", async () => { - const { cache, allocator, vectorStore, signal, gate } = setup() - await allocator.initialize(vectorStore, signal, gate) - const generations = await Promise.all(Array.from({ length: 1024 }, () => allocator.allocate())) - expect(generations).toEqual(Array.from({ length: 1024 }, (_, offset) => offset + 1)) - expect(cache.reserveGenerations).toHaveBeenCalledExactlyOnceWith(1024) - await expect(allocator.allocate()).resolves.toBe(1025) - expect(cache.reserveGenerations).toHaveBeenLastCalledWith(2048) - }) - - it("resumes above the entire durable block without consulting the clock", async () => { - const { cache, allocator, vectorStore, signal, gate } = setup(4096) - await allocator.initialize(vectorStore, signal, gate) - await expect(allocator.allocate()).resolves.toBe(4097) - expect(cache.reserveGenerations).toHaveBeenCalledWith(5120) - expect(cache.clearCacheFile).not.toHaveBeenCalled() - }) - - it.each([false, true])("resets unknown ordering with indexed data %s", async (hasData) => { - const { cache, store, allocator, vectorStore, signal, gate } = setup() - cache.generationWatermark = undefined - store.hasIndexedData.mockResolvedValue(hasData) - await allocator.initialize(vectorStore, signal, gate) - expect(store.clearCollection).toHaveBeenCalledTimes(hasData ? 1 : 0) - expect(cache.clearCacheFile).toHaveBeenCalledOnce() - expect(cache.reserveGenerations).toHaveBeenCalledWith(1024) - }) - - it("refuses initialization after reset failure without reserving", async () => { - const { cache, store, allocator, vectorStore, signal, gate } = setup() - cache.generationWatermark = undefined - store.hasIndexedData.mockResolvedValue(true) - store.clearCollection.mockRejectedValue(new Error("reset failed")) - await expect(allocator.initialize(vectorStore, signal, gate)).rejects.toThrow("reset failed") - expect(cache.reserveGenerations).not.toHaveBeenCalled() - }) - - it("recovers the allocation queue after reservation failure", async () => { - const { cache, allocator } = setup(1024) - cache.reserveGenerations.mockRejectedValueOnce(new Error("disk full")) - await expect(allocator.allocate()).rejects.toThrow("disk full") - await expect(allocator.allocate()).resolves.toBe(1025) - expect(cache.reserveGenerations).toHaveBeenLastCalledWith(2048) - }) - - it("rejects cancellation without consuming a generation", async () => { - const { allocator, vectorStore, signal, gate } = setup() - await allocator.initialize(vectorStore, signal, gate) - await expect(allocator.allocate(AbortSignal.abort())).rejects.toMatchObject({ name: "AbortError" }) - await expect(allocator.allocate(signal)).resolves.toBe(1) - }) - - it("rejects unsafe integer exhaustion before persistence", async () => { - const { cache, allocator, vectorStore, signal, gate } = setup(Number.MAX_SAFE_INTEGER) - await expect(allocator.initialize(vectorStore, signal, gate)).rejects.toThrow("counter exhausted") - expect(cache.reserveGenerations).not.toHaveBeenCalled() - }) -}) diff --git a/src/services/code-index/__tests__/index-write-gate.spec.ts b/src/services/code-index/__tests__/index-write-gate.spec.ts deleted file mode 100644 index 350744a55e..0000000000 --- a/src/services/code-index/__tests__/index-write-gate.spec.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" - -function deferred() { - let resolve!: () => void - let reject!: (error: Error) => void - const promise = new Promise((complete, fail) => { - resolve = complete - reject = fail - }) - return { promise, resolve, reject } -} - -describe("IndexWriteGate", () => { - beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(100_000) - }) - - afterEach(() => { - vi.clearAllTimers() - vi.useRealTimers() - }) - - it.each(["already aborted", "aborted before issue"])("refuses %s writes without uncertainty", async (timing) => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - const write = vi.fn() - if (timing === "already aborted") attempt.abort() - const rejected = expect(gate.run(attempt.signal, write, ["file.ts"])).rejects.toMatchObject({ - name: "AbortError", - }) - attempt.abort() - await rejected - expect(write).not.toHaveBeenCalled() - expect(gate.uncertainFiles(attempt.signal)).toEqual([]) - expect(gate.needsReconciliation("file.ts")).toBe(false) - expect(gate.hasPending(attempt.signal)).toBe(false) - }) - - it.each(["resolve", "reject"] as const)( - "never waits across attempts when an old write will %s", - async (outcome) => { - const gate = new IndexWriteGate() - const oldAttempt = new AbortController() - const nextAttempt = new AbortController() - const held = deferred() - const oldWrite = vi.fn(() => held.promise) - const pending = gate.run(oldAttempt.signal, oldWrite, ["file.ts"]).catch((error) => error) - await Promise.resolve() - expect(oldWrite).toHaveBeenCalledOnce() - oldAttempt.abort() - const newWrite = vi.fn(() => "current") - await expect(gate.run(nextAttempt.signal, newWrite, ["retry.ts"])).resolves.toBe("current") - expect(newWrite).toHaveBeenCalledOnce() - expect(gate.hasPending(oldAttempt.signal)).toBe(true) - expect(gate.hasPending(nextAttempt.signal)).toBe(false) - if (outcome === "resolve") held.resolve() - else held.reject(new Error("Request timeout")) - await pending - expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(outcome === "reject" ? ["file.ts"] : []) - expect(gate.needsReconciliation("file.ts")).toBe(outcome === "reject") - expect(gate.needsReconciliation("retry.ts")).toBe(false) - }, - ) - - it("discards a successful write after abort without extending an existing uncertainty deadline", async () => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - gate.recover(["file.ts"]) - const deadline = gate.graceDeadline("file.ts") - await vi.advanceTimersByTimeAsync(1000) - await expect( - gate.run( - attempt.signal, - async () => { - attempt.abort() - return "remote success" - }, - ["file.ts"], - ), - ).rejects.toMatchObject({ name: "AbortError" }) - expect(gate.uncertainFiles(attempt.signal)).toEqual([]) - expect(gate.graceDeadline("file.ts")).toBe(deadline) - }) - - it.each([new Error("Transport failure"), new DOMException("Cancelled remotely", "AbortError")])( - "marks issued rejection uncertain even for $name", - async (failure) => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - await expect( - gate.run(attempt.signal, () => Promise.reject(failure), ["first.ts", "second.ts"]), - ).rejects.toBe(failure) - expect(gate.uncertainFiles(attempt.signal)).toEqual(["first.ts", "second.ts"]) - const done = vi.fn() - const grace = gate.waitForGrace("first.ts", attempt.signal).then(done) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(done).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await grace - expect(done).toHaveBeenCalledOnce() - expect(gate.needsReconciliation("first.ts")).toBe(true) - gate.reconciled("first.ts") - expect(gate.needsReconciliation("first.ts")).toBe(false) - }, - ) - - it("starts rejection grace when rejection is observed, not when the request was issued", async () => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - const held = deferred() - const pending = gate.run(attempt.signal, () => held.promise, ["file.ts"]).catch(() => {}) - await vi.advanceTimersByTimeAsync(50_000) - held.reject(new Error("Timeout")) - await pending - const done = vi.fn() - const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(done).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await grace - }) - - it("starts stop-expiry grace on abandon and tracks only pending writes from that attempt", async () => { - const gate = new IndexWriteGate() - const oldAttempt = new AbortController() - const otherAttempt = new AbortController() - const held = deferred() - const otherHeld = deferred() - const pending = gate.run(oldAttempt.signal, () => held.promise, ["old.ts"]).catch(() => {}) - const otherPending = gate.run(otherAttempt.signal, () => otherHeld.promise, ["other.ts"]) - await vi.advanceTimersByTimeAsync(2000) - oldAttempt.abort() - gate.abandon(oldAttempt.signal) - expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(["old.ts"]) - expect(gate.needsReconciliation("other.ts")).toBe(false) - const done = vi.fn() - const grace = gate.waitForGrace("old.ts", otherAttempt.signal).then(done) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(done).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await grace - held.resolve() - otherHeld.resolve() - await Promise.all([pending, otherPending]) - }) - - it("uses cache-load time and extends an active wait without shortening its deadline", async () => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - gate.recover(["file.ts"], Date.now() - 10_000) - const done = vi.fn() - const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) - await vi.advanceTimersByTimeAsync(5000) - gate.recover(["file.ts"]) - gate.recover(["file.ts"], Date.now() - 50_000) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) - expect(done).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await grace - expect(done).toHaveBeenCalledOnce() - }) - - it("cancels grace promptly and cleans up its timer while retaining uncertainty", async () => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - gate.recover(["file.ts"]) - const rejected = expect(gate.waitForGrace("file.ts", attempt.signal)).rejects.toMatchObject({ - name: "AbortError", - }) - expect(vi.getTimerCount()).toBe(1) - attempt.abort() - await rejected - expect(vi.getTimerCount()).toBe(0) - expect(gate.needsReconciliation("file.ts")).toBe(true) - await expect(gate.waitForGrace("unaffected.ts", attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) - }) - - it("tracks combined watcher cancellation under its orchestrator attempt", async () => { - const gate = new IndexWriteGate() - const owner = new AbortController() - const local = new AbortController() - const combined = AbortSignal.any([owner.signal, local.signal]) - gate.own(combined, owner.signal) - const held = deferred() - const pending = gate.run(combined, () => held.promise, ["watcher.ts"]).catch(() => {}) - await Promise.resolve() - expect(gate.hasPending(owner.signal)).toBe(true) - owner.abort() - gate.abandon(owner.signal) - expect(gate.uncertainFiles(owner.signal)).toEqual(["watcher.ts"]) - held.resolve() - await pending - await gate.settled(owner.signal) - expect(gate.hasPending(owner.signal)).toBe(false) - }) - - it("completion excludes its current owner, live attempts and unrelated aborted files", async () => { - const gate = new IndexWriteGate() - const current = new AbortController() - const local = new AbortController() - const combined = AbortSignal.any([current.signal, local.signal]) - gate.own(combined, current.signal) - const live = new AbortController() - const old = new AbortController() - const held = deferred() - const writes = [ - gate.run(current.signal, () => held.promise, ["file.ts"]), - gate.run(live.signal, () => held.promise, ["file.ts"]), - gate.run(old.signal, () => held.promise, ["unrelated.ts"]).catch(() => {}), - ] - await Promise.resolve() - old.abort() - await expect(gate.waitForPriorWrites("file.ts", combined)).resolves.toBe(false) - expect(gate.graceDeadline("file.ts")).toBe(0) - held.resolve() - await Promise.all(writes) - }) - - it("bounds a prior aborted write without an orchestrator stop and never repeats abandonment", async () => { - const gate = new IndexWriteGate() - const old = new AbortController() - const current = new AbortController() - const held = deferred() - const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) - await Promise.resolve() - old.abort() - const done = vi.fn() - const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) - await vi.advanceTimersByTimeAsync(1999) - expect(done).not.toHaveBeenCalled() - expect(gate.graceDeadline("file.ts")).toBe(0) - await vi.advanceTimersByTimeAsync(1) - await wait - expect(done).toHaveBeenCalledWith(true) - expect(gate.graceDeadline("file.ts")).toBe(132_000) - await vi.advanceTimersByTimeAsync(1000) - gate.abandon(old.signal) - await expect(gate.waitForPriorWrites("file.ts", current.signal)).resolves.toBe(false) - expect(gate.graceDeadline("file.ts")).toBe(132_000) - held.reject(new Error("Late rejection")) - await pending - expect(gate.graceDeadline("file.ts")).toBe(133_000) - }) - - it("uses the orchestrator expiry without rebuilding it on abort", async () => { - const gate = new IndexWriteGate() - const old = new AbortController() - const current = new AbortController() - const held = deferred() - const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) - await Promise.resolve() - gate.stopAt(old.signal, Date.now() + 2000) - await vi.advanceTimersByTimeAsync(500) - old.abort() - const done = vi.fn() - const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) - await vi.advanceTimersByTimeAsync(1499) - expect(done).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await wait - expect(gate.graceDeadline("file.ts")).toBe(132_000) - held.resolve() - await pending - }) - - it("uses the abort stop deadline even when completion starts after its expiry", async () => { - const gate = new IndexWriteGate() - const old = new AbortController() - const current = new AbortController() - const held = deferred() - const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) - await Promise.resolve() - old.abort() - await vi.advanceTimersByTimeAsync(5000) - await gate.waitForPriorWrites("file.ts", current.signal) - expect(gate.graceDeadline("file.ts")).toBe(132_000) - held.resolve() - await pending - }) - - it("cancels the prior-outcome wait and clears its timer without creating uncertainty", async () => { - const gate = new IndexWriteGate() - const old = new AbortController() - const current = new AbortController() - const held = deferred() - const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) - await Promise.resolve() - old.abort() - const wait = expect(gate.waitForPriorWrites("file.ts", current.signal)).rejects.toMatchObject({ - name: "AbortError", - }) - expect(vi.getTimerCount()).toBe(1) - current.abort() - await wait - expect(vi.getTimerCount()).toBe(0) - expect(gate.graceDeadline("file.ts")).toBe(0) - held.resolve() - await pending - }) - - it("settles only its own attempt and successful writes introduce no uncertainty", async () => { - const gate = new IndexWriteGate() - const attempt = new AbortController() - const other = new AbortController() - const held = deferred() - const pending = gate.run(other.signal, () => held.promise, ["other.ts"]) - await gate.run(attempt.signal, () => 42, ["file.ts"]) - await gate.settled(attempt.signal) - expect(gate.hasPending(other.signal)).toBe(true) - const done = vi.fn() - const settled = gate.settled(other.signal).then(done) - await Promise.resolve() - expect(done).not.toHaveBeenCalled() - held.resolve() - await Promise.all([pending, settled]) - expect(gate.needsReconciliation("file.ts")).toBe(false) - expect(gate.uncertainFiles(other.signal)).toEqual([]) - }) -}) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 6b8c05c69b..a9a629b1df 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -1,418 +1,153 @@ -import { createHash } from "node:crypto" import { ProviderPacer, ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" -import { codeIndexLog } from "../code-index-log" -vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) +const rateLimit = (retryAfterMs?: number) => Object.assign(new Error("limited"), { status: 429, retryAfterMs }) -function deferred() { - let resolve!: (value: T) => void - let reject!: (error: unknown) => void - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - return { promise, resolve, reject } +async function settle(promise: Promise) { + return promise.then( + (value) => ({ value }), + (error: unknown) => ({ error }), + ) } -const limited = (retryAfterMs?: number) => ({ status: 429, retryAfterMs }) -const flush = () => vi.advanceTimersByTimeAsync(0) - -beforeEach(() => { - vi.mocked(codeIndexLog.info).mockClear() - vi.useFakeTimers() - vi.setSystemTime(0) - vi.spyOn(Math, "random").mockReturnValue(0) -}) - -afterEach(() => { - vi.clearAllTimers() - vi.useRealTimers() - vi.restoreAllMocks() -}) - -describe("ProviderPacer", () => { - it("T35 shares a 30s pause, keeps in-flight results, and isolates allowance keys and stores", async () => { - const registry = new ProviderPacerRegistry() - const scope = makeRateLimitScope("openai", "https://example.test", "first") - const pacer = registry.get(scope) - expect(registry.get(scope)).toBe(pacer) - const inFlight = deferred() - const kept = pacer.run("kept.ts", undefined, () => inFlight.promise) - const first = vi.fn().mockRejectedValueOnce(limited(30_000)).mockResolvedValue("first") - const firstResult = pacer.run("a.ts", undefined, first) - await flush() - const second = vi.fn().mockResolvedValue("second") - const secondResult = pacer.run("b.ts", undefined, second) - await flush() - inFlight.resolve("kept") - expect(await kept).toBe("kept") - expect( - await registry - .get(makeRateLimitScope("qdrant", "https://store.test")) - .run(undefined, undefined, async () => "store"), - ).toBe("store") - expect( - await registry - .get(makeRateLimitScope("openai", "https://example.test", "second")) - .run(undefined, undefined, async () => "other key"), - ).toBe("other key") - await vi.advanceTimersByTimeAsync(29_999) - expect(first).toHaveBeenCalledTimes(1) - expect(second).not.toHaveBeenCalled() - expect(pacer.state()).toMatchObject({ limit: 5, pausedUntil: 30_000, waitingPaths: ["a.ts", "b.ts"] }) - await vi.advanceTimersByTimeAsync(1) - expect(await firstResult).toBe("first") - expect(second).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(125) - expect(await secondResult).toBe("second") +describe("makeRateLimitScope (CI-PP-01)", () => { + it.each([ + ["same configuration", ["openai", "https://api.test/v1", "key"], ["openai", "https://api.test/v1", "key"]], + ["trailing slash", ["openai", "https://api.test/v1/", "key"], ["openai", "https://api.test/v1", "key"]], + ["credentials in URL", ["openai", "https://u:p@api.test/v1", "key"], ["openai", "https://api.test/v1", "key"]], + ["URL hash", ["openai", "https://api.test/v1#x", "key"], ["openai", "https://api.test/v1", "key"]], + ])("gives equal scopes for %s", (_label, first, second) => { + const [provider, url, key] = first + const [provider2, url2, key2] = second + expect(makeRateLimitScope(provider, url, key)).toBe(makeRateLimitScope(provider2, url2, key2)) }) - it("T36 counts ten same-clock 429s once and ignores stale missing-delay reports after expiry", async () => { - const pacer = new ProviderPacer("openai:test") - const controllers = Array.from({ length: 10 }, () => new AbortController()) - const requests = controllers.map(() => deferred()) - const outcomes = requests.map((request, index) => - pacer.run(`${index}.ts`, controllers[index].signal, () => request.promise).catch((error) => error), + it.each([ + ["API key", ["openai", "https://api.test", "a"], ["openai", "https://api.test", "b"]], + ["endpoint", ["openai", "https://a.test", "k"], ["openai", "https://b.test", "k"]], + ["provider", ["openai", "https://a.test", "k"], ["mistral", "https://a.test", "k"]], + ])("gives unequal scopes for a different %s", (_label, first, second) => { + expect(makeRateLimitScope(first[0], first[1], first[2])).not.toBe( + makeRateLimitScope(second[0], second[1], second[2]), ) - await flush() - for (const request of requests) request.reject(limited()) - await flush() - expect(pacer.state()).toMatchObject({ - limit: 5, - consecutiveRateLimits: 1, - episodeSeq: 10, - issueSeq: 10, - pausedUntil: 500, - }) - expect(codeIndexLog.info).toHaveBeenCalledTimes(1) - for (const controller of controllers) controller.abort() - await Promise.all(outcomes) - await vi.advanceTimersByTimeAsync(500) - pacer.reportRateLimit(limited(), 9) - expect(pacer.state()).toMatchObject({ limit: 5, consecutiveRateLimits: 1, pausedUntil: undefined }) - pacer.reportRateLimit(limited(2000), 9) - expect(pacer.state()).toMatchObject({ limit: 5, consecutiveRateLimits: 1, pausedUntil: 2500 }) }) - it("T36 stale same-clock success neither resets backoff nor contributes to adaptive growth", async () => { - const pacer = new ProviderPacer("openai:test") - const stale = deferred() - const staleResult = pacer.run("stale", undefined, () => stale.promise) - await flush() - pacer.reportRateLimit(limited(), pacer.markIssued()) - stale.resolve() - await staleResult - expect(pacer.state().consecutiveRateLimits).toBe(1) - await vi.advanceTimersByTimeAsync(500) - const newIssue = pacer.markIssued() - pacer.reportRateLimit(limited(), newIssue) - expect(pacer.state()).toMatchObject({ limit: 2, consecutiveRateLimits: 2, pausedUntil: 1500 }) - await vi.advanceTimersByTimeAsync(1000) - for (let successCount = 0; successCount < 19; successCount++) { - const result = pacer.run("growth", undefined, async () => "ok") - await vi.advanceTimersByTimeAsync(125) - await result - } - expect(pacer.state().limit).toBe(2) - await pacer.run("growth", undefined, async () => "ok") - expect(pacer.state()).toMatchObject({ limit: 3, consecutiveRateLimits: 0 }) - for (let successCount = 0; successCount < 200; successCount++) - await pacer.run(undefined, undefined, async () => undefined) - expect(pacer.state().limit).toBe(10) + it("distinguishes extra inputs and never embeds the key", () => { + const scope = makeRateLimitScope("bedrock", undefined, undefined, "us-east-1|default") + expect(scope).not.toBe(makeRateLimitScope("bedrock", undefined, undefined, "us-west-2|default")) + expect(makeRateLimitScope("openai", "https://api.test", "secret-key")).not.toContain("secret-key") + expect(scope.startsWith("bedrock:")).toBe(true) }) +}) - it.each([0, 1])( - "T36 exponential equal jitter respects both bounds and the 300s base ceiling (random=%s)", - async (random) => { - vi.mocked(Math.random).mockReturnValue(random) - const pacer = new ProviderPacer("openai:test") - for (let episode = 1; episode <= 12; episode++) { - const startedAt = Date.now() - pacer.reportRateLimit(limited(), pacer.markIssued()) - const expected = Math.min(300_000, 1000 * 2 ** (episode - 1)) * (0.5 + random * 0.5) - expect(pacer.state().pausedUntil).toBe(startedAt + expected) - await vi.advanceTimersByTimeAsync(expected) - } - }, - ) +describe("ProviderPacer (CI-PP-02)", () => { + beforeEach(() => vi.useFakeTimers({ now: 0 })) + afterEach(() => vi.useRealTimers()) - it("requires consecutive successes for growth, resetting progress on non-429 failures", async () => { - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit(limited(1), pacer.markIssued()) + it.each([61_000, 120_000])("honors a %d ms Retry-After in full", async (delay) => { + const pacer = new ProviderPacer("openai:1") + const request = vi.fn().mockRejectedValueOnce(rateLimit(delay)).mockResolvedValue("ok") + const run = pacer.run("a.ts", undefined, request) + await vi.advanceTimersByTimeAsync(delay - 1) + expect(request).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(1) - for (let successCount = 0; successCount < 19; successCount++) { - const result = pacer.run(undefined, undefined, async () => undefined) - await vi.advanceTimersByTimeAsync(125) - await result - } - const error = { status: 500 } - await expect( - pacer.run(undefined, undefined, async () => { - throw error - }), - ).rejects.toBe(error) - await pacer.run(undefined, undefined, async () => undefined) - expect(pacer.state().limit).toBe(5) - for (let successCount = 0; successCount < 19; successCount++) - await pacer.run(undefined, undefined, async () => undefined) - expect(pacer.state().limit).toBe(6) + await expect(run).resolves.toBe("ok") + expect(request).toHaveBeenCalledTimes(2) }) - it("a probe 429 creates the next episode and leaves followers blocked", async () => { - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit(limited(100), pacer.markIssued()) - const controller = new AbortController() - const request = vi.fn().mockRejectedValue(limited()) - const pending = pacer.run("probe", controller.signal, request).catch((error) => error) - const follower = vi.fn().mockResolvedValue(undefined) - const waiting = pacer.run("follower", controller.signal, follower).catch((error) => error) - await vi.advanceTimersByTimeAsync(100) - expect(pacer.state()).toMatchObject({ limit: 2, consecutiveRateLimits: 2, pausedUntil: 1100 }) - expect(request).toHaveBeenCalledOnce() - expect(follower).not.toHaveBeenCalled() - controller.abort() - await Promise.all([pending, waiting]) + it("backs off exponentially without a delay, caps at 300 s and resets on success", async () => { + const pacer = new ProviderPacer("openai:1") + const request = vi.fn() + for (let iter = 0; iter < 12; iter++) request.mockRejectedValueOnce(rateLimit()) + request.mockResolvedValue("ok") + const run = pacer.run(undefined, undefined, request) + await vi.advanceTimersByTimeAsync(0) + const waits: number[] = [] + while (request.mock.calls.length < 13) { + const before = Date.now() + const resume = pacer.resumeAt! + await vi.advanceTimersByTimeAsync(resume - before) + waits.push(resume - before) + } + await expect(run).resolves.toBe("ok") + expect(waits.slice(0, 4)).toEqual([1000, 2000, 4000, 8000]) + expect(Math.max(...waits)).toBe(300_000) + + request.mockRejectedValueOnce(rateLimit()).mockResolvedValue("ok") + const next = pacer.run(undefined, undefined, request) + await vi.advanceTimersByTimeAsync(0) + expect(pacer.resumeAt! - Date.now()).toBe(1000) + await vi.advanceTimersByTimeAsync(1000) + await next }) - it("releases and requeues a granted permit when another 429 pauses before consumption", async () => { - const pacer = new ProviderPacer("openai:test") - const request = vi.fn().mockResolvedValue("done") - const result = pacer.run("granted.ts", undefined, request) - pacer.reportRateLimit(limited(1000), pacer.markIssued()) - await flush() - expect(request).not.toHaveBeenCalled() - expect(pacer.state()).toMatchObject({ waitingPaths: ["granted.ts"], issueSeq: 1 }) - expect(pacer["active"]).toBe(0) - await vi.advanceTimersByTimeAsync(999) - expect(request).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(await result).toBe("done") - expect(request).toHaveBeenCalledOnce() - expect(pacer.state()).toMatchObject({ waitingPaths: [], issueSeq: 2 }) - expect(pacer["active"]).toBe(0) - expect(vi.getTimerCount()).toBe(0) + it("never shortens an existing pause", async () => { + const pacer = new ProviderPacer("openai:1") + const slow = settle(pacer.run(undefined, undefined, vi.fn().mockRejectedValueOnce(rateLimit(60_000)))) + await vi.advanceTimersByTimeAsync(0) + const short = settle(pacer.run(undefined, undefined, vi.fn().mockResolvedValue("ok"))) + expect(pacer.resumeAt).toBe(60_000) + await vi.advanceTimersByTimeAsync(60_000) + await Promise.all([slow, short]) }) - it("rechecks cancellation after granting a permit but before invoking the caller", async () => { - const pacer = new ProviderPacer("openai:test") + it("rejects a waiter at once with the abort reason and drops its path", async () => { + const pacer = new ProviderPacer("openai:1") + const first = settle(pacer.run("a.ts", undefined, vi.fn().mockRejectedValueOnce(rateLimit(60_000)))) + await vi.advanceTimersByTimeAsync(0) const controller = new AbortController() - const request = vi.fn().mockResolvedValue(undefined) - const result = pacer.run(undefined, controller.signal, request) - controller.abort() - await expect(result).rejects.toBe(controller.signal.reason) + const request = vi.fn() + const waiting = pacer.run("b.ts", controller.signal, request) + expect(pacer.waitingPaths).toContain("b.ts") + const reason = new Error("stopped") + controller.abort(reason) + await expect(waiting).rejects.toBe(reason) + expect(pacer.waitingPaths).not.toContain("b.ts") expect(request).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(60_000) + await first }) - it.each([0, -1, NaN, Infinity, "invalid", 1e300, 8.64e15 + 1])( - "uses fallback for invalid, elapsed, or out-of-range provider delay %s", - (retryAfterMs) => { - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit({ status: 429, retryAfterMs }, pacer.markIssued()) - expect(pacer.state().pausedUntil).toBe(500) - }, - ) - - // The deadline limit is 8.64e15 ms (max Date), so the same delay flips from valid to invalid as time advances. - it.each([ - { advanceMs: 0, retryAfterMs: 8.64e15, pausedUntil: 8.64e15, source: "provider" }, - { advanceMs: 0, retryAfterMs: 8.64e15 - 500, pausedUntil: 8.64e15 - 500, source: "provider" }, - { advanceMs: 1000, retryAfterMs: 8.64e15 - 500, pausedUntil: 1500, source: "backoff" }, - ])( - "validates the provider deadline at report time (advance $advanceMs ms, delay $retryAfterMs)", - async ({ advanceMs, retryAfterMs, pausedUntil, source }) => { - const pacer = new ProviderPacer("openai:test") - await vi.advanceTimersByTimeAsync(advanceMs) - pacer.reportRateLimit(limited(retryAfterMs), pacer.markIssued()) - expect(pacer.state().pausedUntil).toBe(pausedUntil) - expect(codeIndexLog.info).toHaveBeenCalledWith("Provider pacing wait", expect.objectContaining({ source })) - expect(new Date(pausedUntil).toISOString()).toEqual(expect.any(String)) - }, - ) - - it("paused search reports extend without counting episodes or shortening waits", async () => { - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit(limited(1000), pacer.markIssued()) - await vi.advanceTimersByTimeAsync(100) - pacer.reportRateLimit(limited(100), pacer.markIssued()) - pacer.reportRateLimit(limited(2000), pacer.markIssued()) - expect(pacer.state()).toMatchObject({ limit: 5, episodeSeq: 1, consecutiveRateLimits: 1, pausedUntil: 2100 }) - }) - - it.each(["server", "connection", "permanent", "abort"])( - "T36 reopens one at a time after a probe %s outcome", - async (outcome) => { - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit(limited(1000), pacer.markIssued()) - const probe = deferred() - const controller = new AbortController() - const probeRequest = vi.fn(() => probe.promise) - const probeResult = pacer.run("probe", controller.signal, probeRequest).catch((error) => error) - const startedAt: number[] = [] - const followers = Array.from({ length: 6 }, (_, index) => - pacer.run(`follower-${index}`, undefined, async () => { - startedAt.push(Date.now()) - }), - ) - await vi.advanceTimersByTimeAsync(1000) - expect(probeRequest).toHaveBeenCalledOnce() - expect(startedAt).toEqual([]) - await vi.advanceTimersByTimeAsync(1000) - expect(startedAt).toEqual([]) - const error = - outcome === "abort" - ? new DOMException("aborted", "AbortError") - : outcome === "connection" - ? { code: "ECONNRESET" } - : { status: outcome === "server" ? 500 : 400 } - if (outcome === "abort") controller.abort(error) - probe.reject(error) - expect(await probeResult).toBe(error) - await vi.advanceTimersByTimeAsync(124) - expect(startedAt).toEqual([]) - await vi.advanceTimersByTimeAsync(1) - expect(startedAt).toEqual([2125]) - await vi.advanceTimersByTimeAsync(125) - expect(startedAt).toEqual([2125, 2250]) - await vi.runAllTimersAsync() - await Promise.all(followers) - expect(startedAt.slice(0, 4)).toEqual([2125, 2250, 2375, 2500]) - }, - ) - - it("uses the upper 375ms staged gap and lets probe cancellation reopen before transport settles", async () => { - vi.mocked(Math.random).mockReturnValue(1) - const pacer = new ProviderPacer("openai:test") - pacer.reportRateLimit(limited(100), pacer.markIssued()) - const probe = deferred() - const controller = new AbortController() - const probeResult = pacer.run(undefined, controller.signal, () => probe.promise) - const follower = vi.fn().mockResolvedValue(undefined) - const followerResult = pacer.run(undefined, undefined, follower) - await vi.advanceTimersByTimeAsync(100) - controller.abort() - await vi.advanceTimersByTimeAsync(374) - expect(follower).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - await followerResult - probe.resolve() - await probeResult - }) - - it("absorbs unlimited 429s but preserves a subsequent non-429 error unchanged", async () => { - const pacer = new ProviderPacer("openai:test") - const error = { status: 503 } - const request = vi.fn().mockRejectedValue(limited(1)) - const result = pacer.run("retry", undefined, request).catch((caught) => caught) - await vi.advanceTimersByTimeAsync(40) - expect(request.mock.calls.length).toBeGreaterThan(30) - request.mockRejectedValueOnce(error) - await vi.advanceTimersByTimeAsync(1) - expect(await result).toBe(error) - expect(vi.getTimerCount()).toBe(0) - }) - - it("cancels permit waits and pause waits before invoking the caller's gate", async () => { - const pacer = new ProviderPacer("openai:test") - const active = deferred() - const activeResults = Array.from({ length: 10 }, () => pacer.run(undefined, undefined, () => active.promise)) - await flush() - const controller = new AbortController() - const gate = vi.fn().mockResolvedValue(undefined) - const blocked = pacer.run("blocked", controller.signal, gate).catch((error) => error) - controller.abort() - expect(await blocked).toBe(controller.signal.reason) - expect(gate).not.toHaveBeenCalled() - active.resolve() - await Promise.all(activeResults) - pacer.reportRateLimit(limited(30_000), pacer.markIssued()) - const pausedController = new AbortController() - const paused = pacer.run("paused", pausedController.signal, gate).catch((error) => error) - pausedController.abort() - expect(await paused).toBe(pausedController.signal.reason) - expect(pacer.state().waitingPaths).toEqual([]) - expect(pacer.state().pausedUntil).toBe(30_000) - expect(gate).not.toHaveBeenCalled() - await expect(pacer.run("already aborted", pausedController.signal, gate)).rejects.toBe( - pausedController.signal.reason, + it("rethrows non-rate-limit errors unchanged", async () => { + const failure = Object.assign(new Error("server"), { status: 500 }) + await expect(new ProviderPacer("openai:1").run(undefined, undefined, () => Promise.reject(failure))).rejects.toBe( + failure, ) }) - it("retains uncapped delays beyond 32-bit timers without sending early", async () => { - const pacer = new ProviderPacer("openai:test") - const delay = 2 ** 31 + 60_000 - pacer.reportRateLimit({ status: 429, headers: { "retry-after-ms": String(delay) } }, pacer.markIssued()) - const request = vi.fn().mockResolvedValue("ok") - const result = pacer.run("long", undefined, request) - expect(pacer.state().pausedUntil).toBe(delay) - await vi.advanceTimersByTimeAsync(2 ** 31 - 1) - expect(request).not.toHaveBeenCalled() - expect(vi.getTimerCount()).toBe(1) - await vi.advanceTimersByTimeAsync(60_000) - expect(request).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(await result).toBe("ok") + it("keeps waits longer than the 32-bit timer limit", async () => { + const pacer = new ProviderPacer("openai:1") + const delay = 2 ** 31 + 10_000 + const request = vi.fn().mockRejectedValueOnce(rateLimit(delay)).mockResolvedValue("ok") + const run = pacer.run(undefined, undefined, request) + await vi.advanceTimersByTimeAsync(2 ** 31) + expect(request).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(10_000) + await expect(run).resolves.toBe("ok") }) }) -describe("ProviderPacerRegistry and scope", () => { - it("normalizes URL identity, strips credentials and hashes keys without disclosing configuration", () => { - const scope = makeRateLimitScope("openai", "https://user:password@EXAMPLE.test:443/v1/", "secret") - expect(scope).toBe(makeRateLimitScope("openai", "https://example.test/v1", "secret")) - expect(scope).not.toContain("secret") - expect(scope).not.toContain("password") - expect(scope).not.toContain("example") - expect(scope).not.toBe(makeRateLimitScope("openai", "https://example.test/v1", "other")) - expect(makeRateLimitScope("bedrock", undefined, undefined, "region/profile")).not.toBe( - makeRateLimitScope("bedrock", undefined, undefined, "other/profile"), - ) - expect(makeRateLimitScope("ollama", "http://localhost:11434")).toMatch(/^ollama:[a-f0-9]{64}$/) - const keyHash = createHash("sha256").update("secret").digest("hex") - expect(scope).toBe( - `openai:${createHash("sha256") - .update(JSON.stringify(["openai", "https://example.test/v1", keyHash, ""])) - .digest("hex")}`, - ) - }) +describe("ProviderPacerRegistry", () => { + beforeEach(() => vi.useFakeTimers({ now: 0 })) + afterEach(() => vi.useRealTimers()) - it("notifies status, returns detached paused snapshots and deduplicates all waiting paths", async () => { + it("shares a scope's pause, isolates other scopes and reports deduplicated waits", async () => { const registry = new ProviderPacerRegistry() const changed = vi.fn() - const disposables: { dispose(): void }[] = [] - registry.onDidChange(changed, undefined, disposables) - const embedder = registry.get("openai:https://private.test?key=secret") - const store = registry.get("qdrant:https://private-store.test") - embedder.reportRateLimit(limited(1000), embedder.markIssued()) - store.reportRateLimit(limited(2000), store.markIssued()) - const controller = new AbortController() - const queued: [typeof embedder, string][] = [ - [embedder, "same.ts"], - [embedder, "embed-only.ts"], - [embedder, "same.ts"], - [store, "same.ts"], - [store, "store-only.ts"], - ] - const outcomes = queued.map(([pacer, file]) => - pacer.run(file, controller.signal, async () => undefined).catch((error) => error), - ) - await flush() - const allPaths = ["same.ts", "embed-only.ts", "store-only.ts"] - expect(registry.waitingPaths()).toEqual(allPaths) - expect(registry.providerWaits()).toEqual([ - { service: "openai", resumeAt: 1000, waitingFiles: ["same.ts", "embed-only.ts"] }, - { service: "qdrant", resumeAt: 2000, waitingFiles: ["same.ts", "store-only.ts"] }, - ]) - registry.providerWaits()[0].waitingFiles.push("mutated") - expect(registry.waitingPaths()).toEqual(allPaths) - controller.abort() - await Promise.all(outcomes) - expect(registry.waitingPaths()).toEqual([]) + registry.onDidChange(changed) + const paused = settle(registry.get("openai:1").run("a.ts", undefined, vi.fn().mockRejectedValueOnce(rateLimit(5000)))) + await vi.advanceTimersByTimeAsync(0) + const same = settle(registry.get("openai:1").run("a.ts", undefined, vi.fn().mockResolvedValue("ok"))) + const other = vi.fn().mockResolvedValue("other") + await expect(registry.get("qdrant:2").run("b.ts", undefined, other)).resolves.toBe("other") + + expect(registry.providerWaits()).toEqual([{ service: "openai", resumeAt: 5000, waitingFiles: ["a.ts"] }]) + expect(registry.waitingPaths()).toEqual(["a.ts"]) + expect(registry.hasWaitingRequests()).toBe(true) expect(changed).toHaveBeenCalled() - disposables[0].dispose() - changed.mockClear() - await vi.advanceTimersByTimeAsync(2000) - expect(changed).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(5000) + await Promise.all([paused, same]) expect(registry.providerWaits()).toEqual([]) - expect(registry.get("https://user:secret@host").service).toBe("https") - expect(registry.get("unsafe/key:secret").service).toBe("provider") + expect(registry.hasWaitingRequests()).toBe(false) }) }) diff --git a/src/services/code-index/__tests__/state-manager.spec.ts b/src/services/code-index/__tests__/state-manager.spec.ts deleted file mode 100644 index f9605e32c9..0000000000 --- a/src/services/code-index/__tests__/state-manager.spec.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { CodeIndexStateManager } from "../state-manager" -import { codeIndexLog } from "../code-index-log" -import { clearAllMocks } from "../../../test-utils/reset" - -vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) -vi.mock("vscode", () => ({ - EventEmitter: class { - event = vi.fn() - // Prototype methods give each vi.spyOn a fresh spy instead of an instance mock with earlier calls. - fire(): void {} - dispose(): void {} - }, -})) - -beforeEach(() => clearAllMocks()) - -describe("CodeIndexStateManager", () => { - it("logs changed state and messages with diagnostic context only", () => { - const state = new CodeIndexStateManager() - state.setDiagnosticContext({ source: "manager", attempt: 12 }) - state.setSystemState("Indexing", "Starting", "start-request") - expect(codeIndexLog.info).toHaveBeenLastCalledWith("State transition", { - oldState: "Standby", - newState: "Indexing", - message: "Starting", - source: "start-request", - attempt: 12, - }) - state.setSystemState("Indexing", "Starting") - expect(codeIndexLog.info).toHaveBeenCalledOnce() - state.setSystemState("Standby") - expect(codeIndexLog.info).toHaveBeenLastCalledWith("State transition", { - oldState: "Indexing", - newState: "Standby", - message: "Ready.", - source: "manager", - attempt: 12, - }) - expect(state.getCurrentStatus()).not.toHaveProperty("attempt") - }) - - it.each(["blocks", "files"] as const)("logs entry but not each %s progress update", (unit) => { - const state = new CodeIndexStateManager() - state.setDiagnosticContext({ source: "manager", attempt: "attempt-2" }) - const progress = (processed: number) => { - if (unit === "blocks") state.reportBlockIndexingProgress(processed, 3, "scan") - else state.reportFileQueueProgress(processed, 3, "file.ts", "scan") - } - progress(1) - progress(2) - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Indexing", - processedItems: 2, - totalItems: 3, - currentItemUnit: unit, - }) - expect(codeIndexLog.info).toHaveBeenCalledExactlyOnceWith( - "State transition", - expect.objectContaining({ - oldState: "Standby", - newState: "Indexing", - source: "scan", - attempt: "attempt-2", - }), - ) - state.setSystemState("Standby") - expect(codeIndexLog.info).toHaveBeenLastCalledWith( - "State transition", - expect.objectContaining({ source: "manager", attempt: "attempt-2" }), - ) - }) - - it.each([ - ["Error", "blocks"], - ["Error", "files"], - ["Stopping", "blocks"], - ["Stopping", "files"], - ] as const)("does not let %s be overwritten by %s progress", (systemStatus, unit) => { - const state = new CodeIndexStateManager() - state.reportBlockIndexingProgress(1, 3) - state.setSystemState(systemStatus, "Preserved message") - const before = state.getCurrentStatus() - const fire = vi.spyOn(state["_progressEmitter"], "fire") - vi.mocked(codeIndexLog.info).mockClear() - if (unit === "blocks") state.reportBlockIndexingProgress(3, 10) - else state.reportFileQueueProgress(3, 10, "late.ts") - expect(state.getCurrentStatus()).toEqual(before) - expect(fire).not.toHaveBeenCalled() - expect(codeIndexLog.info).not.toHaveBeenCalled() - }) - - it.each(["Error", "Stopping"] as const)("allows an explicit restart from %s", (systemStatus) => { - const state = new CodeIndexStateManager() - state.setSystemState(systemStatus, "Previous attempt") - state.setSystemState("Indexing", "New attempt") - state.reportBlockIndexingProgress(1, 2) - expect(state.getCurrentStatus()).toMatchObject({ systemStatus: "Indexing", processedItems: 1, totalItems: 2 }) - }) - - it("does not emit duplicate progress and resets counters on completion", () => { - const state = new CodeIndexStateManager() - state.reportBlockIndexingProgress(1, 2) - const fire = vi.spyOn(state["_progressEmitter"], "fire") - state.reportBlockIndexingProgress(1, 2) - expect(fire).not.toHaveBeenCalled() - state.setSystemState("Indexed") - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Indexed", - message: "Index up-to-date.", - processedItems: 0, - totalItems: 0, - }) - }) -}) diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 6eed825bc5..388bf27eae 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -1,8 +1,10 @@ import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types" /**Parser */ -export const PARSE_TIMEOUT_MS = 10_000 -export const INDEX_STOP_LIMIT_MS = 2_000 +export const MAX_BLOCK_CHARS = 1000 +export const MIN_BLOCK_CHARS = 50 +export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split +export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars /**Search */ export const DEFAULT_SEARCH_MIN_SCORE = CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE @@ -17,10 +19,9 @@ export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000 export const BATCH_SEGMENT_THRESHOLD = 60 // Number of code segments to batch for embeddings/upserts export const FILE_RETRY_ATTEMPTS = 3 export const FILE_RETRY_BASE_DELAY_MS = 500 -export const PACER_INCREASE_AFTER = 20 export const PACER_MAX_BACKOFF_MS = 300_000 -export const PACER_RESUME_SPACING_MS = 250 export const PARSING_CONCURRENCY = 10 +export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting /**OpenAI Embedder */ export const MAX_BATCH_TOKENS = 100000 diff --git a/src/services/code-index/embedders/__tests__/cancellation.spec.ts b/src/services/code-index/embedders/__tests__/cancellation.spec.ts index bfc718c425..d5e5016c75 100644 --- a/src/services/code-index/embedders/__tests__/cancellation.spec.ts +++ b/src/services/code-index/embedders/__tests__/cancellation.spec.ts @@ -1,4 +1,3 @@ -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) vi.mock("../../../../utils/networkProxy", () => ({ createProxyRoutingRequestHandler: vi.fn() })) vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: { captureEvent: vi.fn() } }, diff --git a/src/services/code-index/file-completion.ts b/src/services/code-index/file-completion.ts deleted file mode 100644 index 9f0cbb5f2f..0000000000 --- a/src/services/code-index/file-completion.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { v5 as uuidv5 } from "uuid" -import type { CacheManager } from "./cache-manager" -import { GenerationAllocator } from "./generation-allocator" -import { IndexWriteGate } from "./index-write-gate" -import type { CodeBlock, IEmbedder, IVectorStore } from "./interfaces" -import { BATCH_SEGMENT_THRESHOLD, QDRANT_CODE_BLOCK_NAMESPACE } from "./constants" -import { FileFailureRegistry, REMOVAL_REVISION, type FileFailureActivity } from "./file-failure-registry" -import { ProviderPacerRegistry } from "./provider-pacer" -import { retryWithPolicy } from "./shared/retry-policy" -import { generateRelativeFilePath } from "./shared/get-relative-path" -import { throwIfAborted } from "./parser-errors" - -export class FileCompletion { - public registrationComplete = false - public outstanding = 0 - public failed = false - private finishing?: Promise - - constructor(private readonly finish: () => Promise) {} - - register(): void { - if (this.registrationComplete) throw new Error("File registration already complete") - this.outstanding++ - } - - async settled(success: boolean): Promise { - this.failed ||= !success - this.outstanding-- - await this.tryComplete() - } - - async registered(): Promise { - this.registrationComplete = true - await this.tryComplete() - } - - private async tryComplete(): Promise { - if (!this.registrationComplete || this.outstanding !== 0 || this.failed) return - this.finishing ??= this.finish().catch((error) => { - this.failed = true - throw error - }) - await this.finishing - } -} - -export class FileIndexWriter { - private readonly owners = new WeakMap() - constructor( - private readonly cache: CacheManager, - private readonly store: IVectorStore, - private readonly gate: IndexWriteGate, - private readonly generations: GenerationAllocator, - private readonly workspacePath: string, - private readonly failures = new FileFailureRegistry(), - private readonly pacers = new ProviderPacerRegistry(), - ) {} - - private async run(signal: AbortSignal, operation: () => Promise, files: string[] = []): Promise { - const owner = this.owners.get(signal) ?? signal - // Track revision cancellation separately without treating a late successful save as a rejection. - const outcome = await this.gate.run( - signal, - () => - this.gate - .run( - owner, - () => { - throwIfAborted(signal) - return operation() - }, - files, - ) - .then( - (value) => ({ value, error: undefined }), - (error: unknown) => ({ value: undefined, error }), - ), - files, - ) - if (outcome.error !== undefined) throw outcome.error - return outcome.value as T - } - - async retryTransient( - operation: () => Promise, - options: { - signal: AbortSignal - activity: FileFailureActivity - file: string - revision: string - rateLimitScope?: string - }, - ): Promise { - const { signal, activity, file, revision, rateLimitScope = this.store.rateLimitScope } = options - const relativePath = generateRelativeFilePath(file, this.workspacePath) - const previous = this.failures.getFailure(relativePath, revision, activity) - if (previous?.state === "exhausted") throw Object.assign(new Error(previous.reason), previous) - return await retryWithPolicy( - () => - this.pacers - .get(rateLimitScope) - .run(relativePath, signal, () => - this.run(signal, operation, activity === "upsert" || activity === "delete" ? [file] : []), - ), - { - signal, - activity, - budget: this.failures.getRetryBudget(relativePath, revision, activity), - onFailure: ({ error, attempt, willRetry, classification, ...delay }) => { - this.failures.record( - relativePath, - revision, - activity, - error instanceof Error ? error.message : String(error), - attempt, - willRetry ? "retrying" : "exhausted", - classification, - delay, - ) - }, - }, - ) - } - - private async withRevision( - filePath: string, - revision: string, - parent: AbortSignal, - operation: (signal: AbortSignal) => Promise, - owner = parent, - ): Promise { - throwIfAborted(parent) - const relativePath = generateRelativeFilePath(filePath, this.workspacePath) - const revisionSignal = this.failures.renew(relativePath, revision) - const signal = AbortSignal.any([parent, revisionSignal]) - this.owners.set(signal, owner) - try { - this.failures.assertRetryable(relativePath, revision) - return await operation(signal) - } catch (error) { - if (!signal.aborted && !this.failures.hasRevisionFailure(relativePath, revision)) { - this.failures.record( - relativePath, - revision, - revision === REMOVAL_REVISION ? "delete" : "upsert", - error instanceof Error ? error.message : String(error), - 1, - "exhausted", - ) - } - throw error - } finally { - const failure = this.failures.get(relativePath) - if (!revisionSignal.aborted && failure?.revision === revision && failure.state === "retrying") { - this.failures.record( - relativePath, - revision, - failure.activity, - failure.reason, - failure.attempt, - "exhausted", - failure.classification, - ) - } - } - } - - async remove(filePath: string, signal: AbortSignal, owner = signal): Promise { - return this.withRevision( - filePath, - REMOVAL_REVISION, - signal, - (current) => this.removeCurrent(filePath, current), - owner, - ) - } - - private async removeCurrent(filePath: string, signal: AbortSignal): Promise { - const generation = await this.generations.allocate(signal) - await this.run(signal, () => this.cache.invalidateFile(filePath)) - await this.retryTransient(() => this.store.deletePointsByFilePath(filePath, generation), { - signal, - activity: "delete", - file: filePath, - revision: REMOVAL_REVISION, - }) - throwIfAborted(signal) - this.cache.completeRemoval(filePath) - this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath), REMOVAL_REVISION) - } - - async replace( - filePath: string, - hash: string, - blocks: CodeBlock[], - embedder: IEmbedder, - signal: AbortSignal, - batchSize = BATCH_SEGMENT_THRESHOLD, - onComplete?: (blockCount: number) => void, - owner = signal, - ): Promise { - return this.withRevision( - filePath, - hash, - signal, - (current) => this.replaceCurrent(filePath, hash, blocks, embedder, current, batchSize, onComplete), - owner, - ) - } - - private async replaceCurrent( - filePath: string, - hash: string, - blocks: CodeBlock[], - embedder: IEmbedder, - signal: AbortSignal, - batchSize: number, - onComplete?: (blockCount: number) => void, - ): Promise { - const generation = await this.generations.allocate(signal) - await this.run(signal, () => this.cache.invalidateFile(filePath)) - const retryOptions = { signal, file: filePath, revision: hash } - const selected = blocks.filter((block) => block.content.trim()) - const verify = async () => { - throwIfAborted(signal) - const counts = await this.retryTransient(() => this.store.countPointsForFile(filePath, generation), { - ...retryOptions, - activity: "verify", - }) - throwIfAborted(signal) - if (counts.current !== selected.length || counts.other !== 0) { - const reason = `Index verification failed for ${filePath}: expected ${selected.length}, current ${counts.current}, other ${counts.other}` - this.failures.record( - generateRelativeFilePath(filePath, this.workspacePath), - hash, - "verify", - reason, - 1, - "exhausted", - ) - throw new Error(reason) - } - } - const completion = new FileCompletion(async () => { - await this.retryTransient(() => this.store.deletePointsByFilePath(filePath, generation), { - ...retryOptions, - activity: "delete", - }) - await verify() - const waitedForPriorWrites = await this.gate.waitForPriorWrites(filePath, signal) - if (this.gate.needsReconciliation(filePath)) { - do { - await this.gate.waitForGrace(filePath, signal) - await verify() - } while (this.gate.graceDeadline(filePath) > Date.now()) - } else if (waitedForPriorWrites) { - await verify() - } - throwIfAborted(signal) - this.cache.completeFile(filePath, hash) - this.gate.reconciled(filePath) - this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath), hash) - onComplete?.(selected.length) - }) - const batches: CodeBlock[][] = [] - for (let offset = 0; offset < selected.length; offset += Math.max(1, batchSize)) { - completion.register() - batches.push(selected.slice(offset, offset + Math.max(1, batchSize))) - } - await completion.registered() - for (const batch of batches) { - try { - throwIfAborted(signal) - const { embeddings } = await this.retryTransient( - async () => { - const result = await embedder.createEmbeddings( - batch.map((block) => block.content.trim()), - undefined, - signal, - ) - if (result.embeddings.length !== batch.length) - throw new Error(`Incomplete embeddings for ${filePath}`) - return result - }, - { ...retryOptions, activity: "embed", rateLimitScope: embedder.rateLimitScope }, - ) - throwIfAborted(signal) - const points = batch.map((block, index) => ({ - id: uuidv5(`${block.segmentHash}:${generation}`, QDRANT_CODE_BLOCK_NAMESPACE), - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(filePath, this.workspacePath), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - segmentHash: block.segmentHash, - generation, - }, - })) - await this.retryTransient(() => this.store.upsertPoints(points), { - ...retryOptions, - activity: "upsert", - }) - } catch (error) { - await completion.settled(false) - throw error - } - await completion.settled(true) - } - } -} diff --git a/src/services/code-index/file-failure-registry.ts b/src/services/code-index/file-failure-registry.ts index 95fff45e12..3f02746a34 100644 --- a/src/services/code-index/file-failure-registry.ts +++ b/src/services/code-index/file-failure-registry.ts @@ -2,7 +2,7 @@ import type { Disposable, Event } from "vscode" import type { IndexingErrorClassification } from "./shared/indexing-errors" import type { RetryBudget, RetryFailure } from "./shared/retry-policy" -export type FileFailureActivity = "read" | "parse" | "embed" | "upsert" | "delete" | "verify" +export type FileFailureActivity = "read" | "parse" | "embed" | "upsert" | "delete" export type FileFailureState = "retrying" | "exhausted" export const REMOVAL_REVISION = "removed" export const UNKNOWN_REVISION = "unread" diff --git a/src/services/code-index/file-index-writer.ts b/src/services/code-index/file-index-writer.ts new file mode 100644 index 0000000000..c2353b1745 --- /dev/null +++ b/src/services/code-index/file-index-writer.ts @@ -0,0 +1,117 @@ +import { v5 as uuidv5 } from "uuid" +import type { CodeBlock, IEmbedder, IVectorStore, PointStruct } from "./interfaces" +import type { ICacheManager } from "./interfaces/cache" +import { BATCH_SEGMENT_THRESHOLD, QDRANT_CODE_BLOCK_NAMESPACE } from "./constants" +import { FileFailureRegistry, REMOVAL_REVISION, type FileFailureActivity } from "./file-failure-registry" +import type { ProviderPacerRegistry } from "./provider-pacer" +import { retryWithPolicy } from "./shared/retry-policy" +import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "./shared/get-relative-path" + +/** Embeds, saves and removes the blocks of one file; the only place code-index embeds run. */ +export class FileIndexWriter { + constructor( + private readonly embedder: IEmbedder, + private readonly store: IVectorStore, + private readonly cache: Pick, + private readonly failures: FileFailureRegistry, + private readonly pacers: ProviderPacerRegistry, + private readonly workspacePath: string, + private readonly batchSize = BATCH_SEGMENT_THRESHOLD, + ) {} + + /** Replaces the file's points with `blocks`; returns false when the file failed and was recorded. */ + async replace(filePath: string, hash: string, blocks: CodeBlock[], signal?: AbortSignal): Promise { + const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + this.failures.renew(relativePath, hash) + const selected = blocks.filter((block) => block.content.trim()) + try { + await this.attempt(relativePath, hash, "delete", this.store.rateLimitScope, signal, () => + this.store.deletePointsByFilePath(filePath), + ) + for (let offset = 0; offset < selected.length; offset += this.batchSize) { + const batch = selected.slice(offset, offset + this.batchSize) + const { embeddings } = await this.attempt( + relativePath, + hash, + "embed", + this.embedder.rateLimitScope, + signal, + () => this.embedder.createEmbeddings(batch.map((block) => block.content)), + ) + const points = batch.map((block, index) => this.toPoint(block, embeddings[index])) + await this.attempt(relativePath, hash, "upsert", this.store.rateLimitScope, signal, () => + this.store.upsertPoints(points), + ) + } + } catch (error) { + if (signal?.aborted) throw error + return false + } + if (signal?.aborted) return false + this.cache.updateHash(filePath, hash) + this.failures.clear(relativePath, hash) + return true + } + + /** Deletes the file's points and hash; returns false when the removal failed and was recorded. */ + async remove(filePath: string, signal?: AbortSignal): Promise { + const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + this.failures.renew(relativePath, REMOVAL_REVISION) + try { + await this.attempt(relativePath, REMOVAL_REVISION, "delete", this.store.rateLimitScope, signal, () => + this.store.deletePointsByFilePath(filePath), + ) + } catch (error) { + if (signal?.aborted) throw error + return false + } + if (signal?.aborted) return false + this.cache.deleteHash(filePath) + this.failures.clear(relativePath, REMOVAL_REVISION) + return true + } + + private attempt( + relativePath: string, + revision: string, + activity: FileFailureActivity, + scope: string, + signal: AbortSignal | undefined, + request: () => Promise, + ): Promise { + return retryWithPolicy(() => this.pacers.get(scope).run(relativePath, signal, request), { + signal, + activity, + budget: this.failures.getRetryBudget(relativePath, revision, activity), + onFailure: ({ error, attempt, willRetry, classification, ...delay }) => { + if (signal?.aborted) return + this.failures.record( + relativePath, + revision, + activity, + error instanceof Error ? error.message : String(error), + attempt, + willRetry ? "retrying" : "exhausted", + classification, + delay, + ) + }, + }) + } + + // Same id as the merge-base watcher so scanner and watcher writes of one block replace each other. + private toPoint(block: CodeBlock, vector: number[]): PointStruct { + const absolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) + return { + id: uuidv5(`${absolutePath}:${block.start_line}`, QDRANT_CODE_BLOCK_NAMESPACE), + vector, + payload: { + filePath: generateRelativeFilePath(absolutePath, this.workspacePath), + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + segmentHash: block.segmentHash, + }, + } + } +} diff --git a/src/services/code-index/generation-allocator.ts b/src/services/code-index/generation-allocator.ts deleted file mode 100644 index 8e7a25ed04..0000000000 --- a/src/services/code-index/generation-allocator.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type { CacheManager } from "./cache-manager" -import type { IVectorStore } from "./interfaces" -import { IndexWriteGate } from "./index-write-gate" -import { throwIfAborted } from "./parser-errors" - -const GENERATION_BLOCK_SIZE = 1024 - -export class GenerationAllocator { - private nextGeneration: number - private reservedThrough: number - private allocationTail: Promise = Promise.resolve() - private initialized = false - - constructor(private readonly cacheManager: CacheManager) { - this.reservedThrough = cacheManager.generationWatermark ?? 0 - this.nextGeneration = this.reservedThrough + 1 - } - - async initialize(vectorStore: IVectorStore, signal: AbortSignal, gate: IndexWriteGate): Promise { - if (this.initialized) return - throwIfAborted(signal) - if (this.cacheManager.generationWatermark === undefined) { - if (await vectorStore.hasIndexedData()) { - await gate.run(signal, () => vectorStore.clearCollection()) - } - throwIfAborted(signal) - await this.cacheManager.clearCacheFile() - } - await this.reserve(signal) - this.initialized = true - } - - private async reserve(signal?: AbortSignal): Promise { - throwIfAborted(signal) - const watermark = - Math.max(this.reservedThrough, this.cacheManager.generationWatermark ?? 0) + GENERATION_BLOCK_SIZE - if (!Number.isSafeInteger(watermark)) throw new Error("Index generation counter exhausted; clear index storage") - await this.cacheManager.reserveGenerations(watermark) - throwIfAborted(signal) - this.reservedThrough = watermark - } - - allocate(signal?: AbortSignal): Promise { - const allocation = this.allocationTail - .catch(() => {}) - .then(async () => { - throwIfAborted(signal) - if (this.nextGeneration > this.reservedThrough) await this.reserve(signal) - throwIfAborted(signal) - return this.nextGeneration++ - }) - this.allocationTail = allocation.then( - () => {}, - () => {}, - ) - return allocation - } -} diff --git a/src/services/code-index/index-write-gate.ts b/src/services/code-index/index-write-gate.ts deleted file mode 100644 index c3951d980f..0000000000 --- a/src/services/code-index/index-write-gate.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { abortError, throwIfAborted } from "./parser-errors" -import { INDEX_STOP_LIMIT_MS } from "./constants" - -export const LATE_WRITE_GRACE_MS = 30_000 - -type IssuedWrite = { pending: Promise; files: string[]; abandoned?: boolean } - -export class IndexWriteGate { - private readonly issued = new Map>() - private readonly deadlines = new Map() - private readonly uncertainByAttempt = new WeakMap>() - private readonly owners = new WeakMap() - - private readonly stopDeadlines = new WeakMap() - private readonly observedOwners = new WeakSet() - - own(signal: AbortSignal, owner: AbortSignal): void { - this.owners.set(signal, owner) - } - - stopAt(signal: AbortSignal, expiresAt: number): void { - this.stopDeadlines.set(signal, expiresAt) - } - - graceDeadline(filePath: string): number { - return this.deadlines.get(filePath) ?? 0 - } - - recover(files: string[], startedAt = Date.now()): void { - for (const filePath of files) { - this.deadlines.set(filePath, Math.max(this.deadlines.get(filePath) ?? 0, startedAt + LATE_WRITE_GRACE_MS)) - } - } - - needsReconciliation(filePath: string): boolean { - return this.deadlines.has(filePath) - } - - reconciled(filePath: string): void { - this.deadlines.delete(filePath) - } - - hasPending(signal: AbortSignal): boolean { - return (this.issued.get(signal)?.size ?? 0) > 0 - } - - async settled(signal: AbortSignal): Promise { - while (this.hasPending(signal)) { - await Promise.allSettled([...this.issued.get(signal)!].map((write) => write.pending)) - } - } - - uncertainFiles(signal: AbortSignal): string[] { - return [...(this.uncertainByAttempt.get(signal) ?? [])] - } - - private markUncertain(signal: AbortSignal, files: string[], startedAt = Date.now()): void { - let uncertain = this.uncertainByAttempt.get(signal) - if (!uncertain) { - uncertain = new Set() - this.uncertainByAttempt.set(signal, uncertain) - } - for (const filePath of files) uncertain.add(filePath) - this.recover(files, startedAt) - } - - abandon(signal: AbortSignal, startedAt = Date.now()): void { - for (const write of this.issued.get(signal) ?? []) { - if (write.abandoned) continue - write.abandoned = true - this.markUncertain(signal, write.files, startedAt) - } - } - - async waitForPriorWrites(filePath: string, signal: AbortSignal): Promise { - const currentOwner = this.owners.get(signal) ?? signal - let waited = false - while (true) { - throwIfAborted(signal) - const pending: Promise[] = [] - let remaining = Infinity - for (const [owner, writes] of this.issued) { - if (owner === currentOwner || !owner.aborted) continue - const relevant = [...writes].filter((write) => !write.abandoned && write.files.includes(filePath)) - if (relevant.length === 0) continue - const stopDeadline = this.stopDeadlines.get(owner) - if (stopDeadline === undefined) throw new Error("Missing indexing stop deadline") - if (stopDeadline <= Date.now()) { - this.abandon(owner, stopDeadline) - continue - } - remaining = Math.min(remaining, stopDeadline - Date.now()) - pending.push(...relevant.map((write) => write.pending)) - } - if (pending.length === 0) return waited - waited = true - await new Promise((resolve, reject) => { - const finish = (error?: Error) => { - clearTimeout(timer) - signal.removeEventListener("abort", onAbort) - if (error) reject(error) - else resolve() - } - const onAbort = () => finish(abortError()) - const timer = setTimeout(() => finish(), remaining) - signal.addEventListener("abort", onAbort, { once: true }) - void Promise.allSettled(pending).then(() => finish()) - if (signal.aborted) onAbort() - }) - } - } - - async waitForGrace(filePath: string, signal: AbortSignal): Promise { - while (true) { - throwIfAborted(signal) - const remaining = (this.deadlines.get(filePath) ?? 0) - Date.now() - if (remaining <= 0) return - await new Promise((resolve, reject) => { - const onAbort = () => { - clearTimeout(timer) - signal.removeEventListener("abort", onAbort) - reject(abortError()) - } - const timer = setTimeout(() => { - signal.removeEventListener("abort", onAbort) - resolve() - }, remaining) - signal.addEventListener("abort", onAbort, { once: true }) - if (signal.aborted) onAbort() - }) - } - } - - async run( - signal: AbortSignal, - write: () => Result | Promise, - files: string[] = [], - ): Promise { - throwIfAborted(signal) - const owner = this.owners.get(signal) ?? signal - if (!this.observedOwners.has(owner)) { - this.observedOwners.add(owner) - const onAbort = () => { - if (!this.stopDeadlines.has(owner)) this.stopAt(owner, Date.now() + INDEX_STOP_LIMIT_MS) - } - if (owner.aborted) onAbort() - else owner.addEventListener("abort", onAbort, { once: true }) - } - let writes = this.issued.get(owner) - if (!writes) { - writes = new Set() - this.issued.set(owner, writes) - } - let issued = false - let succeeded = false - const pending = Promise.resolve().then(() => { - throwIfAborted(signal) - issued = true - return write() - }) - const record = { pending, files: [...files] } - writes.add(record) - try { - const result = await pending - succeeded = true - throwIfAborted(signal) - return result - } catch (error) { - if (issued && !succeeded) this.markUncertain(owner, files) - throw error - } finally { - writes.delete(record) - if (writes.size === 0) this.issued.delete(owner) - } - } -} diff --git a/src/services/code-index/provider-pacer.ts b/src/services/code-index/provider-pacer.ts index 324d0b7c9f..6ddcdf9eec 100644 --- a/src/services/code-index/provider-pacer.ts +++ b/src/services/code-index/provider-pacer.ts @@ -1,25 +1,9 @@ -import { createHash } from "node:crypto" import type { Disposable, Event } from "vscode" -import { - BATCH_PROCESSING_CONCURRENCY, - PACER_INCREASE_AFTER, - PACER_MAX_BACKOFF_MS, - PACER_RESUME_SPACING_MS, -} from "./constants" -import { codeIndexLog } from "./code-index-log" +import { PACER_MAX_BACKOFF_MS } from "./constants" import { classifyIndexingError, EmbeddingError } from "./shared/indexing-errors" const MAX_TIMEOUT_MS = 2 ** 31 - 1 -export interface ProviderPacerState { - pausedUntil?: number - waitingPaths: string[] - limit: number - episodeSeq: number - issueSeq: number - consecutiveRateLimits: number -} - export interface ProviderWait { service: string resumeAt?: number @@ -29,18 +13,19 @@ export interface ProviderWait { type Waiter = { path?: string signal?: AbortSignal - resolve: (permit: Permit) => void + resolve: () => void reject: (error: unknown) => void onAbort: () => void } -type Permit = { probeEpisode?: number } - function serviceLabel(scope: string): string { const prefix = scope.split(":", 1)[0] return /^[a-z][a-z0-9-]{0,63}$/i.test(prefix) ? prefix : "provider" } +// In-memory equality ids: hashing the API key would be flagged as weak password hashing and adds nothing here. +const scopeIds = new Map() + export function makeRateLimitScope(provider: string, baseUrl?: string, apiKey?: string, extra?: string): string { let normalizedUrl = "" if (baseUrl) { @@ -50,28 +35,20 @@ export function makeRateLimitScope(provider: string, baseUrl?: string, apiKey?: parsedUrl.hash = "" normalizedUrl = parsedUrl.toString().replace(/\/+$/, "") } - const keyHash = apiKey ? createHash("sha256").update(apiKey).digest("hex") : "" - // Hash the tuple as well: configured URLs can contain query-string credentials. - const allowanceHash = createHash("sha256") - .update(JSON.stringify([provider, normalizedUrl, keyHash, extra ?? ""])) - .digest("hex") - return `${serviceLabel(provider)}:${allowanceHash}` + const key = JSON.stringify([provider, normalizedUrl, apiKey ?? "", extra ?? ""]) + let id = scopeIds.get(key) + if (id === undefined) { + id = scopeIds.size + 1 + scopeIds.set(key, id) + } + return `${serviceLabel(provider)}:${id}` } +/** Pause gate shared by every request on one service allowance. */ export class ProviderPacer { readonly service: string - private nextIssueSeq = 0 - private episodeSeq = 0 - private consecutiveRateLimits = 0 - private successes = 0 - private limit = BATCH_PROCESSING_CONCURRENCY private pausedUntil = 0 - private active = 0 - private needsProbe = false - private probeEpisode?: number - private probeGeneration = 0 - private stagedRemaining = 0 - private nextReleaseAt = 0 + private consecutiveRateLimits = 0 private timer?: ReturnType private readonly waiters = new Set() private readonly changes = new Set<() => void>() @@ -83,117 +60,52 @@ export class ProviderPacer { readonly onDidChange: Event = (listener, thisArgument, disposables) => { const callback = () => listener.call(thisArgument, undefined) this.changes.add(callback) - const subscription: Disposable = { - dispose: () => { - this.changes.delete(callback) - }, - } + const subscription: Disposable = { dispose: () => void this.changes.delete(callback) } disposables?.push(subscription) return subscription } - state(): ProviderPacerState { - return { - pausedUntil: this.pausedUntil > Date.now() ? this.pausedUntil : undefined, - waitingPaths: [ - ...new Set([...this.waiters].flatMap((waiter) => (waiter.path === undefined ? [] : [waiter.path]))), - ], - limit: this.limit, - episodeSeq: this.episodeSeq, - issueSeq: this.nextIssueSeq, - consecutiveRateLimits: this.consecutiveRateLimits, - } + get resumeAt(): number | undefined { + return this.pausedUntil > Date.now() ? this.pausedUntil : undefined } - get hasWaitingRequests(): boolean { - return this.waiters.size > 0 + get waitingPaths(): string[] { + return [...new Set([...this.waiters].flatMap((waiter) => (waiter.path === undefined ? [] : [waiter.path])))] } - markIssued(): number { - return this.nextIssueSeq++ - } - - reportRateLimit(error: unknown, issueSeq: number): void { - const now = Date.now() - const paused = now < this.pausedUntil - const stale = issueSeq < this.episodeSeq - const requestedDelay = new EmbeddingError("Provider rate limited", error).retryAfterMs - // An out-of-range deadline would break the ISO resume time, so back off instead. - const providerDelay = - typeof requestedDelay === "number" && - requestedDelay > 0 && - Number.isFinite(new Date(now + requestedDelay).getTime()) - ? requestedDelay - : undefined - if (stale && providerDelay === undefined) return - const newEpisode = !paused && !stale - if (newEpisode) { - this.episodeSeq = this.nextIssueSeq - this.consecutiveRateLimits++ - this.limit = Math.max(1, Math.floor(this.limit / 2)) - this.successes = 0 - } - const base = Math.min( - PACER_MAX_BACKOFF_MS, - 1000 * 2 ** Math.min(30, Math.max(0, this.consecutiveRateLimits - 1)), - ) - const delay = providerDelay ?? base / 2 + (Math.random() * base) / 2 - const extendedUntil = Math.max(this.pausedUntil, now + delay) - if (!newEpisode && extendedUntil === this.pausedUntil) return - this.pausedUntil = extendedUntil - this.needsProbe = true - this.probeGeneration++ - this.probeEpisode = undefined - this.stagedRemaining = 0 - this.nextReleaseAt = 0 - codeIndexLog.info("Provider pacing wait", { - service: this.service, - delayMs: this.pausedUntil - now, - source: providerDelay === undefined ? "backoff" : "provider", - waitingCount: this.state().waitingPaths.length, - }) - this.changed() - this.pump() + get hasWaitingRequests(): boolean { + return this.waiters.size > 0 } async run(path: string | undefined, signal: AbortSignal | undefined, request: () => Promise): Promise { while (true) { - const permit = await this.acquire(path, signal) - let rateLimited = false - const onAbort = () => this.finishProbe(permit) - signal?.addEventListener("abort", onAbort, { once: true }) + if (Date.now() < this.pausedUntil) await this.wait(path, signal) + signal?.throwIfAborted() try { - signal?.throwIfAborted() - // Another completion may pause the allowance before this granted permit is consumed. - if (Date.now() < this.pausedUntil) continue - const issueSeq = this.markIssued() - try { - const result = await request() - if (!signal?.aborted && issueSeq >= this.episodeSeq) { - this.consecutiveRateLimits = 0 - if (++this.successes >= PACER_INCREASE_AFTER) { - this.successes = 0 - this.limit = Math.min(BATCH_PROCESSING_CONCURRENCY, this.limit + 1) - this.changed() - } - } - return result - } catch (error) { - if (issueSeq >= this.episodeSeq) this.successes = 0 - if (classifyIndexingError(error, signal) !== "rate-limit") throw error - rateLimited = true - this.reportRateLimit(error, issueSeq) - } - } finally { - signal?.removeEventListener("abort", onAbort) - this.active-- - if (!rateLimited) this.finishProbe(permit) - this.pump() + const result = await request() + this.consecutiveRateLimits = 0 + return result + } catch (error) { + if (classifyIndexingError(error, signal) !== "rate-limit") throw error + this.pause(error) } } } - private acquire(path: string | undefined, signal?: AbortSignal): Promise { + private pause(error: unknown): void { + const now = Date.now() + this.consecutiveRateLimits++ + const requested = new EmbeddingError("Provider rate limited", error).retryAfterMs + const delay = + typeof requested === "number" && requested > 0 && Number.isFinite(new Date(now + requested).getTime()) + ? requested + : Math.min(PACER_MAX_BACKOFF_MS, 1000 * 2 ** Math.min(30, this.consecutiveRateLimits - 1)) + this.pausedUntil = Math.max(this.pausedUntil, now + delay) + this.schedule() + this.changed() + } + + private wait(path: string | undefined, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { if (signal?.aborted) { reject(signal.reason) @@ -206,80 +118,32 @@ export class ProviderPacer { reject, onAbort: () => { this.waiters.delete(waiter) - signal?.removeEventListener("abort", waiter.onAbort) reject(signal?.reason) this.changed() - this.pump() }, } this.waiters.add(waiter) signal?.addEventListener("abort", waiter.onAbort, { once: true }) + this.schedule() this.changed() - this.pump() }) } - private finishProbe(permit: Permit): void { - if (permit.probeEpisode === undefined || permit.probeEpisode !== this.probeEpisode) return - this.probeEpisode = undefined - this.stagedRemaining = this.limit - 1 - this.nextReleaseAt = Date.now() + this.resumeGap() - this.pump() - } - - private resumeGap(): number { - return PACER_RESUME_SPACING_MS * (0.5 + Math.random()) - } - - private pump(): void { + private schedule(): void { if (this.timer !== undefined) clearTimeout(this.timer) - this.timer = undefined - const now = Date.now() - if (now < this.pausedUntil) { - this.schedule(this.pausedUntil - now) - return - } - if (this.pausedUntil !== 0) { - this.pausedUntil = 0 - this.changed() - } - if (this.probeEpisode !== undefined || this.waiters.size === 0 || this.active >= this.limit) return - if (now < this.nextReleaseAt) { - this.schedule(this.nextReleaseAt - now) + const remaining = this.pausedUntil - Date.now() + if (remaining > 0) { + // setTimeout clamps delays above 2^31-1 ms to 1 ms, so long waits run in chunks. + this.timer = setTimeout(() => this.schedule(), Math.min(MAX_TIMEOUT_MS, Math.ceil(remaining))) return } + this.timer = undefined for (const waiter of this.waiters) { - if (this.active >= this.limit) break - this.waiters.delete(waiter) waiter.signal?.removeEventListener("abort", waiter.onAbort) - const permit: Permit = {} - if (this.needsProbe) { - this.needsProbe = false - this.probeEpisode = this.probeGeneration - permit.probeEpisode = this.probeEpisode - } - this.active++ - waiter.resolve(permit) - this.changed() - if (permit.probeEpisode !== undefined) break - if (this.stagedRemaining > 0) { - this.stagedRemaining-- - this.nextReleaseAt = now + this.resumeGap() - if (this.waiters.size > 0) this.schedule(this.nextReleaseAt - now) - break - } + waiter.resolve() } - } - - private schedule(delay: number): void { - // Node clamps oversized timeouts to 1 ms; retain the full deadline and wait in chunks. - this.timer = setTimeout( - () => { - this.timer = undefined - this.pump() - }, - Math.min(MAX_TIMEOUT_MS, Math.ceil(delay)), - ) + this.waiters.clear() + this.changed() } private changed(): void { @@ -294,11 +158,7 @@ export class ProviderPacerRegistry { readonly onDidChange: Event = (listener, thisArgument, disposables) => { const callback = () => listener.call(thisArgument, undefined) this.changes.add(callback) - const subscription: Disposable = { - dispose: () => { - this.changes.delete(callback) - }, - } + const subscription: Disposable = { dispose: () => void this.changes.delete(callback) } disposables?.push(subscription) return subscription } @@ -317,26 +177,18 @@ export class ProviderPacerRegistry { providerWaits(): ProviderWait[] { return [...this.pacers.values()].flatMap((pacer) => { - const state = pacer.state() - return state.pausedUntil === undefined + const resumeAt = pacer.resumeAt + return resumeAt === undefined ? [] - : [ - { - service: pacer.service, - resumeAt: state.pausedUntil, - waitingFiles: state.waitingPaths, - }, - ] + : [{ service: pacer.service, resumeAt, waitingFiles: pacer.waitingPaths }] }) } hasWaitingRequests(): boolean { - return [...this.pacers.values()].some( - (pacer) => pacer.hasWaitingRequests && pacer.state().pausedUntil !== undefined, - ) + return [...this.pacers.values()].some((pacer) => pacer.hasWaitingRequests && pacer.resumeAt !== undefined) } waitingPaths(): string[] { - return [...new Set([...this.pacers.values()].flatMap((pacer) => pacer.state().waitingPaths))] + return [...new Set([...this.pacers.values()].flatMap((pacer) => pacer.waitingPaths))] } } diff --git a/src/services/code-index/search-service.ts b/src/services/code-index/search-service.ts index 6479fd5197..2840f1cd97 100644 --- a/src/services/code-index/search-service.ts +++ b/src/services/code-index/search-service.ts @@ -1,13 +1,14 @@ import * as path from "path" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" import type { VectorStoreSearchResult } from "./interfaces/vector-store" import type { IEmbedder } from "./interfaces/embedder" import type { IVectorStore } from "./interfaces/vector-store" import type { CodeIndexConfigManager } from "./config-manager" import type { CodeIndexStateManager } from "./state-manager" -import { codeIndexLog } from "./code-index-log" import type { FileFailureRegistry } from "./file-failure-registry" +import type { ProviderPacerRegistry } from "./provider-pacer" import { retryWithPolicy } from "./shared/retry-policy" -import { ProviderPacerRegistry } from "./provider-pacer" import { classifyIndexingError, SearchProviderRateLimitError } from "./shared/indexing-errors" import { t } from "../../i18n" @@ -28,22 +29,13 @@ export class CodeIndexSearchService { "isFeatureEnabled" | "isFeatureConfigured" | "currentSearchMinScore" | "currentSearchMaxResults" >, private readonly stateManager: Pick, - private readonly embedder: Pick, + private readonly embedder: Pick, private readonly vectorStore: Pick, - private readonly failures: FileFailureRegistry, - private readonly providerPacers: ProviderPacerRegistry, - private readonly getCurrentStatus: () => Pick< - ReturnType, - "systemStatus" - > = () => this.stateManager.getCurrentStatus(), + private readonly failures: Pick, + private readonly providerPacers: Pick, ) {} - public async searchIndex( - query: string, - directoryPrefix?: string, - signal?: AbortSignal, - ): Promise { - signal?.throwIfAborted() + public async searchIndex(query: string, directoryPrefix?: string): Promise { if (!this.configManager.isFeatureEnabled || !this.configManager.isFeatureConfigured) { throw new Error("Code index feature is disabled or not configured.") } @@ -51,40 +43,25 @@ export class CodeIndexSearchService { const minScore = this.configManager.currentSearchMinScore const maxResults = this.configManager.currentSearchMaxResults - try { - let currentState = this.getCurrentStatus().systemStatus - let partialStandby = false - if ( - currentState === "Standby" && - (this.failures.list().length > 0 || this.providerPacers.waitingPaths().length > 0) - ) { - partialStandby = await this.vectorStore.collectionExists() - signal?.throwIfAborted() - currentState = this.getCurrentStatus().systemStatus - partialStandby &&= - currentState === "Standby" && - (this.failures.list().length > 0 || this.providerPacers.waitingPaths().length > 0) - } - if (currentState !== "Indexed" && currentState !== "Indexing" && !partialStandby) { - throw new Error(`Code index is not ready for search. Current state: ${currentState}`) - } + const currentState = this.stateManager.getCurrentStatus().systemStatus + const partialStandby = + currentState === "Standby" && this.hasOutstandingWork() && (await this.vectorStore.collectionExists()) + if (currentState !== "Indexed" && currentState !== "Indexing" && !partialStandby) { + throw new Error(`Code index is not ready for search. Current state: ${currentState}`) + } - const pacer = this.providerPacers.get(this.embedder.rateLimitScope) + try { const embeddingResponse = await retryWithPolicy( async () => { - const issueSeq = pacer.markIssued() try { - return await this.embedder.createEmbeddings([query], undefined, signal) + return await this.embedder.createEmbeddings([query]) } catch (error) { - signal?.throwIfAborted() if (classifyIndexingError(error) !== "rate-limit") throw error - pacer.reportRateLimit(error, issueSeq) throw new SearchProviderRateLimitError(t("embeddings:searchProviderRateLimited"), error) } }, - { signal, budget: { retriesUsed: 0 } }, + { budget: { retriesUsed: 0 } }, ) - signal?.throwIfAborted() const vector = embeddingResponse?.embeddings[0] if (!vector) { throw new Error("Failed to generate embedding for query.") @@ -96,14 +73,27 @@ export class CodeIndexSearchService { } const results = await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) - signal?.throwIfAborted() - const failedPaths = this.failures.list().map((failure) => failure.path) + const failedPaths = [...new Set(this.failures.list().map((failure) => failure.path))] const waitingPaths = this.providerPacers.waitingPaths() - return { results, incomplete: failedPaths.length > 0 || waitingPaths.length > 0, failedPaths, waitingPaths } + const scanning = this.stateManager.getCurrentStatus().systemStatus === "Indexing" + return { + results, + incomplete: scanning || failedPaths.length > 0 || waitingPaths.length > 0, + failedPaths, + waitingPaths, + } } catch (error) { - signal?.throwIfAborted() - codeIndexLog.error("Search failed", { error }) + console.error("[CodeIndexSearchService] Error during search:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: (error as Error).message, + stack: (error as Error).stack, + location: "searchIndex", + }) throw error } } + + private hasOutstandingWork(): boolean { + return this.failures.list().length > 0 || this.providerPacers.waitingPaths().length > 0 + } } diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts index be323c14ba..ece1da5403 100644 --- a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -9,10 +9,35 @@ import { isSourceMissing, readSource, SOURCE_MISSING, + isQuotaError, } from "../indexing-errors" import { formatEmbeddingError } from "../validation-helpers" import { IndexingRetryError } from "../indexing-errors" +describe("quota 429 classification", () => { + it.each([ + ["code", { status: 429, code: "insufficient_quota" }], + ["type", { status: 429, type: "insufficient_quota" }], + ["nested body", { status: 429, error: { code: "insufficient_quota" } }], + ])("CI-IE-01: insufficient_quota by %s is permanent", (_label, error) => { + expect(classifyIndexingError(error)).toBe("permanent") + expect(isQuotaError(error)).toBe(true) + }) + + it.each([ + ["quota wording", { status: 429, message: "You exceeded your current quota per minute" }], + ["RESOURCE_EXHAUSTED", { status: 429, code: "RESOURCE_EXHAUSTED" }], + ["unknown body", { status: 429, error: { code: "something_else" } }], + ])("CI-IE-02: %s stays rate-limit", (_label, error) => { + expect(classifyIndexingError(error)).toBe("rate-limit") + expect(isQuotaError(error)).toBe(false) + }) + + it("is not a quota error without a 429 status", () => { + expect(isQuotaError({ status: 400, code: "insufficient_quota" })).toBe(false) + }) +}) + vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) describe("T44 isSourceMissing and readSource", () => { diff --git a/src/services/code-index/shared/__tests__/retry-policy.spec.ts b/src/services/code-index/shared/__tests__/retry-policy.spec.ts index f85a5283dc..03377a6e33 100644 --- a/src/services/code-index/shared/__tests__/retry-policy.spec.ts +++ b/src/services/code-index/shared/__tests__/retry-policy.spec.ts @@ -1,4 +1,3 @@ -import "../../__tests__/mock-code-index-log" import { ProviderPacer } from "../../provider-pacer" import { retryWithPolicy, type RetryFailure } from "../retry-policy" import { EmbeddingError, IndexingRetryError } from "../indexing-errors" @@ -48,7 +47,7 @@ it("paces repeated limits even after both retries were consumed, then preserves }) it.each([undefined, 100, 2000, 60_000, 120_000, NaN, -1])( - "rethrows rate limits unchanged before callbacks or budget use: %s", + "CI-RP-03: rethrows rate limits unchanged before callbacks or budget use: %s", async (retryAfterMs) => { const original = Object.freeze(new EmbeddingError("limited", { status: 429 }, { retryAfterMs })) const request = vi.fn().mockRejectedValue(original) @@ -62,7 +61,7 @@ it.each([undefined, 100, 2000, 60_000, 120_000, NaN, -1])( }, ) -it.each([503, { code: "ECONNRESET" }])("retains bounded connection/server waits: %s", async (failure) => { +it.each([503, { code: "ECONNRESET" }])("CI-RP-01/02: at most 3 requests, waits 500 ms then 1 s: %s", async (failure) => { const cause = typeof failure === "number" ? { status: failure } : failure const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", cause)) const failures: RetryFailure[] = [] @@ -79,7 +78,7 @@ it.each([503, { code: "ECONNRESET" }])("retains bounded connection/server waits: expect(request).toHaveBeenCalledTimes(3) }) -it("aborts a long wait without another request", async () => { +it("CI-RP-02: abort ends a wait at once without another request", async () => { const controller = new AbortController() const request = vi.fn().mockRejectedValue(new EmbeddingError("unavailable", { status: 503 })) const pending = retryWithPolicy(request, { signal: controller.signal }).catch((error) => error) @@ -90,7 +89,7 @@ it("aborts a long wait without another request", async () => { expect(request).toHaveBeenCalledOnce() }) -it.each([401, 408, 409])("does not retry permanent HTTP %s", async (status) => { +it.each([401, 408, 409])("CI-RP-04: does not retry permanent HTTP %s", async (status) => { const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", { status })) await expect(retryWithPolicy(request)).rejects.toMatchObject({ attempts: 1, @@ -151,14 +150,14 @@ it("T33 keeps independent metadata when one error is reused across operations", const firstPending = retryWithPolicy(request, { activity: "upsert" }).catch((error) => error) await vi.advanceTimersByTimeAsync(1500) const first = await firstPending - const second = await retryWithPolicy(request, { activity: "verify", budget: { retriesUsed: 2 } }).catch( + const second = await retryWithPolicy(request, { activity: "delete", budget: { retriesUsed: 2 } }).catch( (error) => error, ) expect(first).toBeInstanceOf(IndexingRetryError) expect(second).toBeInstanceOf(IndexingRetryError) expect(first).not.toBe(second) expect(first).toMatchObject({ attempts: 3, activity: "upsert", classification: "server", statusCode: 503 }) - expect(second).toMatchObject({ attempts: 3, activity: "verify", classification: "server", statusCode: 503 }) + expect(second).toMatchObject({ attempts: 3, activity: "delete", classification: "server", statusCode: 503 }) expect((first as IndexingRetryError).cause).toBe(original) expect((second as IndexingRetryError).cause).toBe(original) expect(Object.getOwnPropertyDescriptors(original)).toEqual(properties) diff --git a/src/services/code-index/shared/indexing-errors.ts b/src/services/code-index/shared/indexing-errors.ts index d7a658ccfa..f021ba8a9d 100644 --- a/src/services/code-index/shared/indexing-errors.ts +++ b/src/services/code-index/shared/indexing-errors.ts @@ -13,6 +13,8 @@ type ErrorDetails = { type ErrorShape = ErrorDetails & { name?: string code?: string + type?: string + error?: { code?: string; type?: string } status?: number headers?: RetryAfterHeaders response?: { status?: number } @@ -131,7 +133,7 @@ export function classifyIndexingError(error: unknown, signal?: AbortSignal): Ind if (chain.some((entry) => entry instanceof APIUserAbortError || entry.name === "AbortError")) return "permanent" const status = getIndexingErrorStatusCode(error) if (status !== undefined) { - if (status === 429) return "rate-limit" + if (status === 429) return isQuotaExhausted(chain) ? "permanent" : "rate-limit" return status >= 500 && status < 600 ? "server" : "permanent" } const networkCodes = new Set(["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "EPIPE"]) @@ -154,6 +156,17 @@ export function classifyIndexingError(error: unknown, signal?: AbortSignal): Ind return "permanent" } +// Only the structured OpenAI-style marker counts: per-minute throttle bodies can also mention "quota". +function isQuotaExhausted(chain: ErrorShape[]): boolean { + return chain.some((entry) => + [entry.code, entry.type, entry.error?.code, entry.error?.type].includes("insufficient_quota"), + ) +} + +export function isQuotaError(error: unknown): boolean { + return getIndexingErrorStatusCode(error) === 429 && isQuotaExhausted(errorChain(error)) +} + class IndexingError extends Error { readonly classification: IndexingErrorClassification readonly statusCode?: number diff --git a/src/services/code-index/state-manager.ts b/src/services/code-index/state-manager.ts index 3c8d71f2fd..b678825147 100644 --- a/src/services/code-index/state-manager.ts +++ b/src/services/code-index/state-manager.ts @@ -1,5 +1,4 @@ import * as vscode from "vscode" -import { codeIndexLog } from "./code-index-log" export type IndexingState = "Standby" | "Indexing" | "Indexed" | "Error" | "Stopping" @@ -11,22 +10,6 @@ export class CodeIndexStateManager { private _currentItemUnit: string = "blocks" private _progressEmitter = new vscode.EventEmitter>() - private diagnosticContext: { source: string; attempt: number | string } = { source: "state-manager", attempt: 0 } - - public setDiagnosticContext(context: { source: string; attempt: number | string }): void { - this.diagnosticContext = { ...context } - } - - private logTransition(oldState: IndexingState, source?: string): void { - codeIndexLog.info("State transition", { - oldState, - newState: this._systemStatus, - message: this._statusMessage, - ...this.diagnosticContext, - source: source ?? this.diagnosticContext.source, - }) - } - // --- Public API --- public readonly onProgressUpdate = this._progressEmitter.event @@ -47,12 +30,11 @@ export class CodeIndexStateManager { // --- State Management --- - public setSystemState(newState: IndexingState, message?: string, source?: string): void { + public setSystemState(newState: IndexingState, message?: string): void { const stateChanged = newState !== this._systemStatus || (message !== undefined && message !== this._statusMessage) if (stateChanged) { - const oldState = this._systemStatus this._systemStatus = newState if (message !== undefined) { this._statusMessage = message @@ -69,15 +51,15 @@ export class CodeIndexStateManager { if (newState === "Error" && message === undefined) this._statusMessage = "An error occurred." } - this.logTransition(oldState, source) this._progressEmitter.fire(this.getCurrentStatus()) } } - public reportBlockIndexingProgress(processedItems: number, totalItems: number, source = "block-progress"): void { + public reportBlockIndexingProgress(processedItems: number, totalItems: number): void { const progressChanged = processedItems !== this._processedItems || totalItems !== this._totalItems - if (this._systemStatus === "Error" || this._systemStatus === "Stopping") return + // Don't override Stopping state with progress updates + if (this._systemStatus === "Stopping") return // Update if progress changes OR if the system wasn't already in 'Indexing' state if (progressChanged || this._systemStatus !== "Indexing") { this._processedItems = processedItems @@ -91,7 +73,6 @@ export class CodeIndexStateManager { this._systemStatus = "Indexing" // Ensure state is Indexing this._statusMessage = message - if (oldStatus !== this._systemStatus) this.logTransition(oldStatus, source) // Only fire update if status, message or progress actually changed if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { this._progressEmitter.fire(this.getCurrentStatus()) @@ -99,20 +80,15 @@ export class CodeIndexStateManager { } } - public reportFileQueueProgress( - processedFiles: number, - totalFiles: number, - currentFileBasename?: string, - source = "file-queue-progress", - ): void { + public reportFileQueueProgress(processedFiles: number, totalFiles: number, currentFileBasename?: string): void { const progressChanged = processedFiles !== this._processedItems || totalFiles !== this._totalItems - if (this._systemStatus === "Error" || this._systemStatus === "Stopping") return + // Don't override Stopping state with progress updates + if (this._systemStatus === "Stopping") return if (progressChanged || this._systemStatus !== "Indexing") { this._processedItems = processedFiles this._totalItems = totalFiles this._currentItemUnit = "files" - const oldStatus = this._systemStatus this._systemStatus = "Indexing" let message: string @@ -126,10 +102,10 @@ export class CodeIndexStateManager { message = `File queue processed.` } + const oldStatus = this._systemStatus const oldMessage = this._statusMessage this._statusMessage = message - if (oldStatus !== this._systemStatus) this.logTransition(oldStatus, source) if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { this._progressEmitter.fire(this.getCurrentStatus()) From 35e1aa2563f37b93a0cffa7a251a219ae3f8cafb Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 12 Nov 2026 22:10:51 -0800 Subject: [PATCH 21/42] fix(code-index): route scanner and watcher writes through one writer; stop and recovery fixes --- packages/types/src/vscode-extension-host.ts | 2 +- .../__tests__/cache-manager.spec.ts | 419 ++-- .../code-index-scan-executor.spec.ts | 232 +-- .../__tests__/manager.reliability.spec.ts | 226 +++ .../code-index/__tests__/manager.spec.ts | 1713 +++++++---------- .../code-index/__tests__/orchestrator.spec.ts | 1043 +++------- .../__tests__/service-factory.spec.ts | 188 -- src/services/code-index/cache-manager.ts | 216 ++- .../code-index/code-index-scan-executor.ts | 93 +- .../interfaces/directory-scanner-factory.ts | 12 +- .../interfaces/file-watcher-factory.ts | 12 +- src/services/code-index/manager.ts | 788 ++++---- src/services/code-index/orchestrator.ts | 571 +++--- .../__tests__/file-preparation.spec.ts | 136 +- .../processors/__tests__/file-watcher.spec.ts | 1252 ++++-------- .../processors/__tests__/parser.spec.ts | 40 +- .../processors/__tests__/parser.txt.spec.ts | 3 +- .../processors/__tests__/parser.vb.spec.ts | 2 +- .../__tests__/processor-factories.spec.ts | 22 +- .../processors/__tests__/scanner.spec.ts | 1284 ++++++------ .../processors/directory-scanner-factory.ts | 16 +- .../file-preparation-dependencies.ts | 3 +- .../code-index/processors/file-preparation.ts | 75 +- .../processors/file-watcher-factory.ts | 16 +- .../code-index/processors/file-watcher.ts | 468 ++--- src/services/code-index/processors/parser.ts | 247 ++- src/services/code-index/processors/scanner.ts | 380 ++-- src/services/code-index/service-factory.ts | 43 +- 28 files changed, 3774 insertions(+), 5728 deletions(-) create mode 100644 src/services/code-index/__tests__/manager.reliability.spec.ts diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index 2b714fefd6..50ea4a82df 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -878,7 +878,7 @@ export type WebViewMessagePayload = export interface FailedFileStatus { path: string - activity: "read" | "parse" | "embed" | "upsert" | "delete" | "verify" + activity: "read" | "parse" | "embed" | "upsert" | "delete" reason: string state: "retrying" | "exhausted" } diff --git a/src/services/code-index/__tests__/cache-manager.spec.ts b/src/services/code-index/__tests__/cache-manager.spec.ts index 6831400424..852241c0c3 100644 --- a/src/services/code-index/__tests__/cache-manager.spec.ts +++ b/src/services/code-index/__tests__/cache-manager.spec.ts @@ -1,278 +1,219 @@ +import type { Mock } from "vitest" import * as vscode from "vscode" import { createHash } from "crypto" -import { CacheManager } from "../cache-manager" +import debounce from "lodash.debounce" +import { CacheManager, CacheReadError } from "../cache-manager" + +import { clearAllMocks } from "../../../test-utils/reset" + +// Mock safeWriteJson utility +vitest.mock("../../../utils/safeWriteJson", () => ({ + safeWriteJson: vitest.fn().mockResolvedValue(undefined), +})) + +vitest.mock("../../../i18n", () => ({ t: (key: string) => key })) + +// Import the mocked version import { safeWriteJson } from "../../../utils/safeWriteJson" -import { codeIndexLog } from "../code-index-log" -vi.mock("../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn() })) -vi.mock("../code-index-log", () => ({ codeIndexLog: { error: vi.fn() } })) -vi.mock("vscode", () => ({ - Uri: { joinPath: vi.fn() }, - workspace: { fs: { readFile: vi.fn() } }, +// Mock vscode +vitest.mock("vscode", () => ({ + Uri: { + joinPath: vitest.fn(), + }, + workspace: { + fs: { + readFile: vitest.fn(), + writeFile: vitest.fn(), + delete: vitest.fn(), + }, + }, })) -function deferred() { - let resolve!: () => void - const promise = new Promise((complete) => { - resolve = complete - }) - return { promise, resolve } -} +// Mock debounce to execute immediately +vitest.mock("lodash.debounce", () => ({ + default: vitest.fn(function (fn) { + return fn + }), +})) -describe("CacheManager", () => { - const cachePath = { fsPath: "/storage/cache.json" } as vscode.Uri - // CacheManager uses only globalStorageUri from the extension context. - const context = { globalStorageUri: { fsPath: "/storage" } } as vscode.ExtensionContext - let cache: CacheManager - let persisted: unknown +// Mock TelemetryService +vitest.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { + captureEvent: vitest.fn(), + }, + }, +})) - function freshCache() { - return new CacheManager(context, "/workspace") - } +describe("CacheManager", () => { + let mockContext: vscode.ExtensionContext + let mockWorkspacePath: string + let mockCachePath: vscode.Uri + let cacheManager: CacheManager beforeEach(() => { - vi.useFakeTimers() - vi.resetAllMocks() - persisted = undefined - vi.mocked(vscode.Uri.joinPath).mockReturnValue(cachePath) - vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async () => { - if (persisted === undefined) throw new Error("Missing cache") - return Buffer.from(JSON.stringify(persisted)) + // Reset all mocks + clearAllMocks() + + // Mock context + mockWorkspacePath = "/mock/workspace" + mockCachePath = { fsPath: "/mock/storage/cache.json" } as vscode.Uri + mockContext = { + globalStorageUri: { fsPath: "/mock/storage" } as vscode.Uri, + } as vscode.ExtensionContext + + // Mock Uri.joinPath + ;(vscode.Uri.joinPath as Mock).mockReturnValue(mockCachePath) + + // Create cache manager instance + cacheManager = new CacheManager(mockContext, mockWorkspacePath) + }) + + describe("constructor", () => { + it("should correctly set up cachePath using Uri.joinPath and crypto.createHash", () => { + const expectedHash = createHash("sha256").update(mockWorkspacePath).digest("hex") + + expect(vscode.Uri.joinPath).toHaveBeenCalledWith( + mockContext.globalStorageUri, + `roo-index-cache-${expectedHash}.json`, + ) }) - vi.mocked(safeWriteJson).mockImplementation(async (_path, data: unknown) => { - persisted = structuredClone(data) + + it("should set up debounced save function", () => { + expect(debounce).toHaveBeenCalledWith(expect.any(Function), 1500) }) - cache = freshCache() }) - afterEach(() => { - vi.clearAllTimers() - vi.useRealTimers() - vi.restoreAllMocks() - }) + describe("initialize", () => { + it("should load existing cache file successfully", async () => { + const mockCache = { "file1.ts": "hash1", "file2.ts": "hash2" } + const mockBuffer = Buffer.from(JSON.stringify(mockCache)) + ;(vscode.workspace.fs.readFile as Mock).mockResolvedValue(mockBuffer) - it("uses Zoo's workspace-specific hashed cache path", () => { - const workspaceHash = createHash("sha256").update("/workspace").digest("hex") - expect(vscode.Uri.joinPath).toHaveBeenCalledWith( - context.globalStorageUri, - `roo-index-cache-${workspaceHash}.json`, - ) - }) + await cacheManager.initialize() - it("loads legacy hashes without inventing a generation watermark or changing path keys", async () => { - persisted = { "/workspace/file.ts": "hash", "relative.ts": "relative-hash" } - await cache.initialize() - expect(cache.loadSucceeded).toBe(true) - expect(cache.getAllHashes()).toEqual(persisted) - expect(cache.getHash("/workspace/file.ts")).toBe("hash") - expect(cache.getHash("file.ts")).toBeUndefined() - expect(cache.generationWatermark).toBeUndefined() - }) + expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(mockCachePath) + expect(cacheManager.getAllHashes()).toEqual(mockCache) + }) - it("loads open records ahead of stale hashes and returns independent snapshots", async () => { - persisted = { - version: 2, - fileHashes: { "open.ts": "stale", "complete.ts": "hash" }, - openFiles: { "open.ts": true, "deleted.ts": true }, - generationWatermark: 2048, - } - vi.setSystemTime(12345) - await cache.initialize() - expect(cache.loadedAt).toBe(12345) - expect(cache.loadSucceeded).toBe(true) - expect(cache.generationWatermark).toBe(2048) - expect(cache.getHash("open.ts")).toBeUndefined() - expect(cache.getTrackedPaths().sort()).toEqual(["complete.ts", "deleted.ts", "open.ts"]) - const openFiles = cache.getOpenFiles() - delete openFiles["deleted.ts"] - const hashes = cache.getAllHashes() - hashes["complete.ts"] = "changed" - expect(cache.getOpenFiles()).toEqual({ "open.ts": true, "deleted.ts": true }) - expect(cache.getHash("complete.ts")).toBe("hash") - }) + it.each([ + ["missing (ENOENT)", () => Promise.reject(Object.assign(new Error("gone"), { code: "ENOENT" }))], + ["missing (FileNotFound)", () => Promise.reject(Object.assign(new Error("gone"), { code: "FileNotFound" }))], + ["invalid JSON", () => Promise.resolve(Buffer.from("{not json"))], + ["a non-object", () => Promise.resolve(Buffer.from("[1, 2]"))], + ["non-string hashes", () => Promise.resolve(Buffer.from('{"a.ts": 3}'))], + ])("CI-CM-01: treats a %s cache as empty", async (_label, read) => { + ;(vscode.workspace.fs.readFile as Mock).mockImplementation(read) + await cacheManager.initialize() + expect(cacheManager.getAllHashes()).toEqual({}) + }) - it.each([ - undefined, - "invalid JSON", - null, - [], - { version: 2, fileHashes: { "file.ts": 4 }, openFiles: {} }, - { version: 2, fileHashes: {}, openFiles: { "file.ts": false } }, - { version: 2, fileHashes: [], openFiles: {} }, - { version: 2, fileHashes: {}, openFiles: [] }, - { version: 2, fileHashes: {} }, - { "legacy.ts": 42 }, - ])("resets unreadable or invalid cache state (%j)", async (data: unknown) => { - cache.completeFile("previous.ts", "old") - persisted = data - if (data === "invalid JSON") { - vi.mocked(vscode.workspace.fs.readFile).mockResolvedValueOnce(Buffer.from(data)) - } - vi.setSystemTime(54321) - await cache.initialize() - expect(cache.loadedAt).toBe(54321) - expect(cache.loadSucceeded).toBe(false) - expect(cache.getAllHashes()).toEqual({}) - expect(cache.getOpenFiles()).toEqual({}) - expect(cache.generationWatermark).toBeUndefined() + it("CI-CM-01: an unreadable cache throws CacheReadError and keeps nothing in memory changed", async () => { + cacheManager.updateHash("kept.ts", "hash") + const denied = Object.assign(new Error("denied"), { code: "EACCES" }) + ;(vscode.workspace.fs.readFile as Mock).mockRejectedValue(denied) + const error = await cacheManager.initialize().catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(CacheReadError) + expect((error as CacheReadError).cause).toBe(denied) + expect(cacheManager.getAllHashes()).toEqual({ "kept.ts": "hash" }) + expect(safeWriteJson).not.toHaveBeenCalledWith(expect.anything(), {}) + }) }) - it.each([0, 2048, Number.MAX_SAFE_INTEGER])("loads a valid watermark %s", async (watermark) => { - persisted = { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: watermark } - await cache.initialize() - expect(cache.loadSucceeded).toBe(true) - expect(cache.generationWatermark).toBe(watermark) - }) + describe("hash management", () => { + it("should update hash and trigger save", () => { + const filePath = "test.ts" + const hash = "testhash" - it.each([-1, 0.5, "2048", Number.MAX_SAFE_INTEGER + 1, null])( - "does not trust an invalid watermark %j", - async (watermark) => { - persisted = { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: watermark } - await cache.initialize() - expect(cache.loadSucceeded).toBe(true) - expect(cache.generationWatermark).toBeUndefined() - }, - ) - - it("awaits durable invalidation and reloads an unfinished file without a hash", async () => { - cache.completeFile("file.ts", "hash") - await cache.flush() - const held = deferred() - vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { - await held.promise - persisted = structuredClone(data) + cacheManager.updateHash(filePath, hash) + + expect(cacheManager.getHash(filePath)).toBe(hash) + expect(safeWriteJson).toHaveBeenCalled() }) - const saved = vi.fn() - const invalidation = cache.invalidateFile("file.ts").then(saved) - await vi.advanceTimersByTimeAsync(0) - expect(saved).not.toHaveBeenCalled() - expect(cache.getHash("file.ts")).toBeUndefined() - held.resolve() - await invalidation - const reloaded = freshCache() - await reloaded.initialize() - expect(reloaded.getHash("file.ts")).toBeUndefined() - expect(reloaded.getOpenFiles()).toEqual({ "file.ts": true }) - }) - it("serializes an already-issued older snapshot before durable invalidation", async () => { - const held = deferred() - vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { - await held.promise - persisted = structuredClone(data) + it("should delete hash and trigger save", () => { + const filePath = "test.ts" + const hash = "testhash" + + cacheManager.updateHash(filePath, hash) + cacheManager.deleteHash(filePath) + + expect(cacheManager.getHash(filePath)).toBeUndefined() + expect(safeWriteJson).toHaveBeenCalled() }) - cache.completeFile("file.ts", "old") - await vi.advanceTimersByTimeAsync(1500) - expect(safeWriteJson).toHaveBeenCalledTimes(1) - const invalidation = cache.invalidateFile("file.ts") - await vi.advanceTimersByTimeAsync(0) - expect(safeWriteJson).toHaveBeenCalledTimes(1) - held.resolve() - await invalidation - expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) - await vi.advanceTimersByTimeAsync(1500) - expect(safeWriteJson).toHaveBeenCalledTimes(2) - }) - it("propagates durable-save failure, retains the open record, and allows a later save", async () => { - cache.completeFile("file.ts", "hash") - const failure = new Error("Disk full") - vi.mocked(safeWriteJson).mockRejectedValueOnce(failure) - await expect(cache.invalidateFile("file.ts")).rejects.toBe(failure) - expect(cache.getHash("file.ts")).toBeUndefined() - expect(cache.getOpenFiles()).toEqual({ "file.ts": true }) - await cache.flush() - expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) - }) + it("should return shallow copy of hashes", () => { + const filePath = "test.ts" + const hash = "testhash" - it("debounces completion and removes the open record in the same snapshot as its hash", async () => { - await cache.invalidateFile("file.ts") - vi.mocked(safeWriteJson).mockClear() - cache.completeFile("file.ts", "current") - expect(safeWriteJson).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1499) - expect(safeWriteJson).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(safeWriteJson).toHaveBeenCalledWith(cachePath.fsPath, { - version: 2, - fileHashes: { "file.ts": "current" }, - openFiles: {}, - generationWatermark: undefined, + cacheManager.updateHash(filePath, hash) + const hashes = cacheManager.getAllHashes() + + // Modify the returned object + hashes[filePath] = "modified" + + // Original should remain unchanged + expect(cacheManager.getHash(filePath)).toBe(hash) }) }) - it("tracks completed and open files until removal without enumerating paths", async () => { - const enumerate = vi.spyOn(cache, "getTrackedPaths") - cache.completeFile("file.ts", "hash") - expect(cache.isTracked("file.ts")).toBe(true) - expect(cache.isTracked("missing.ts")).toBe(false) - expect(cache.isTracked("toString")).toBe(false) - expect(cache.getHash("toString")).toBeUndefined() - await cache.invalidateFile("file.ts") - expect(cache.isTracked("file.ts")).toBe(true) - expect(cache.getHash("file.ts")).toBeUndefined() - expect(enumerate).not.toHaveBeenCalled() - expect(cache.getTrackedPaths()).toEqual(["file.ts"]) - cache.completeRemoval("file.ts") - await cache.flush() - expect(cache.isTracked("file.ts")).toBe(false) - expect(cache.getTrackedPaths()).toEqual([]) - expect(persisted).toMatchObject({ fileHashes: {}, openFiles: {} }) - }) + describe("saving", () => { + it("should save cache to disk with correct data", async () => { + const filePath = "test.ts" + const hash = "testhash" - it("persists the reserved watermark through hash and open-record clears", async () => { - await cache.reserveGenerations(4096) - cache.completeFile("complete.ts", "hash") - await cache.invalidateFile("open.ts") - await cache.clearCacheFile() - expect(cache.generationWatermark).toBe(4096) - expect(persisted).toEqual({ version: 2, fileHashes: {}, openFiles: {}, generationWatermark: 4096 }) - const reloaded = freshCache() - await reloaded.initialize() - expect(reloaded.generationWatermark).toBe(4096) - }) + cacheManager.updateHash(filePath, hash) - it("does not acknowledge a reservation before it is durable", async () => { - const held = deferred() - vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data: unknown) => { - await held.promise - persisted = structuredClone(data) + expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, expect.any(Object)) + + // Verify the saved data + const savedData = (safeWriteJson as Mock).mock.calls[0][1] + expect(savedData).toEqual({ [filePath]: hash }) }) - const saved = vi.fn() - const reservation = cache.reserveGenerations(1024).then(saved) - await vi.advanceTimersByTimeAsync(0) - expect(saved).not.toHaveBeenCalled() - held.resolve() - await reservation - expect(persisted).toMatchObject({ generationWatermark: 1024 }) - }) - it.each(["clearCacheFile", "flush", "saveNow"] as const)("propagates %s write errors", async (method) => { - vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) - await expect(cache[method]()).rejects.toThrow("Write denied") - }) + it("should handle save errors gracefully", async () => { + const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(function () {}) + ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) - it("propagates generation reservation failure and permits a later reservation", async () => { - vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) - await expect(cache.reserveGenerations(1024)).rejects.toThrow("Write denied") - await cache.reserveGenerations(2048) - expect(persisted).toMatchObject({ generationWatermark: 2048 }) - }) + cacheManager.updateHash("test.ts", "hash") + + // Wait for any pending promises + await new Promise((resolve) => setTimeout(resolve, 0)) - it("keeps legacy hash mutation aliases compatible with Zoo callers", async () => { - cache.updateHash("file.ts", "hash") - expect(cache.getHash("file.ts")).toBe("hash") - await cache.flush() - cache.deleteHash("file.ts") - await cache.flush() - expect(cache.getHash("file.ts")).toBeUndefined() - expect(cache.isTracked("file.ts")).toBe(false) + expect(consoleErrorSpy).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) + + consoleErrorSpy.mockRestore() + }) }) - it("logs debounced-save errors without an unhandled rejection", async () => { - vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) - cache.completeFile("file.ts", "hash") - await vi.advanceTimersByTimeAsync(1500) - expect(codeIndexLog.error).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) + describe("clearCacheFile", () => { + it("should clear cache file and reset state", async () => { + cacheManager.updateHash("test.ts", "hash") + + // Reset the mock to ensure safeWriteJson succeeds for clearCacheFile + ;(safeWriteJson as Mock).mockClear() + ;(safeWriteJson as Mock).mockResolvedValue(undefined) + + await cacheManager.clearCacheFile() + + expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, {}) + expect(cacheManager.getAllHashes()).toEqual({}) + }) + + it("should handle clear errors gracefully", async () => { + const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(function () {}) + ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) + + await cacheManager.clearCacheFile() + + expect(consoleErrorSpy).toHaveBeenCalledWith( + "Failed to clear cache file:", + expect.any(Error), + mockCachePath, + ) + + consoleErrorSpy.mockRestore() + }) }) }) diff --git a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts index 778ead0516..d9a0e39f8d 100644 --- a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts +++ b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts @@ -1,162 +1,134 @@ import { CodeIndexScanExecutor } from "../code-index-scan-executor" import type { IDirectoryScanner } from "../interfaces" -import { ParserFatalError } from "../parser-errors" -import { EmbeddingError, IndexingRetryError } from "../shared/indexing-errors" -vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn() } })) +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) function setup() { const scanner = { scanDirectory: vi.fn() } + const vectorStore = { markIndexingIncomplete: vi.fn().mockResolvedValue(undefined) } const stateManager = { setSystemState: vi.fn(), reportBlockIndexingProgress: vi.fn() } - let current = true return { scanner, + vectorStore, stateManager, - supersede: () => { - current = false - }, - executor: new CodeIndexScanExecutor("/workspace", scanner, stateManager, () => current), + executor: new CodeIndexScanExecutor("/workspace", scanner, vectorStore, stateManager), } } -const methods = ["runFullScan", "runIncrementalScan"] as const - describe("CodeIndexScanExecutor", () => { - it.each(methods)("%s preserves progress and signal wiring", async (method) => { - const { executor, scanner, stateManager } = setup() - const signal = new AbortController().signal - scanner.scanDirectory.mockImplementation(async (directory, _onError, onIndexed, onParsed, receivedSignal) => { - expect(directory).toBe("/workspace") - expect(receivedSignal).toBe(signal) - onParsed?.(2) - onIndexed?.(1) - onParsed?.(1) - onIndexed?.(2) - return { stats: { processed: 2, skipped: 0 }, totalBlockCount: 3 } - }) - await expect(executor[method](signal)).resolves.toBe(true) - expect(stateManager.reportBlockIndexingProgress.mock.calls).toEqual([ - [0, 2, "scan"], - [1, 2, "scan"], - [1, 3, "scan"], - [3, 3, "scan"], - ]) - expect(executor.partial).toBe(false) - }) - - it.each(methods)("%s rejects a missing scanner result", async (method) => { - const { executor } = setup() - await expect(executor[method](new AbortController().signal)).rejects.toThrow("is scanner initialized?") - }) - - it.each(methods)("%s propagates genuine scanner failures unchanged", async (method) => { - const { executor, scanner } = setup() - const failure = new Error("collection failed") - scanner.scanDirectory.mockRejectedValue(failure) - await expect(executor[method](new AbortController().signal)).rejects.toBe(failure) - }) - - it.each(methods)("%s refuses dispatch after cancellation", async (method) => { - const { executor, scanner, stateManager } = setup() - const controller = new AbortController() - controller.abort() - await expect(executor[method](controller.signal)).resolves.toBe(false) - expect(scanner.scanDirectory).not.toHaveBeenCalled() - expect(stateManager.setSystemState).not.toHaveBeenCalled() - }) - - it.each(methods)("%s ignores stopped and superseded callbacks", async (method) => { - for (const ending of ["stop", "supersede"]) { - const { executor, scanner, stateManager, supersede } = setup() + it.each(["runFullScan", "runIncrementalScan"] as const)( + "%s skips failure validation when cancellation occurs during the scan", + async (method) => { + const { executor, scanner } = setup() const controller = new AbortController() - scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { - if (ending === "stop") controller.abort() - else supersede() + scanner.scanDirectory.mockImplementation(async (_path, onError, _onIndexed, onParsed) => { onParsed?.(3) - onIndexed?.(1) - onError?.(new ParserFatalError("parser-abort", "bad.ts")) + onError?.(new Error("batch failure")) + controller.abort() return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 3 } }) await expect(executor[method](controller.signal)).resolves.toBe(false) - expect(stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() - } - }) - - it.each(methods)("%s retains file failures as partial regardless of percentage", async (method) => { - for (const indexed of [0, 8, 9]) { - const { executor, scanner } = setup() - scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { - onParsed?.(10) - onIndexed?.(indexed) - onError?.(new Error("file failed")) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 10, incompleteFiles: ["bad.ts"] } - }) - await expect(executor[method](new AbortController().signal)).resolves.toBe(true) - expect(executor.partial).toBe(true) - } - }) + }, + ) - it.each(methods)("%s accepts an empty workspace", async (method) => { + it("completes an incremental scan with no changed files", async () => { const { executor, scanner } = setup() scanner.scanDirectory.mockResolvedValue({ stats: { processed: 0, skipped: 2 }, totalBlockCount: 0 }) - await expect(executor[method](new AbortController().signal)).resolves.toBe(true) - expect(executor.partial).toBe(false) + await expect(executor.runIncrementalScan(new AbortController().signal)).resolves.toBe(true) }) - it.each(methods)("%s propagates a typed fatal callback", async (method) => { - const { executor, scanner } = setup() - const fatal = new ParserFatalError("parse-timeout", "bad.ts") - scanner.scanDirectory.mockImplementation(async (_directory, onError) => { - onError?.(fatal) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - await expect(executor[method](new AbortController().signal)).rejects.toBe(fatal) + it.each(["runFullScan", "runIncrementalScan"] as const)( + "%s preserves progress and signal wiring", + async (method) => { + const { executor, scanner, vectorStore, stateManager } = setup() + const signal = new AbortController().signal + scanner.scanDirectory.mockImplementation(async (path, _onError, onIndexed, onParsed, receivedSignal) => { + expect(path).toBe("/workspace") + expect(receivedSignal).toBe(signal) + expect(vectorStore.markIndexingIncomplete).toHaveBeenCalledOnce() + onParsed?.(2) + onIndexed?.(1) + onParsed?.(1) + onIndexed?.(2) + return { stats: { processed: 2, skipped: 0 }, totalBlockCount: 3 } + }) + await expect(executor[method](signal)).resolves.toBe(true) + expect(stateManager.reportBlockIndexingProgress.mock.calls).toEqual([ + [0, 2], + [1, 2], + [1, 3], + [3, 3], + ]) + expect(stateManager.setSystemState).toHaveBeenCalledExactlyOnceWith( + "Indexing", + method === "runFullScan" + ? "Services ready. Starting workspace scan..." + : "Checking for new or modified files...", + ) + }, + ) + + it.each(["runFullScan", "runIncrementalScan"] as const)("%s rejects missing results", async (method) => { + const { executor } = setup() + // An unconfigured mock simulates a scanner violating its return contract. + await expect(executor[method](new AbortController().signal)).rejects.toThrow( + method === "runFullScan" + ? "Scan failed, is scanner initialized?" + : "Incremental scan failed, is scanner initialized?", + ) }) - it.each([400, 401, 403, 404, 422, 429, 500, undefined])( - "escalates only all-embedding authentication/configuration HTTP %s failures in a full scan", - async (status) => { + it.each(["runFullScan", "runIncrementalScan"] as const)( + "%s returns cancellation before validating results", + async (method) => { const { executor, scanner } = setup() - const error = new Error("file embedding failed", { - cause: new IndexingRetryError( - new EmbeddingError("provider failed", { status }), - 1, - "permanent", - "embed", - ), - }) - scanner.scanDirectory.mockImplementation(async (_directory, onError, _onIndexed, onParsed) => { - onParsed?.(1) - onError?.(error) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 1 } - }) - const result = executor.runFullScan(new AbortController().signal) - if (status !== undefined && [400, 401, 403, 404, 422].includes(status)) - await expect(result).rejects.toBe(error) - else { - await expect(result).resolves.toBe(true) - expect(executor.partial).toBe(true) - } + const controller = new AbortController() + // Abort before the unconfigured scanner returns a missing result. + controller.abort() + await expect(executor[method](controller.signal)).resolves.toBe(false) + expect(scanner.scanDirectory).toHaveBeenCalledOnce() }, ) - function embeddingFailure(status: number): Error { - return new Error("file embedding failed", { - cause: new IndexingRetryError(new EmbeddingError("provider failed", { status }), 1, "permanent", "embed"), - }) - } + it.each(["runFullScan", "runIncrementalScan"] as const)( + "%s propagates scanner and marker failures unchanged", + async (method) => { + const { executor, scanner, vectorStore } = setup() + const signal = new AbortController().signal + const failure = new Error("scan failed") + scanner.scanDirectory.mockRejectedValueOnce(failure) + await expect(executor[method](signal)).rejects.toBe(failure) + const abort = new DOMException("stopped", "AbortError") + scanner.scanDirectory.mockRejectedValueOnce(abort) + await expect(executor[method](signal)).rejects.toBe(abort) + vectorStore.markIndexingIncomplete.mockRejectedValueOnce(failure) + await expect(executor[method](signal)).rejects.toBe(failure) + expect(scanner.scanDirectory).toHaveBeenCalledTimes(2) + }, + ) - it.each([ - { method: "runIncrementalScan" as const, statuses: [401] }, - { method: "runFullScan" as const, statuses: [401, 500] }, - ])("does not escalate $method with statuses $statuses", async ({ method, statuses }) => { - const { executor, scanner } = setup() - scanner.scanDirectory.mockImplementation(async (_directory, onError, _onIndexed, onParsed) => { - onParsed?.(statuses.length) - for (const status of statuses) onError?.(embeddingFailure(status)) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: statuses.length } - }) - await expect(executor[method](new AbortController().signal)).resolves.toBe(true) - expect(executor.partial).toBe(true) - }) + const scanCases = (["runFullScan", "runIncrementalScan"] as const).flatMap((method) => + [ + { found: 0, indexed: 0, errors: 0, partial: false }, + { found: 3, indexed: 3, errors: 0, partial: false }, + { found: 3, indexed: 0, errors: 3, partial: true }, + { found: 10, indexed: 1, errors: 9, partial: true }, + { found: 10, indexed: 8, errors: 0, partial: true }, + ].map((scanCase) => ({ method, ...scanCase })), + ) + + it.each(scanCases)( + "$method completes with a partial index for $indexed/$found blocks and $errors file failures", + async ({ method, found, indexed, errors, partial }) => { + const { executor, scanner } = setup() + scanner.scanDirectory.mockImplementation(async (_path, onError, onIndexed, onParsed) => { + onParsed?.(found) + onIndexed?.(indexed) + for (let iter = 0; iter < errors; iter++) onError?.(new Error("file failure")) + return { stats: { processed: 1, skipped: 0 }, totalBlockCount: found } + }) + await expect(executor[method](new AbortController().signal)).resolves.toBe(true) + expect(executor.partial).toBe(partial) + }, + ) }) diff --git a/src/services/code-index/__tests__/manager.reliability.spec.ts b/src/services/code-index/__tests__/manager.reliability.spec.ts new file mode 100644 index 0000000000..f253bd3bbe --- /dev/null +++ b/src/services/code-index/__tests__/manager.reliability.spec.ts @@ -0,0 +1,226 @@ +import * as path from "path" +import type * as vscode from "vscode" +import { CodeIndexManager } from "../manager" +import { CodeIndexStateManager } from "../state-manager" +import { CacheManager, CacheReadError } from "../cache-manager" +import type { CodeIndexOrchestrator } from "../orchestrator" + +vi.mock("vscode", () => { + const emitter = function () { + const listeners = new Set<(value: unknown) => void>() + return { + event: (listener: (value: unknown) => void) => { + listeners.add(listener) + return { dispose: () => listeners.delete(listener) } + }, + fire: (value: unknown) => listeners.forEach((listener) => listener(value)), + dispose: () => listeners.clear(), + } + } + return { + EventEmitter: vi.fn().mockImplementation(emitter), + Uri: { joinPath: vi.fn(() => ({ fsPath: "/storage/cache.json" })) }, + workspace: { fs: { readFile: vi.fn() } }, + } +}) +vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: { captureEvent: vi.fn() } } })) +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) + +const workspacePath = path.join(path.sep, "test", "workspace") + +type OrchestratorDouble = Pick + +function makeManager() { + const store = new Map() + const memento = { + get: (key: string, fallback?: unknown) => (store.has(key) ? store.get(key) : fallback), + update: async (key: string, value: unknown) => void store.set(key, value), + } + // The double provides only the context members the manager and cache manager read. + const context = { + workspaceState: memento, + globalState: memento, + globalStorageUri: { fsPath: "/storage" }, + } as unknown as vscode.ExtensionContext + const folderUri = { toString: () => `file://${workspacePath}` } as unknown as vscode.Uri + const stateManager = new CodeIndexStateManager() + const manager = new CodeIndexManager(workspacePath, folderUri, context, stateManager) + // Feature enablement is read from the config manager; these tests only need it to report enabled. + manager["_configManager"] = { + isFeatureEnabled: true, + isFeatureConfigured: true, + } as unknown as (typeof manager)["_configManager"] + manager["_cacheManager"] = {} as unknown as CacheManager + manager["_searchService"] = {} as unknown as (typeof manager)["_searchService"] + return { manager, stateManager } +} + +function attach(manager: CodeIndexManager, orchestrator: OrchestratorDouble) { + // The double provides only start/stop/isScanning/state. + manager["_orchestrator"] = orchestrator as unknown as CodeIndexOrchestrator +} + +function deferred() { + let resolve!: () => void + const promise = new Promise((done) => (resolve = done)) + return { promise, resolve } +} + +describe("CodeIndexManager reliability", () => { + it("CI-MG-01: a repeated start joins the running start", async () => { + const { manager } = makeManager() + const running = deferred() + const orchestrator: OrchestratorDouble = { + startIndexing: vi.fn(() => running.promise), + stopIndexing: vi.fn(async () => {}), + isScanning: true, + state: "Indexing", + } + attach(manager, orchestrator) + + const first = manager.startIndexing() + const second = manager.startIndexing() + running.resolve() + await Promise.all([first, second]) + expect(orchestrator.startIndexing).toHaveBeenCalledOnce() + }) + + it("CI-MG-01: a start while stopping waits for the stop, then starts", async () => { + const { manager } = makeManager() + const stopping = deferred() + const order: string[] = [] + let stopPromise: Promise | undefined + const orchestrator: OrchestratorDouble = { + // Mirrors the orchestrator contract: a start while stopping awaits the stop first. + startIndexing: vi.fn(async () => { + if (stopPromise) await stopPromise + order.push("start") + }), + stopIndexing: vi.fn(() => { + stopPromise = stopping.promise.then(() => void order.push("stopped")) + return stopPromise + }), + isScanning: false, + state: "Stopping", + } + attach(manager, orchestrator) + + const stop = manager.stopIndexing() + const start = manager.startIndexing() + stopping.resolve() + await Promise.all([stop, start]) + expect(order).toEqual(["stopped", "start"]) + }) + + it.each(["handleSettingsChange", "recoverFromError"] as const)( + "CI-MG-02: %s stops the scan and watcher before services are recreated", + async (entry) => { + const { manager } = makeManager() + const order: string[] = [] + const orchestrator: OrchestratorDouble = { + startIndexing: vi.fn(async () => {}), + stopIndexing: vi.fn(async () => void order.push("stop")), + isScanning: false, + state: "Indexed", + } + attach(manager, orchestrator) + manager["_configManager"] = { + isFeatureEnabled: true, + isFeatureConfigured: true, + currentEmbedderProvider: "semble", + loadConfiguration: async () => ({ requiresRestart: true }), + } as unknown as (typeof manager)["_configManager"] + const recreate = vi.spyOn(manager as unknown as { _recreateServices(): Promise }, "_recreateServices") + recreate.mockImplementation(async () => { + await orchestrator.stopIndexing() + order.push("recreate") + }) + + await manager[entry]() + + expect(order[0]).toBe("stop") + if (entry === "handleSettingsChange") expect(order).toEqual(["stop", "recreate"]) + else expect(manager["_orchestrator"]).toBeUndefined() + }, + ) + + it("CI-MG-02: a replacement failure ends in Error and stays restartable", async () => { + const { manager, stateManager } = makeManager() + manager["_configManager"] = { + isFeatureEnabled: true, + isFeatureConfigured: true, + loadConfiguration: async () => ({ requiresRestart: true }), + } as unknown as (typeof manager)["_configManager"] + vi.spyOn(manager as unknown as { _recreateServices(): Promise }, "_recreateServices").mockRejectedValue( + new Error("bad config"), + ) + await expect(manager.handleSettingsChange()).rejects.toThrow("bad config") + expect(stateManager.state).toBe("Error") + }) + + describe("CI-MG-03: status precedence", () => { + const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs: 60_000 }) + + async function withWait(manager: CodeIndexManager) { + const controller = new AbortController() + const pending = manager["pacers"] + .get("openai:1") + .run("w.ts", controller.signal, () => Promise.reject(limited)) + .catch(() => {}) + await new Promise((resolve) => setTimeout(resolve, 0)) + return async () => { + controller.abort() + await pending + } + } + + function withFailure(manager: CodeIndexManager) { + manager["failureRegistry"].renew("f.ts", "h") + manager["failureRegistry"].record("f.ts", "h", "embed", "down", 3, "exhausted") + } + + it.each([ + { base: "Error", wait: true, failure: true, expected: "Error" }, + { base: "Stopping", wait: true, failure: true, expected: "Stopping" }, + { base: "Indexed", wait: true, failure: true, expected: "Indexing" }, + { base: "Indexing", wait: false, failure: true, expected: "Indexing" }, + { base: "Indexed", wait: false, failure: true, expected: "Standby" }, + { base: "Indexed", wait: false, failure: false, expected: "Indexed" }, + { base: "Standby", wait: false, failure: false, expected: "Standby" }, + ] as const)("$base with wait=$wait failure=$failure is $expected", async ({ base, wait, failure, expected }) => { + const { manager, stateManager } = makeManager() + stateManager.setSystemState(base, "base") + if (failure) withFailure(manager) + const release = wait ? await withWait(manager) : async () => {} + try { + const status = manager.getCurrentStatus() + expect(status.systemStatus).toBe(expected) + expect(status.failedFiles).toHaveLength(failure ? 1 : 0) + expect(status.providerWaits).toHaveLength(wait ? 1 : 0) + } finally { + await release() + } + }) + }) + + it("CI-MG-04: a CacheReadError from initialize() ends in Error and keeps existing data", async () => { + const { manager, stateManager } = makeManager() + manager["_cacheManager"] = undefined + const clear = vi.spyOn(CacheManager.prototype, "clearCacheFile") + vi.spyOn(CacheManager.prototype, "initialize").mockRejectedValue(new CacheReadError(new Error("EACCES"))) + const recreate = vi.spyOn(manager as unknown as { _recreateServices(): Promise }, "_recreateServices") + manager["_configManager"] = { + isFeatureEnabled: true, + isFeatureConfigured: true, + loadConfiguration: async () => ({ requiresRestart: false }), + } as unknown as (typeof manager)["_configManager"] + + // The double stands in for ContextProxy; initialize() only passes it to an existing config manager. + await manager.initialize({} as unknown as Parameters[0]) + + expect(stateManager.state).toBe("Error") + expect(recreate).not.toHaveBeenCalled() + expect(clear).not.toHaveBeenCalled() + expect(manager["_cacheManager"]).toBeUndefined() + }) +}) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 4c668b8029..1263aaef14 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -1,263 +1,205 @@ -import * as path from "path" -import * as vscode from "vscode" import { ContextProxy } from "../../../core/config/ContextProxy" +import { makeExtensionContext } from "../../../test-utils/vscode" import { CodeIndexManager } from "../manager" -import { CodeIndexConfigManager } from "../config-manager" -import { CodeIndexStateManager } from "../state-manager" +import { CodeIndexManagerRegistry } from "../code-index-manager-registry" import { CodeIndexServiceFactory } from "../service-factory" import { CodeIndexOrchestrator } from "../orchestrator" import { CodeIndexSearchService } from "../search-service" -import { CacheManager } from "../cache-manager" -import { GenerationAllocator } from "../generation-allocator" -import { clearAllMocks, restoreGlobals } from "../../../test-utils/reset" +import type { MockedClass } from "vitest" +import * as path from "path" import { providerIdentifiers } from "@roo-code/types/provider-identifiers" -import { SembleProvider } from "../semble" -import { codeIndexLog } from "../code-index-log" -import { ParserFatalError } from "../parser-errors" -import { LATE_WRITE_GRACE_MS } from "../index-write-gate" -import fs from "fs/promises" -import type { IDirectoryScanner, IFileWatcher, IVectorStore } from "../interfaces" -import type { BatchProcessingSummary } from "../interfaces/file-processor" -import { ParseWorkerClient } from "../parse-worker-client" -import { retryWithPolicy } from "../shared/retry-policy" - -vi.mock("vscode", () => ({ - EventEmitter: class { - private listeners = new Set<(value: Value) => void>() - event = (listener: (value: Value) => void) => { - this.listeners.add(listener) - return { - dispose: () => { - this.listeners.delete(listener) - }, - } - } - fire(value: Value) { - this.listeners.forEach((listener) => listener(value)) - } - dispose() { - this.listeners.clear() - } - }, - Uri: { - file: (filePath: string) => ({ fsPath: filePath, toString: () => `file://${filePath}` }), - joinPath: (uri: { fsPath: string }, suffix: string) => ({ fsPath: `${uri.fsPath}/${suffix}` }), - }, - workspace: { workspaceFolders: [{ uri: { fsPath: "/workspace" } }] }, -})) -vi.mock("../../../i18n", () => ({ t: (key: string) => key })) -vi.mock("../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: class { - initialize = vi.fn().mockResolvedValue(undefined) - }, -})) -vi.mock("fs/promises", () => ({ default: { readFile: vi.fn().mockResolvedValue("") } })) -vi.mock("ignore", () => ({ default: () => ({ add: vi.fn() }) })) -vi.mock("../code-index-log-host", () => ({ initializeCodeIndexHostLogging: vi.fn() })) -vi.mock("../service-factory") -vi.mock("../orchestrator") -vi.mock("../search-service") -vi.mock("../parse-worker-client") -vi.mock("../cache-manager") -vi.mock("../generation-allocator") -vi.mock("../semble", () => ({ SembleProvider: vi.fn() })) -vi.mock("../code-index-log", () => ({ - codeIndexLog: { - configure: vi.fn(), - info: vi.fn(), - debug: vi.fn(), - error: vi.fn(), - redact: (value: string) => value.replaceAll("configured-secret", "[REDACTED]"), - }, -})) - -function deferred() { - let resolve!: () => void - let reject!: (error: Error) => void - const promise = new Promise((complete, fail) => { - resolve = complete - reject = fail - }) - return { promise, resolve, reject } -} -async function drain() { - for (let iteration = 0; iteration < 30; iteration++) await Promise.resolve() +// Helper: create a mock vscode.Uri from an fsPath +function mockUri(fsPath: string, scheme = "file") { + return { + fsPath, + scheme, + authority: "", + path: fsPath, + toString: (_skipEncoding?: boolean) => `${scheme}://${fsPath}`, + } } -function memento(): vscode.Memento { - const values = new Map() +// Mock vscode module +vi.mock("vscode", () => { + const testPath = require("path") + const testWorkspacePath = testPath.join(testPath.sep, "test", "workspace") return { - keys: () => [...values.keys()], - get(key: string, defaultValue?: Value): Value { - return (values.has(key) ? values.get(key) : defaultValue) as Value + Uri: { + file: (p: string) => ({ + fsPath: p, + scheme: "file", + authority: "", + path: p, + toString: (_skipEncoding?: boolean) => `file://${p}`, + }), + joinPath: vi.fn((...args: any[]) => ({ fsPath: args.join("/") })), }, - update: async (key: string, value: unknown) => { - values.set(key, value) + window: { + activeTextEditor: null, }, + workspace: { + workspaceFolders: [ + { + uri: { + fsPath: testWorkspacePath, + scheme: "file", + authority: "", + path: testWorkspacePath, + toString: (_skipEncoding?: boolean) => `file://${testWorkspacePath}`, + }, + name: "test", + index: 0, + }, + ], + createFileSystemWatcher: vi.fn().mockReturnValue({ + onDidCreate: vi.fn().mockReturnValue({ dispose: vi.fn() }), + onDidChange: vi.fn().mockReturnValue({ dispose: vi.fn() }), + onDidDelete: vi.fn().mockReturnValue({ dispose: vi.fn() }), + dispose: vi.fn(), + }), + getWorkspaceFolder: vi.fn(), + }, + RelativePattern: vi.fn().mockImplementation(function (base: any, pattern: any) { + return { base, pattern } + }), } -} +}) -function extensionContext(): vscode.ExtensionContext { - // The manager and configuration reader use storage, secrets and extension paths only. +// Mock only the essential dependencies +vi.mock("../../../utils/path", () => { + const testPath = require("path") + const testWorkspacePath = testPath.join(testPath.sep, "test", "workspace") return { - workspaceState: memento(), - globalState: memento(), - subscriptions: [], - globalStorageUri: vscode.Uri.file("/storage"), - extensionPath: "/extension", - secrets: { get: vi.fn(), store: vi.fn(), delete: vi.fn() }, - } as unknown as vscode.ExtensionContext -} + getWorkspacePath: vi.fn(() => testWorkspacePath), + } +}) -function configuration(context: vscode.ExtensionContext) { - const proxy = new ContextProxy(context) - const config = new CodeIndexConfigManager(proxy) - vi.spyOn(config, "isFeatureEnabled", "get").mockReturnValue(true) - vi.spyOn(config, "isFeatureConfigured", "get").mockReturnValue(true) - vi.spyOn(config, "currentEmbedderProvider", "get").mockReturnValue("openai") - vi.spyOn(config, "getConfig").mockReturnValue({ - isConfigured: true, - embedderProvider: providerIdentifiers.openai, - openAiOptions: { openAiNativeApiKey: "configured-secret" }, - }) - const load = vi.spyOn(config, "loadConfiguration").mockResolvedValue({ - requiresRestart: false, - configSnapshot, - currentConfig: config.getConfig(), - }) - return { config, proxy, load } -} +// Mock fs/promises for RooIgnoreController +vi.mock("fs/promises", () => ({ + default: { + readFile: vi.fn().mockRejectedValue(new Error("File not found")), // Simulate no .gitignore/.rooignore + }, +})) -const configSnapshot = { - enabled: true, - configured: true, - embedderProvider: providerIdentifiers.openai, -} +// Mock file utils for RooIgnoreController +vi.mock("../../../utils/fs", () => ({ + fileExistsAtPath: vi.fn().mockResolvedValue(false), // Simulate no .rooignore file +})) -// Factory-only fixtures are never invoked; manager tests verify the dependency identities. -const services = { - embedder: { rateLimitScope: "embedding:test" }, - vectorStore: {}, - scanner: {}, - fileWatcher: { dispose: vi.fn() }, -} as unknown as Omit, "parser"> - -function semble() { - const provider = { - initialize: vi.fn().mockResolvedValue(undefined), - startIndexing: vi.fn().mockResolvedValue(undefined), - stopIndexing: vi.fn(), - searchIndex: vi.fn().mockResolvedValue([]), - clearIndexData: vi.fn().mockResolvedValue(undefined), - dispose: vi.fn(), - state: "Indexed" as const, - } - // Semble's public lifecycle surface is sufficient; its CLI fields are not used by the manager. - vi.mocked(SembleProvider).mockImplementation(function () { - return provider as unknown as SembleProvider - }) - return provider -} +// Mock ignore module +vi.mock("ignore", () => ({ + default: vi.fn().mockReturnValue({ + add: vi.fn(), + ignores: vi.fn().mockReturnValue(false), + }), +})) -describe("CodeIndexManager lifecycle", () => { - let manager: CodeIndexManager - let state: CodeIndexStateManager - let context: vscode.ExtensionContext - let configured: ReturnType +vi.mock("../state-manager", () => ({ + CodeIndexStateManager: vi.fn().mockImplementation(function () { + return { + onProgressUpdate: vi.fn(), + getCurrentStatus: vi.fn(), + dispose: vi.fn(), + setSystemState: vi.fn(), + } + }), +})) - beforeEach(() => { - clearAllMocks() - context = extensionContext() - state = new CodeIndexStateManager() - manager = new CodeIndexManager("/workspace", vscode.Uri.file("/workspace"), context, state) - configured = configuration(context) - manager["_configManager"] = configured.config - vi.mocked(fs.readFile).mockResolvedValue("") - vi.mocked(CacheManager.prototype.initialize).mockResolvedValue(undefined) - vi.mocked(CacheManager.prototype.getOpenFiles).mockReturnValue({}) - vi.mocked(CodeIndexServiceFactory.prototype.createServices).mockImplementation(() => ({ - ...services, - parser: manager["parser"]!, - })) - vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockResolvedValue({ valid: true }) - vi.mocked(CodeIndexOrchestrator).mockImplementation(function () { - return orchestratorDouble() - }) - semble() - }) +// Mock TelemetryService +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), + }, + }, +})) - function orchestratorDouble(): CodeIndexOrchestrator { - // These tests exercise manager coordination, not the orchestrator's bounded-stop implementation. - return { - startupResult: Promise.resolve(), - stopped: false, - isScanning: false, - isActive: true, - isNaturallySettledPartial: false, - startIndexing: vi.fn().mockResolvedValue(undefined), - stopIndexing: vi.fn().mockResolvedValue(undefined), - stopWatcher: vi.fn().mockResolvedValue(undefined), - clearIndexData: vi.fn().mockResolvedValue(undefined), - } as unknown as CodeIndexOrchestrator - } +vi.mock("../service-factory") +const MockedCodeIndexServiceFactory = CodeIndexServiceFactory as MockedClass - function replaceOnStart(replacement = orchestratorDouble()) { - const rebuild = vi.fn().mockImplementation(async () => { - manager["_orchestrator"] = replacement - manager["_searchService"] = new CodeIndexSearchService( - configured.config, - state, - services.embedder, - services.vectorStore, - manager.fileFailureRegistry, - manager.providerPacerRegistry, - ) - manager["servicesStopped"] = false - }) - manager["_recreateServices"] = rebuild - return { rebuild, replacement } - } +describe("CodeIndexManager - handleSettingsChange regression", () => { + let mockContext: any + let manager: CodeIndexManager - function recordFailure(failureState: "retrying" | "exhausted" = "exhausted") { - manager.fileFailureRegistry.renew("bad.ts", "revision") - manager.fileFailureRegistry.record("bad.ts", "revision", "embed", "configured-secret failed", 1, failureState) - } + // Define test paths for use in tests + const testWorkspacePath = path.join(path.sep, "test", "workspace") + const testExtensionPath = path.join(path.sep, "test", "extension") + const testStoragePath = path.join(path.sep, "test", "storage") + const testGlobalStoragePath = path.join(path.sep, "test", "global-storage") + const testLogPath = path.join(path.sep, "test", "log") + + beforeEach(() => { + // Clear all instances before each test + CodeIndexManagerRegistry.disposeAll() + + const workspaceStateStore: Record = {} + const globalStateStore: Record = {} + mockContext = { + subscriptions: [], + workspaceState: { + get: vi.fn((key: string, defaultValue?: any) => workspaceStateStore[key] ?? defaultValue), + update: vi.fn(async (key: string, value: any) => { + workspaceStateStore[key] = value + }), + } as any, + globalState: { + get: vi.fn((key: string, defaultValue?: any) => globalStateStore[key] ?? defaultValue), + update: vi.fn(async (key: string, value: any) => { + globalStateStore[key] = value + }), + } as any, + extensionUri: {} as any, + extensionPath: testExtensionPath, + asAbsolutePath: vi.fn(), + storageUri: {} as any, + storagePath: testStoragePath, + globalStorageUri: {} as any, + globalStoragePath: testGlobalStoragePath, + logUri: {} as any, + logPath: testLogPath, + extensionMode: 3, // vscode.ExtensionMode.Test + secrets: {} as any, + environmentVariableCollection: {} as any, + extension: {} as any, + languageModelAccessInformation: {} as any, + } + + manager = CodeIndexManagerRegistry.getOrCreate(mockContext)! + }) afterEach(() => { - manager.dispose() - restoreGlobals() - vi.useRealTimers() + CodeIndexManagerRegistry.disposeAll() }) describe("configuration readiness", () => { - function unloadedConfiguration() { - manager["_configManager"] = undefined - const proxy = new ContextProxy(context) - vi.spyOn(proxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) - vi.spyOn(proxy, "getSecret").mockReturnValue(undefined) - return proxy - } - it("does not regain readiness when a detached configuration finishes loading", async () => { - const proxy = unloadedConfiguration() - const refresh = deferred() - vi.spyOn(proxy, "refreshSecrets").mockReturnValue(refresh.promise) - const initialization = manager.initialize(proxy) + let finishRefresh!: () => void + const contextProxy = new ContextProxy(makeExtensionContext()) + vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) + vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) + vi.spyOn(contextProxy, "refreshSecrets").mockReturnValue( + new Promise((resolve) => { + finishRefresh = resolve + }), + ) + + const initialization = manager.initialize(contextProxy) expect(manager.isConfigurationLoaded).toBe(false) await manager.recoverFromError() - refresh.resolve() + finishRefresh() await initialization expect(manager.isConfigurationLoaded).toBe(false) - expect(CodeIndexServiceFactory).not.toHaveBeenCalled() }) it("keeps failed configuration unloaded until a settings refresh succeeds", async () => { - const proxy = unloadedConfiguration() - vi.spyOn(proxy, "refreshSecrets") - .mockRejectedValueOnce(new Error("secrets refresh failed")) - .mockResolvedValue(undefined) - await expect(manager.initialize(proxy)).rejects.toThrow("secrets refresh failed") + const error = new Error("secrets refresh failed") + const contextProxy = new ContextProxy(makeExtensionContext()) + vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) + vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) + vi.spyOn(contextProxy, "refreshSecrets").mockRejectedValueOnce(error).mockResolvedValue(undefined) + + await expect(manager.initialize(contextProxy)).rejects.toThrow(error) expect(manager.isConfigurationLoaded).toBe(false) await manager.handleSettingsChange() expect(manager.isConfigurationLoaded).toBe(true) @@ -265,825 +207,652 @@ describe("CodeIndexManager lifecycle", () => { }) it("marks disabled configuration loaded only after secrets refresh completes", async () => { - const proxy = unloadedConfiguration() - const refresh = deferred() - vi.spyOn(proxy, "refreshSecrets").mockReturnValue(refresh.promise) - const initialization = manager.initialize(proxy) - expect(proxy.refreshSecrets).toHaveBeenCalledOnce() + let finishRefresh!: () => void + const refresh = new Promise((resolve) => { + finishRefresh = resolve + }) + const contextProxy = new ContextProxy(makeExtensionContext()) + vi.spyOn(contextProxy, "getGlobalState").mockReturnValue({ codebaseIndexEnabled: false }) + vi.spyOn(contextProxy, "getSecret").mockReturnValue(undefined) + vi.spyOn(contextProxy, "refreshSecrets").mockReturnValue(refresh) + + expect(manager.isConfigurationLoaded).toBe(false) + const initialization = manager.initialize(contextProxy) + expect(contextProxy.refreshSecrets).toHaveBeenCalledOnce() expect(manager.isConfigurationLoaded).toBe(false) - refresh.resolve() + finishRefresh() await initialization + expect(manager.isConfigurationLoaded).toBe(true) expect(manager.isFeatureEnabled).toBe(false) expect(manager.isInitialized).toBe(false) + await manager.recoverFromError() expect(manager.isConfigurationLoaded).toBe(false) }) }) - it.each(["initialize", "settings"] as const)( - "loads cache before allocator and recovers load-time grace through %s", - async (entry) => { - vi.useFakeTimers() - vi.setSystemTime(100_000) - vi.mocked(CacheManager.prototype.initialize).mockImplementation(async function (this: CacheManager) { - this.loadedAt = Date.now() - vi.setSystemTime(105_000) - }) - vi.mocked(CacheManager.prototype.getOpenFiles).mockReturnValue({ "open.ts": true }) - configured.load.mockResolvedValue({ - requiresRestart: true, - configSnapshot, - currentConfig: configured.config.getConfig(), + describe("handleSettingsChange", () => { + it("should log background indexing failures", async () => { + const indexingError = new Error("indexing startup failed") + const startIndexing = vi.fn().mockRejectedValue(indexingError) + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined) + Object.defineProperty(manager, "_orchestrator", { + value: { startIndexing, stopIndexing: vi.fn() }, + configurable: true, }) - if (entry === "initialize") await manager.initialize(configured.proxy) - else await manager.handleSettingsChange() - expect(manager["writeGate"].graceDeadline("open.ts")).toBe(100_000 + LATE_WRITE_GRACE_MS) - expect(vi.mocked(CacheManager.prototype.initialize).mock.invocationCallOrder[0]).toBeLessThan( - vi.mocked(GenerationAllocator).mock.invocationCallOrder[0], + + manager["startIndexingInBackground"]() + await Promise.resolve() + + expect(consoleErrorSpy).toHaveBeenCalledWith( + "[CodeIndexManager] Background indexing failed:", + indexingError, ) - }, - ) - - it("shares startup while rebuilding and resolves before a delayed scan", async () => { - const held = deferred() - const scan = deferred() - const { rebuild, replacement } = replaceOnStart() - vi.mocked(replacement.startIndexing).mockReturnValue(scan.promise) - rebuild.mockImplementation(async () => { - await held.promise - manager["_orchestrator"] = replacement + consoleErrorSpy.mockRestore() }) - manager["_searchService"] = new CodeIndexSearchService( - configured.config, - state, - services.embedder, - services.vectorStore, - manager.fileFailureRegistry, - manager.providerPacerRegistry, - ) - const startup = manager.startIndexing() - expect(manager.startIndexing()).toBe(startup) - await drain() - expect(rebuild).toHaveBeenCalledOnce() - held.resolve() - await startup - expect(replacement.startIndexing).toHaveBeenCalledOnce() - scan.resolve() - }) - it("joins only a live active scan's exact startup promise", () => { - const active = orchestratorDouble() - Object.defineProperty(active, "isScanning", { value: true }) - manager["_orchestrator"] = active - expect(manager.startIndexing()).toBe(active.startupResult) - expect(active.startIndexing).not.toHaveBeenCalled() - }) + it("should not throw when called on uninitialized manager (regression test)", async () => { + // This is the core regression test: handleSettingsChange() should not throw + // when called before the manager is initialized (during first-time configuration) - it.each(["stopIndexing", "stopWatcher"] as const)( - "returns the exact shared %s promise without waiting for clear", - (stop) => { - const active = orchestratorDouble() - const shared = deferred() - vi.mocked(active[stop]).mockReturnValue(shared.promise) - manager["_orchestrator"] = active - manager["clearing"] = deferred().promise - expect(manager[stop]()).toBe(shared.promise) - expect(manager[stop]()).toBe(shared.promise) - shared.resolve() - }, - ) + // Ensure manager is not initialized + expect(manager.isInitialized).toBe(false) - it("returns a shared idle stop without services", () => { - expect(manager.stopIndexing()).toBe(manager.stopWatcher()) - }) + // Mock a minimal config manager that simulates first-time configuration + const mockConfigManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), + isFeatureConfigured: true, + isFeatureEnabled: true, + getConfig: vi.fn().mockReturnValue({ + isConfigured: true, + embedderProvider: providerIdentifiers.openai, + modelId: "text-embedding-3-small", + openAiOptions: { openAiNativeApiKey: "test-key" }, + qdrantUrl: "http://localhost:6333", + qdrantApiKey: "test-key", + searchMinScore: 0.4, + }), + } + ;(manager as any)._configManager = mockConfigManager - it.each(["stopIndexing", "stopWatcher", "disable", "feature-disable"] as const)( - "fences a late rejected rebuild after %s", - async (ending) => { - const held = deferred() - const { rebuild } = replaceOnStart() - rebuild.mockReturnValue(held.promise) - const startup = manager.startIndexing() - await drain() - if (ending === "disable") await manager.setWorkspaceEnabled(false) - else if (ending === "feature-disable") { - vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) - await manager.handleSettingsChange() - } else await manager[ending]() - held.reject(new Error("late rebuild")) - await expect(startup).resolves.toBeUndefined() - expect(manager.state).not.toBe("Error") - expect(codeIndexLog.error).not.toHaveBeenCalled() - }, - ) - - it.each(["rebuild", "startup"] as const)("reports and rejects sanitized %s failures", async (phase) => { - const { rebuild, replacement } = replaceOnStart() - if (phase === "rebuild") rebuild.mockRejectedValue(new Error("configured-secret failed")) - else - vi.mocked(replacement.startIndexing).mockImplementation(() => { - replacement.startupResult = Promise.reject(new Error("configured-secret failed")) - return Promise.resolve() + // Mock cache manager + const mockCacheManager = { + initialize: vi.fn(), + clearCacheFile: vi.fn(), + } + ;(manager as any)._cacheManager = mockCacheManager + + // Mock the feature state to simulate valid configuration that would normally trigger restart + vi.spyOn(manager, "isFeatureEnabled", "get").mockReturnValue(true) + vi.spyOn(manager, "isFeatureConfigured", "get").mockReturnValue(true) + + // Mock service factory to handle _recreateServices call + const mockServiceFactoryInstance = { + configManager: mockConfigManager, + workspacePath: testWorkspacePath, + cacheManager: mockCacheManager, + createEmbedder: vi.fn().mockReturnValue({ embedderInfo: { name: "openai" } }), + createVectorStore: vi.fn().mockReturnValue({}), + createDirectoryScanner: vi.fn().mockReturnValue({}), + createFileWatcher: vi.fn().mockReturnValue({ + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + }), + createServices: vi.fn().mockReturnValue({ + embedder: { embedderInfo: { name: "openai" } }, + vectorStore: {}, + scanner: {}, + fileWatcher: { + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + }, + }), + validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), + } + MockedCodeIndexServiceFactory.mockImplementation(function () { + return mockServiceFactoryInstance as any }) - await expect(manager.startIndexing()).rejects.toThrow("configured-secret failed") - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Error", message: "[REDACTED] failed" }) - expect(manager["startInFlight"]).toBeUndefined() - }) - it.each([true, false])("retries startup failure on settings save even if changed=%s", async (requiresRestart) => { - const { rebuild } = replaceOnStart() - rebuild.mockRejectedValueOnce(new Error("bad configuration")) - await expect(manager.startIndexing()).rejects.toThrow("bad configuration") - configured.load.mockResolvedValue({ - requiresRestart, - configSnapshot, - currentConfig: configured.config.getConfig(), + // The key test: this should NOT throw "CodeIndexManager not initialized" error + await expect(manager.handleSettingsChange()).resolves.not.toThrow() + + // Verify that loadConfiguration was called (the method should still work) + expect(mockConfigManager.loadConfiguration).toHaveBeenCalled() }) - await manager.handleSettingsChange() - expect(rebuild).toHaveBeenCalledTimes(2) - expect(manager["startupFailureMessage"]).toBeUndefined() - }) - it("drains a settings rebuild before scanning and shares Start throughout", async () => { - const held = deferred() - const secondHeld = deferred() - const { rebuild, replacement } = replaceOnStart() - rebuild - .mockImplementationOnce(async () => { - await held.promise - manager["_orchestrator"] = replacement - }) - .mockImplementationOnce(async () => { - await secondHeld.promise - manager["_orchestrator"] = replacement + it("should work normally when manager is initialized", async () => { + // Mock a complete config manager with all required properties + const mockConfigManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), + isFeatureConfigured: true, + isFeatureEnabled: true, + getConfig: vi.fn().mockReturnValue({ + isConfigured: true, + embedderProvider: providerIdentifiers.openai, + modelId: "text-embedding-3-small", + openAiOptions: { openAiNativeApiKey: "test-key" }, + qdrantUrl: "http://localhost:6333", + qdrantApiKey: "test-key", + searchMinScore: 0.4, + }), + } + ;(manager as any)._configManager = mockConfigManager + + // Mock cache manager + const mockCacheManager = { + initialize: vi.fn(), + clearCacheFile: vi.fn(), + } + ;(manager as any)._cacheManager = mockCacheManager + + // Simulate an initialized manager by setting the required properties + ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } + ;(manager as any)._searchService = {} + + // Verify manager is considered initialized + expect(manager.isInitialized).toBe(true) + + // Mock the feature state + vi.spyOn(manager, "isFeatureEnabled", "get").mockReturnValue(true) + vi.spyOn(manager, "isFeatureConfigured", "get").mockReturnValue(true) + + // Mock service factory to handle _recreateServices call + const mockServiceFactoryInstance = { + configManager: mockConfigManager, + workspacePath: testWorkspacePath, + cacheManager: mockCacheManager, + createEmbedder: vi.fn().mockReturnValue({ embedderInfo: { name: "openai" } }), + createVectorStore: vi.fn().mockReturnValue({}), + createDirectoryScanner: vi.fn().mockReturnValue({}), + createFileWatcher: vi.fn().mockReturnValue({ + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + }), + createServices: vi.fn().mockReturnValue({ + embedder: { embedderInfo: { name: "openai" } }, + vectorStore: {}, + scanner: {}, + fileWatcher: { + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + }, + }), + validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), + } + MockedCodeIndexServiceFactory.mockImplementation(function () { + return mockServiceFactoryInstance as any }) - manager["_searchService"] = new CodeIndexSearchService( - configured.config, - state, - services.embedder, - services.vectorStore, - manager.fileFailureRegistry, - manager.providerPacerRegistry, - ) - const startup = manager.startIndexing() - await drain() - configured.load.mockResolvedValue({ - requiresRestart: true, - configSnapshot, - currentConfig: configured.config.getConfig(), - }) - const settings = manager.handleSettingsChange() - await drain() - held.resolve() - await drain() - expect(rebuild).toHaveBeenCalledTimes(2) - expect(manager.startIndexing()).toBe(startup) - expect(replacement.startIndexing).not.toHaveBeenCalled() - secondHeld.resolve() - await Promise.all([startup, settings]) - expect(replacement.startIndexing).toHaveBeenCalledOnce() - }) - it.each(["stop", "fatal"] as const)("does not drain pending settings after %s", async (ending) => { - const held = deferred() - const { rebuild, replacement } = replaceOnStart() - rebuild.mockImplementation(async () => { - await held.promise - manager["_orchestrator"] = replacement + // Mock the methods that would be called during restart + const recreateServicesSpy = vi.spyOn(manager as any, "_recreateServices") + + await manager.handleSettingsChange() + + // Verify that the restart sequence was called + expect(mockConfigManager.loadConfiguration).toHaveBeenCalled() + // _recreateServices should be called when requiresRestart is true + expect(recreateServicesSpy).toHaveBeenCalled() + // Note: startIndexing is NOT called by handleSettingsChange - it's only called by initialize() }) - manager["_searchService"] = new CodeIndexSearchService( - configured.config, - state, - services.embedder, - services.vectorStore, - manager.fileFailureRegistry, - manager.providerPacerRegistry, - ) - const startup = manager.startIndexing() - await drain() - configured.load.mockResolvedValue({ - requiresRestart: true, - configSnapshot, - currentConfig: configured.config.getConfig(), + + it("should handle case when config manager is not set", async () => { + // Ensure config manager is not set (edge case) + ;(manager as any)._configManager = undefined + + // This should not throw an error + await expect(manager.handleSettingsChange()).resolves.not.toThrow() }) - const settings = manager.handleSettingsChange() - await drain() - if (ending === "stop") await manager.stopIndexing() - else manager["parserFailed"] = true - held.resolve() - await Promise.all([startup, settings]) - expect(rebuild).toHaveBeenCalledOnce() - expect(replacement.startIndexing).not.toHaveBeenCalled() }) - it("blocks automatic paths on fatal failure but explicit retry rebuilds without recoverFromError", async () => { - manager["parserFailed"] = true - manager["parserFailureMessage"] = "fatal" - const { rebuild, replacement } = replaceOnStart() - const recovery = vi.spyOn(manager, "recoverFromError") - await manager.initialize(configured.proxy) - await manager.handleSettingsChange() - expect(rebuild).not.toHaveBeenCalled() - await manager.startIndexing() - expect(recovery).not.toHaveBeenCalled() - expect(manager["parserFailed"]).toBe(false) - expect(replacement.startIndexing).toHaveBeenCalledOnce() - }) + describe("embedder validation integration", () => { + let mockServiceFactoryInstance: any + let mockStateManager: any + let mockEmbedder: any + let mockVectorStore: any + let mockScanner: any + let mockFileWatcher: any + + beforeEach(() => { + // Mock service factory objects + mockEmbedder = { embedderInfo: { name: "openai" } } + mockVectorStore = {} + mockScanner = {} + mockFileWatcher = { + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + } - it.each(["resolve", "reject"] as const)( - "waits for clear to %s before recreation, without invalidating the clearing owner", - async (ending) => { - await manager.startIndexing() - const old = manager["_orchestrator"]! - const held = deferred() - vi.mocked(old.clearIndexData).mockReturnValue(held.promise) - const clearing = manager.clearIndexData().catch(() => {}) - const oldAttempt = manager["currentAttempt"] - const created = vi.mocked(CodeIndexServiceFactory).mock.calls.length - const startup = manager.startIndexing() - await drain() - expect(manager["currentAttempt"]).toBe(oldAttempt) - expect(vi.mocked(CodeIndexServiceFactory).mock.calls).toHaveLength(created) - if (ending === "resolve") held.resolve() - else held.reject(new Error("delete failed")) - await Promise.all([clearing, startup]) - expect(vi.mocked(CodeIndexServiceFactory).mock.calls).toHaveLength(created + 1) - }, - ) - - it("retains workspace dependencies, retires the old owner, and fences stale fatal callbacks", async () => { - await manager.startIndexing() - const first = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! - const factory = vi.mocked(CodeIndexServiceFactory).mock.calls.at(-1)! - const old = manager["_orchestrator"]! - const parser = manager["parser"]! - await manager.startIndexing() - const second = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! - expect(second[8]).toBe(first[8]) - expect(second[9]).toBe(first[9]) - expect(second[10]).toBe(manager.fileFailureRegistry) - expect(factory[6]).toBe(first[7]) - expect(second[7]).not.toBe(parser) - expect(parser.stop).toHaveBeenCalled() - expect(old.stopWatcher).toHaveBeenCalled() - expect(first[11]!()).toBe(false) - first[12]!(new ParserFatalError("parser-abort", "old.ts"), "stale failure") - expect(manager["parserFailed"]).toBe(false) - expect(CodeIndexServiceFactory.prototype.validateEmbedder).toHaveBeenCalledTimes(2) - expect(factory[8]).toBe(manager.providerPacerRegistry) - expect(first[13]).toBe(manager.providerPacerRegistry) - expect(second[13]).toBe(first[13]) - }) + // Mock service factory instance + mockServiceFactoryInstance = { + createServices: vi.fn().mockReturnValue({ + embedder: mockEmbedder, + vectorStore: mockVectorStore, + scanner: mockScanner, + fileWatcher: mockFileWatcher, + }), + validateEmbedder: vi.fn(), + } - it("aborts real watcher retry backoff when explicit Start replaces its owner", async () => { - vi.useFakeTimers() - await manager.startIndexing() - const { CodeIndexOrchestrator: ActualOrchestrator } = - await vi.importActual("../orchestrator") - const batchStarted = new vscode.EventEmitter() - const batchProgress = new vscode.EventEmitter<{ - processedInBatch: number - totalInBatch: number - currentFile?: string - }>() - const batchFinished = new vscode.EventEmitter() - let watcherSignal!: AbortSignal - const watcher: IFileWatcher = { - initialize: vi.fn(async (signal?: AbortSignal) => { - watcherSignal = signal! - }), - dispose: vi.fn(), - processFile: vi.fn(), - onDidStartBatchProcessing: batchStarted.event, - onBatchProgressUpdate: batchProgress.event, - onDidFinishBatchProcessing: batchFinished.event, - } - const scanner: IDirectoryScanner = { - scanDirectory: vi.fn().mockResolvedValue({ - stats: { processed: 0, skipped: 0 }, - totalBlockCount: 0, - }), - } - // Only startup and completeness writes are used; no vector search or file writes occur. - const store = { - initialize: vi.fn().mockResolvedValue(false), - hasIndexedData: vi.fn().mockResolvedValue(false), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - } as unknown as IVectorStore - // The real orchestrator only retires this worker and waits for physical exit. - const parser = { - stop: vi.fn(), - stopped: Promise.resolve(), - logStopTimeout: vi.fn(), - } as unknown as ParseWorkerClient - const attempt = manager["currentAttempt"] - const active = new ActualOrchestrator( - configured.config, - state, - "/workspace", - manager["_cacheManager"]!, - store, - scanner, - watcher, - parser, - manager["writeGate"], - manager["generationAllocator"]!, - manager.fileFailureRegistry, - () => manager["currentAttempt"] === attempt, - ) - manager["_orchestrator"] = active - await active.startIndexing() - batchStarted.fire(["bad.ts"]) - expect(active.isScanning).toBe(false) - expect(manager.state).toBe("Indexing") - const registry = manager.fileFailureRegistry - registry.renew("bad.ts", "revision") - const operation = vi - .fn<() => Promise>() - .mockRejectedValue(Object.assign(new Error("unavailable"), { status: 503 })) - const retry = retryWithPolicy(operation, { - signal: watcherSignal, - budget: registry.getRetryBudget("bad.ts", "revision", "embed"), - onFailure: (failure) => - registry.record( - "bad.ts", - "revision", - "embed", - "unavailable", - failure.attempt, - failure.willRetry ? "retrying" : "exhausted", - failure.classification, - ), - }).catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - expect(registry.get("bad.ts")).toMatchObject({ state: "retrying", attempt: 1 }) - const startup = manager.startIndexing() - expect(manager.startIndexing()).toBe(startup) - await startup - expect(await retry).toBe(watcherSignal.reason) - expect(watcherSignal.aborted).toBe(true) - expect(watcher.dispose).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(2_000) - expect(operation).toHaveBeenCalledOnce() - expect(registry.getFailure("bad.ts", "revision", "embed")).toBeUndefined() - batchStarted.dispose() - batchProgress.dispose() - batchFinished.dispose() - }) + // Mock the ServiceFactory constructor + MockedCodeIndexServiceFactory.mockImplementation(function () { + return mockServiceFactoryInstance + }) - it("retains failure budgets on automatic rebuild and resets them only on explicit start or re-enable", async () => { - await manager.startIndexing() - recordFailure() - const registry = manager.fileFailureRegistry - const reset = vi.spyOn(registry, "resetAllBudgets") - configured.load.mockResolvedValue({ - requiresRestart: true, - configSnapshot, - currentConfig: configured.config.getConfig(), + // Mock state manager methods directly on the existing instance + mockStateManager = (manager as any)._stateManager + mockStateManager.setSystemState = vi.fn() + + // Mock config manager + const mockConfigManager = { + loadConfiguration: vitest.fn().mockResolvedValue({ requiresRestart: false }), + isFeatureConfigured: true, + isFeatureEnabled: true, + getConfig: vitest.fn().mockReturnValue({ + isConfigured: true, + embedderProvider: providerIdentifiers.openai, + modelId: "text-embedding-3-small", + openAiOptions: { openAiNativeApiKey: "test-key" }, + qdrantUrl: "http://localhost:6333", + qdrantApiKey: "test-key", + searchMinScore: 0.4, + }), + } + ;(manager as any)._configManager = mockConfigManager }) - await manager.handleSettingsChange() - expect(reset).not.toHaveBeenCalled() - expect(registry.hasRevisionFailure("bad.ts", "revision")).toBe(true) - await manager.startIndexing() - expect(reset).toHaveBeenCalledOnce() - await manager.setWorkspaceEnabled(false) - await manager.setWorkspaceEnabled(true) - expect(reset).toHaveBeenCalledTimes(2) - expect(manager.fileFailureRegistry).toBe(registry) - }) - it.each([ - ["Error", false, false, false, "Error"], - ["Stopping", false, false, false, "Stopping"], - ["Standby", true, false, false, "Indexing"], - ["Standby", false, true, false, "Standby"], - ["Indexed", false, false, false, "Indexed"], - ["Indexed", false, false, true, "Standby"], - ] as const)( - "computes precedence from %s scanning=%s partial=%s stopped=%s", - (raw, scanning, partial, stopped, expected) => { - const active = orchestratorDouble() - Object.defineProperties(active, { - isScanning: { value: scanning }, - isNaturallySettledPartial: { value: partial }, + it("should create indexing services without a startup embedder validation request", async () => { + mockServiceFactoryInstance.validateEmbedder.mockResolvedValue({ + valid: false, + error: "Embedder unavailable", }) - manager["_orchestrator"] = active - manager["servicesStopped"] = stopped - state.setSystemState(raw, "original") - expect(manager.state).toBe(expected) - expect(manager.getCurrentStatus().systemStatus).toBe(expected) - }, - ) - it("does not render an idle live watcher as active indexing", () => { - manager["_orchestrator"] = orchestratorDouble() - state.setSystemState("Indexed", "ready") - expect(manager.state).toBe("Indexed") - }) + await manager["_recreateServices"]() - it("publishes state and sanitized registry updates and preserves stop messages", async () => { - const update = vi.fn() - manager.onProgressUpdate(update) - state.setSystemState("Indexed", "ready") - expect(update).toHaveBeenLastCalledWith(expect.objectContaining({ systemStatus: "Indexed" })) - recordFailure("retrying") - expect(update).toHaveBeenLastCalledWith( - expect.objectContaining({ - systemStatus: "Indexing", - failedFiles: [expect.objectContaining({ reason: "[REDACTED] failed" })], - }), - ) - await manager.stopIndexing() - state.setSystemState("Standby", "user stopped") - expect(manager.getCurrentStatus()).toMatchObject({ - systemStatus: "Standby", - message: "user stopped", - failedFiles: [expect.objectContaining({ state: "exhausted" })], + expect(mockServiceFactoryInstance.createServices).toHaveBeenCalled() + expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() + expect(mockStateManager.setSystemState).not.toHaveBeenCalledWith("Error", expect.any(String)) + expect(manager["_orchestrator"]).toBeInstanceOf(CodeIndexOrchestrator) + expect(manager["_searchService"]).toBeInstanceOf(CodeIndexSearchService) }) - state.setSystemState("Stopping", "stopping") - manager["parserFailed"] = true - manager["parserFailureMessage"] = "configured-secret fatal" - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Error", message: "[REDACTED] fatal" }) - }) - it("forwards search cancellation and injects registry plus computed status", async () => { - await manager.startIndexing() - const search = manager["_searchService"]! - const response = { results: [], incomplete: true, failedPaths: ["bad.ts"], waitingPaths: [] } - vi.mocked(search.searchIndex).mockResolvedValue(response) - const controller = new AbortController() - expect(await manager.searchIndex("query", "src", controller.signal)).toBe(response) - expect(search.searchIndex).toHaveBeenCalledWith("query", "src", controller.signal) - const argumentsUsed = vi.mocked(CodeIndexSearchService).mock.calls.at(-1)! - expect(argumentsUsed[4]).toBe(manager.fileFailureRegistry) - manager["parserFailed"] = true - expect(argumentsUsed[5]).toBe(manager.providerPacerRegistry) - expect(argumentsUsed[6]!().systemStatus).toBe("Error") + it("should handle embedder creation failure", async () => { + // Arrange + mockServiceFactoryInstance.createServices.mockImplementation(() => { + throw new Error("Invalid configuration") + }) + + // Act & Assert - should throw the error + await expect((manager as any)._recreateServices()).rejects.toThrow("Invalid configuration") + + // Should not attempt validation if embedder creation fails + expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() + }) }) - it("returns a failure envelope while disabled", async () => { - vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) - await expect(manager.searchIndex("query")).resolves.toEqual({ - results: [], - incomplete: false, - failedPaths: [], - waitingPaths: [], + describe("recoverFromError", () => { + let mockConfigManager: any + let mockCacheManager: any + let mockStateManager: any + + beforeEach(() => { + // Mock config manager + mockConfigManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: false }), + isFeatureConfigured: true, + isFeatureEnabled: true, + getConfig: vi.fn().mockReturnValue({ + isConfigured: true, + embedderProvider: providerIdentifiers.openai, + modelId: "text-embedding-3-small", + openAiOptions: { openAiNativeApiKey: "test-key" }, + qdrantUrl: "http://localhost:6333", + qdrantApiKey: "test-key", + searchMinScore: 0.4, + }), + } + ;(manager as any)._configManager = mockConfigManager + + // Mock cache manager + mockCacheManager = { + initialize: vi.fn(), + clearCacheFile: vi.fn(), + } + ;(manager as any)._cacheManager = mockCacheManager + + // Mock state manager + mockStateManager = (manager as any)._stateManager + mockStateManager.setSystemState = vi.fn() + mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ + systemStatus: "Error", + message: "Failed during initial scan: fetch failed", + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", + }) + + // Mock orchestrator and search service to simulate initialized state + ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn(), state: "Error" } + ;(manager as any)._searchService = {} + ;(manager as any)._serviceFactory = {} }) - recordFailure() - await expect(manager.searchIndex("query")).resolves.toEqual({ - results: [], - incomplete: true, - failedPaths: ["bad.ts"], - waitingPaths: [], + + it("should clear error state when recoverFromError is called", async () => { + // Act + await manager.recoverFromError() + + // Assert + expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "") }) - }) - it("starts newly recreated Semble through the shared helper and ignores query cancellation", async () => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const provider = semble() - await manager.initialize(configured.proxy) - expect(provider.startIndexing).toHaveBeenCalledOnce() - const controller = new AbortController() - controller.abort() - await expect(manager.searchIndex("query", "src", controller.signal)).resolves.toEqual({ - results: [], - incomplete: false, - failedPaths: [], - waitingPaths: [], + it("should reset internal service instances", async () => { + // Verify initial state + expect((manager as any)._configManager).toBeDefined() + expect((manager as any)._serviceFactory).toBeDefined() + expect((manager as any)._orchestrator).toBeDefined() + expect((manager as any)._searchService).toBeDefined() + + // Act + await manager.recoverFromError() + + // Assert - all service instances should be undefined + expect((manager as any)._configManager).toBeUndefined() + expect((manager as any)._serviceFactory).toBeUndefined() + expect((manager as any)._orchestrator).toBeUndefined() + expect((manager as any)._searchService).toBeUndefined() + }) + + it("should make manager report as not initialized after recovery", async () => { + // Verify initial state + expect(manager.isInitialized).toBe(true) + + // Act + await manager.recoverFromError() + + // Assert + expect(manager.isInitialized).toBe(false) }) - expect(provider.searchIndex).toHaveBeenCalledWith("query", "src") - }) - it.each(["repeat", "stopIndexing", "stopWatcher", "clear"] as const)( - "reuses existing Semble and restores status forwarding after %s", - async (action) => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const provider = semble() - await manager.initialize(configured.proxy) - const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] - provider.startIndexing.mockImplementation(async () => { - providerState.setSystemState("Indexed", "Semble ready") + it("should allow re-initialization after recovery", async () => { + // Setup mock for re-initialization + const mockServiceFactoryInstance = { + createServices: vi.fn().mockReturnValue({ + embedder: { embedderInfo: { name: "openai" } }, + vectorStore: {}, + scanner: {}, + fileWatcher: { + onDidStartBatchProcessing: vi.fn(), + onBatchProgressUpdate: vi.fn(), + watch: vi.fn(), + stopWatcher: vi.fn(), + dispose: vi.fn(), + }, + }), + validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), + } + MockedCodeIndexServiceFactory.mockImplementation(function () { + return mockServiceFactoryInstance as any }) - if (action === "clear") await manager.clearIndexData() - else if (action !== "repeat") await manager[action]() - recordFailure() - const reset = vi.spyOn(manager.fileFailureRegistry, "resetAllBudgets") - const rebuild = vi.fn() - manager["_recreateServices"] = rebuild - const startup = manager.startIndexing(configured.proxy) - expect(manager.startIndexing()).toBe(startup) - await startup - expect(SembleProvider).toHaveBeenCalledOnce() - expect(provider.initialize).toHaveBeenCalledOnce() - expect(provider.dispose).not.toHaveBeenCalled() - expect(provider.startIndexing).toHaveBeenCalledTimes(2) - expect(rebuild).not.toHaveBeenCalled() - expect(reset).not.toHaveBeenCalled() - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Indexed", message: "Semble ready" }) - }, - ) - - it.each(["workspace", "feature"] as const)("does not start existing Semble when %s is disabled", async (scope) => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const provider = semble() - await manager.initialize(configured.proxy) - if (scope === "workspace") await manager.setWorkspaceEnabled(false) - else vi.spyOn(configured.config, "isFeatureEnabled", "get").mockReturnValue(false) - await manager.startIndexing() - expect(provider.startIndexing).toHaveBeenCalledOnce() - expect(SembleProvider).toHaveBeenCalledOnce() - }) - it("retries an existing Semble startup failure without rebuilding", async () => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const provider = semble() - await manager.initialize(configured.proxy) - provider.startIndexing.mockRejectedValueOnce(new Error("Semble unavailable")) - await expect(manager.startIndexing()).rejects.toThrow("Semble unavailable") - expect(manager.state).toBe("Error") - const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] - provider.startIndexing.mockImplementation(async () => { - providerState.setSystemState("Indexed", "Semble recovered") + // Act - recover from error + await manager.recoverFromError() + + // Verify manager is not initialized + expect(manager.isInitialized).toBe(false) + + // Mock context proxy for initialization + const mockContextProxy = { + getValue: vi.fn(), + setValue: vi.fn(), + storeSecret: vi.fn(), + getSecret: vi.fn(), + refreshSecrets: vi.fn().mockResolvedValue(undefined), + getGlobalState: vi.fn().mockReturnValue({ + codebaseIndexEnabled: true, + codebaseIndexQdrantUrl: "http://localhost:6333", + codebaseIndexEmbedderProvider: providerIdentifiers.openai, + codebaseIndexEmbedderModelId: "text-embedding-3-small", + codebaseIndexEmbedderModelDimension: 1536, + codebaseIndexSearchMaxResults: 10, + codebaseIndexSearchMinScore: 0.4, + }), + } + + // Enable workspace indexing before re-initialization + await manager.setWorkspaceEnabled(true) + + // Re-initialize + await manager.initialize(mockContextProxy as any) + + // Assert - manager should be initialized again + expect(manager.isInitialized).toBe(true) + expect(mockServiceFactoryInstance.createServices).toHaveBeenCalled() + expect(mockServiceFactoryInstance.validateEmbedder).not.toHaveBeenCalled() }) - await manager.startIndexing() - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Indexed", message: "Semble recovered" }) - expect(SembleProvider).toHaveBeenCalledOnce() - expect(provider.dispose).not.toHaveBeenCalled() - }) - it("waits for Semble clear before restarting the existing provider", async () => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const provider = semble() - await manager.initialize(configured.proxy) - const held = deferred() - provider.clearIndexData.mockReturnValue(held.promise) - const clearing = manager.clearIndexData() - const startup = manager.startIndexing() - await drain() - expect(provider.startIndexing).toHaveBeenCalledOnce() - await manager.stopIndexing() - held.resolve() - await Promise.all([clearing, startup]) - expect(provider.startIndexing).toHaveBeenCalledOnce() - }) + it("should be safe to call when not in error state (idempotent)", async () => { + // Setup manager in non-error state + mockStateManager.getCurrentStatus.mockReturnValue({ + systemStatus: "Standby", + message: "", + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", + }) - it("recreates Semble after initialization was retired by stop", async () => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const held = deferred() - const retired = semble() - retired.initialize.mockReturnValue(held.promise) - const startup = manager.startIndexing() - await drain() - await manager.stopIndexing() - held.resolve() - await startup - const replacement = semble() - await manager.startIndexing() - expect(retired.dispose).toHaveBeenCalledOnce() - expect(retired.startIndexing).not.toHaveBeenCalled() - expect(replacement.initialize).toHaveBeenCalledOnce() - expect(replacement.startIndexing).toHaveBeenCalledOnce() - }) + // Verify initial state is not error + const initialStatus = manager.getCurrentStatus() + expect(initialStatus.systemStatus).not.toBe("Error") - it.each(["before-create", "during-initialize"] as const)("fences Semble after stop %s", async (phase) => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const held = deferred() - const provider = semble() - if (phase === "before-create") manager["clearing"] = held.promise - else provider.initialize.mockReturnValue(held.promise) - const startup = manager.startIndexing() - await drain() - await manager.stopIndexing() - held.resolve() - await startup - expect(provider.startIndexing).not.toHaveBeenCalled() - if (phase === "before-create") expect(SembleProvider).not.toHaveBeenCalled() - else expect(provider.dispose).toHaveBeenCalled() - }) + // Act - call recoverFromError when not in error state + await expect(manager.recoverFromError()).resolves.not.toThrow() - it("fences late Semble status publications", async () => { - vi.spyOn(configured.config, "currentEmbedderProvider", "get").mockReturnValue("semble") - const held = deferred() - semble().initialize.mockReturnValue(held.promise) - const startup = manager.startIndexing() - await drain() - const providerState = vi.mocked(SembleProvider).mock.calls.at(-1)![2] - await manager.stopIndexing() - providerState.setSystemState("Error", "stale download failure") - held.resolve() - await startup - expect(manager.state).not.toBe("Error") - }) + // Assert - should still clear state and service instances + expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "") + expect((manager as any)._configManager).toBeUndefined() + expect((manager as any)._serviceFactory).toBeUndefined() + expect((manager as any)._orchestrator).toBeUndefined() + expect((manager as any)._searchService).toBeUndefined() + }) - it("recoverFromError retires the old attempt and preserves the fatal latch and cache", async () => { - await manager.startIndexing() - const cache = manager["_cacheManager"] - const old = manager["_orchestrator"]! - const attempt = manager["currentAttempt"] - manager["parserFailed"] = true - await manager.recoverFromError() - expect(old.stopWatcher).toHaveBeenCalled() - expect(manager["currentAttempt"]).toBeGreaterThan(attempt) - expect(manager["parserFailed"]).toBe(true) - expect(manager["_cacheManager"]).toBe(cache) - expect(manager.isInitialized).toBe(false) - expect(manager.isConfigurationLoaded).toBe(false) - }) + it("should continue recovery even if setSystemState throws", async () => { + // Setup state manager to throw on setSystemState + mockStateManager.setSystemState.mockImplementation(() => { + throw new Error("State update failed") + }) + + // Setup manager with service instances + ;(manager as any)._configManager = mockConfigManager + ;(manager as any)._serviceFactory = {} + ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } + ;(manager as any)._searchService = {} + + // Spy on console.error + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}) + + // Act - should not throw despite setSystemState error + await expect(manager.recoverFromError()).resolves.not.toThrow() + + // Assert - error should be logged + expect(consoleErrorSpy).toHaveBeenCalledWith( + "Failed to clear error state during recovery:", + expect.any(Error), + ) - it("continues recovery when publishing Standby throws", async () => { - await manager.startIndexing() - vi.spyOn(state, "setSystemState").mockImplementation(() => { - throw new Error("publication failed") + // Assert - service instances should still be cleared + expect((manager as any)._configManager).toBeUndefined() + expect((manager as any)._serviceFactory).toBeUndefined() + expect((manager as any)._orchestrator).toBeUndefined() + expect((manager as any)._searchService).toBeUndefined() + + // Cleanup + consoleErrorSpy.mockRestore() }) - await expect(manager.recoverFromError()).resolves.toBeUndefined() - expect(manager["_configManager"]).toBeUndefined() - expect(manager["_orchestrator"]).toBeUndefined() }) - it("aggregates actual configured secrets across workspace owners", async () => { - const otherContext = extensionContext() - const otherState = new CodeIndexStateManager() - const other = new CodeIndexManager("/other", vscode.Uri.file("/other"), otherContext, otherState) - try { - other["_configManager"] = configuration(otherContext).config - await manager.initialize(configured.proxy, false) - expect(codeIndexLog.configure).toHaveBeenLastCalledWith({ - workspacePaths: ["/workspace", "/other"], - secrets: ["configured-secret", "configured-secret"], + describe("workspace-enabled gating", () => { + it("should not start indexing when workspace is not enabled", async () => { + await manager.setAutoEnableDefault(false) + + const mockStateManager = (manager as any)._stateManager + mockStateManager.setSystemState = vi.fn() + mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ + systemStatus: "Standby", + message: "", + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", }) - } finally { - other.dispose() - } - }) - it("keeps folder URI enablement distinct and honors auto-enable false", async () => { - const other = new CodeIndexManager( - "/workspace", - Object.assign(vscode.Uri.file("/workspace"), { toString: () => "remote:///workspace" }), - context, - new CodeIndexStateManager(), - ) - try { - await manager.setWorkspaceEnabled(false) - expect(other.isWorkspaceEnabled).toBe(true) + expect(manager.isWorkspaceEnabled).toBe(false) + + await manager.startIndexing() + + expect(mockStateManager.setSystemState).not.toHaveBeenCalledWith("Indexing", expect.any(String)) + }) + + it("should include workspaceEnabled in getCurrentStatus", async () => { await manager.setAutoEnableDefault(false) - expect(other.isWorkspaceEnabled).toBe(false) - } finally { - other.dispose() - } - }) - it("does not construct external services after stop during ignore loading", async () => { - const held = deferred() - vi.mocked(fs.readFile).mockImplementationOnce(async () => { - await held.promise - return "" + const mockStateManager = (manager as any)._stateManager + mockStateManager.getCurrentStatus = vi.fn().mockReturnValue({ + systemStatus: "Standby", + message: "", + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", + }) + + const status = manager.getCurrentStatus() + expect(status.workspaceEnabled).toBe(false) }) - const startup = manager.startIndexing() - await drain() - await manager.stopWatcher() - held.resolve() - await startup - expect(CodeIndexServiceFactory).not.toHaveBeenCalled() - }) - it.each(["paused", "429"])( - "defers %s startup validation and retains one registry across restarts", - async (kind) => { - vi.useFakeTimers() - const { EmbeddingError } = await import("../shared/indexing-errors") - const registry = manager.providerPacerRegistry - const pacer = registry.get(services.embedder.rateLimitScope) - const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }) - const validate = vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder) - if (kind === "paused") pacer.reportRateLimit(error, pacer.markIssued()) - else validate.mockRejectedValueOnce(error) - await manager.startIndexing() - const argumentsUsed = vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)! - expect(argumentsUsed[13]).toBe(registry) - expect(argumentsUsed[14]).toBeTypeOf("function") - expect(validate).toHaveBeenCalledTimes(kind === "paused" ? 0 : 1) - const controller = new AbortController() - const waiting = argumentsUsed[14]!(controller.signal).catch((failure: unknown) => failure) - await drain() - expect(manager.state).toBe("Indexing") - expect(manager.getCurrentStatus().providerWaits).toEqual([ - expect.objectContaining({ resumeAt: Date.now() + 61_000, waitingFiles: [] }), - ]) - controller.abort() - await expect(waiting).resolves.toBe(controller.signal.reason) - await manager.stopIndexing() - await manager.startIndexing() - expect(manager.providerPacerRegistry).toBe(registry) - expect(registry.get(services.embedder.rateLimitScope)).toBe(pacer) - expect(vi.mocked(CodeIndexOrchestrator).mock.calls.at(-1)![14]).toBeTypeOf("function") - }, - ) + it("should persist workspace enabled state", async () => { + await manager.setAutoEnableDefault(false) + expect(manager.isWorkspaceEnabled).toBe(false) - it.each(["resolve", "reject"])("disposes stale parser and watcher after validation %s", async (ending) => { - const held = deferred() - vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockImplementationOnce(async () => { - await held.promise - return { valid: true } + await manager.setWorkspaceEnabled(true) + expect(manager.isWorkspaceEnabled).toBe(true) + + await manager.setWorkspaceEnabled(false) + expect(manager.isWorkspaceEnabled).toBe(false) }) - const startup = manager.startIndexing() - await drain() - const parser = manager["parser"]! - await manager.stopIndexing() - if (ending === "resolve") held.resolve() - else held.reject(new Error("stale configuration error")) - await startup - expect(parser.stop).toHaveBeenCalled() - expect(services.fileWatcher.dispose).toHaveBeenCalled() - expect(CodeIndexOrchestrator).not.toHaveBeenCalled() - expect(CodeIndexSearchService).not.toHaveBeenCalled() - expect(manager["servicesStopped"]).toBe(true) - expect(manager["startupFailureMessage"]).toBeUndefined() - }) - it.each(["invalid", "throw"])("cleans up a non-deferred %s validation failure", async (kind) => { - const validate = vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder) - if (kind === "invalid") validate.mockResolvedValueOnce({ valid: false, error: "invalid configuration" }) - else validate.mockRejectedValueOnce(new Error("invalid configuration")) - await expect(manager.startIndexing()).rejects.toThrow("invalid configuration") - expect(services.fileWatcher.dispose).toHaveBeenCalled() - expect(manager["parser"]!.stop).toHaveBeenCalled() - expect(manager["startupFailureMessage"]).toBe("invalid configuration") - expect(manager.state).toBe("Error") - expect(CodeIndexOrchestrator).not.toHaveBeenCalled() + it("should store enablement per folder URI, not per window", async () => { + CodeIndexManagerRegistry.disposeAll() + + const vscode = await import("vscode") + + const folderAPath = path.join(path.sep, "test", "folderA") + const folderBPath = path.join(path.sep, "test", "folderB") + const folderAUri = mockUri(folderAPath) + const folderBUri = mockUri(folderBPath) + + // Both folders share the same workspaceState (same window) + const sharedStore: Record = {} + const sharedContext = { + ...mockContext, + workspaceState: { + get: vi.fn((key: string, defaultValue?: any) => sharedStore[key] ?? defaultValue), + update: vi.fn(async (key: string, value: any) => { + sharedStore[key] = value + }), + } as any, + globalState: { + get: vi.fn((_key: string, _defaultValue?: any) => false), + update: vi.fn(), + } as any, + } + + // Patch workspaceFolders to include both folders + ;(vscode.workspace as any).workspaceFolders = [ + { uri: folderAUri, name: "folderA", index: 0 }, + { uri: folderBUri, name: "folderB", index: 1 }, + ] + + const managerA = CodeIndexManagerRegistry.getOrCreate(sharedContext, folderAPath)! + const managerB = CodeIndexManagerRegistry.getOrCreate(sharedContext, folderBPath)! + + // Both start disabled (autoEnableDefault is false via globalState mock) + expect(managerA.isWorkspaceEnabled).toBe(false) + expect(managerB.isWorkspaceEnabled).toBe(false) + + // Enable A only + await managerA.setWorkspaceEnabled(true) + + expect(managerA.isWorkspaceEnabled).toBe(true) + expect(managerB.isWorkspaceEnabled).toBe(false) + + // Enable B, disable A + await managerB.setWorkspaceEnabled(true) + await managerA.setWorkspaceEnabled(false) + + expect(managerA.isWorkspaceEnabled).toBe(false) + expect(managerB.isWorkspaceEnabled).toBe(true) + + CodeIndexManagerRegistry.disposeAll() + }) }) - it("a deferred permanent validation failure is Error without startup failure, parser latch or clearing", async () => { - vi.useFakeTimers() - const { EmbeddingError } = await import("../shared/indexing-errors") - const { CodeIndexOrchestrator: ActualOrchestrator } = - await vi.importActual("../orchestrator") - vi.mocked(CodeIndexOrchestrator).mockImplementation(function (...argumentsUsed) { - return new ActualOrchestrator(...argumentsUsed) + describe("stopIndexing", () => { + it("should delegate to orchestrator.stopIndexing()", () => { + const mockOrchestrator = { + stopIndexing: vi.fn(), + stopWatcher: vi.fn(), + state: "Indexing", + } + ;(manager as any)._orchestrator = mockOrchestrator + + manager.stopIndexing() + + expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() }) - const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 1000 }), - pacer.markIssued(), - ) - vi.mocked(CodeIndexServiceFactory.prototype.validateEmbedder).mockResolvedValueOnce({ - valid: false, - error: "bad key", + + it("should be safe to call when orchestrator is not set", () => { + ;(manager as any)._orchestrator = undefined + + expect(() => manager.stopIndexing()).not.toThrow() }) - await manager.startIndexing() - Object.defineProperty(manager["parser"]!, "stopped", { value: Promise.resolve() }) - expect(manager.state).toBe("Indexing") - expect(manager["startupFailureMessage"]).toBeUndefined() - await vi.advanceTimersByTimeAsync(1000) - await drain() - expect(manager.state).toBe("Error") - expect(manager["startupFailureMessage"]).toBeUndefined() - expect(manager["parserFailed"]).toBe(false) - expect(CacheManager.prototype.clearCacheFile).not.toHaveBeenCalled() - expect(manager["_orchestrator"]!["vectorStore"]).toBe(services.vectorStore) }) - it.each(["Error", "Stopping", "Indexed"] as const)("keeps %s during a search-only provider pause", async (raw) => { - vi.useFakeTimers() - const { EmbeddingError } = await import("../shared/indexing-errors") - state.setSystemState(raw, "original") - const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), - pacer.markIssued(), - ) - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: raw, message: "original" }) - expect(manager.getCurrentStatus().providerWaits[0].waitingFiles).toEqual([]) - }) + describe("handleSettingsChange - disable toggle bug fix", () => { + it("should abort active indexing when feature is disabled", async () => { + const mockOrchestrator = { + stopIndexing: vi.fn(), + stopWatcher: vi.fn(), + state: "Indexing", + } + ;(manager as any)._orchestrator = mockOrchestrator - it("publishes redacted relative waiting paths separately from genuine failures", async () => { - vi.useFakeTimers() - const { EmbeddingError } = await import("../shared/indexing-errors") - const update = vi.fn() - manager.onProgressUpdate(update) - recordFailure() - const pacer = manager.providerPacerRegistry.get(services.embedder.rateLimitScope) - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), - pacer.markIssued(), - ) - const controller = new AbortController() - const pending = pacer - .run("/workspace/src/configured-secret.ts", controller.signal, async () => {}) - .catch(() => {}) - await drain() - expect(update).toHaveBeenLastCalledWith( - expect.objectContaining({ - systemStatus: "Indexing", - providerWaits: [expect.objectContaining({ waitingFiles: [path.join("src", "[REDACTED].ts")] })], - failedFiles: [expect.objectContaining({ path: "bad.ts" })], - }), - ) - state.setSystemState("Stopping", "stopping") - controller.abort() - await pending - expect(manager.getCurrentStatus()).toMatchObject({ systemStatus: "Stopping", message: "stopping" }) + const mockConfigManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: false }), + isFeatureConfigured: true, + isFeatureEnabled: false, + } + ;(manager as any)._configManager = mockConfigManager + + const mockStateManager = (manager as any)._stateManager + mockStateManager.setSystemState = vi.fn() + + await manager.handleSettingsChange() + + expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() + expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "Code indexing is disabled") + }) }) }) diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index c633799bbd..c51663e2b7 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -1,854 +1,273 @@ import { CodeIndexOrchestrator } from "../orchestrator" -import { ParserFatalError } from "../parser-errors" -import { INDEX_STOP_LIMIT_MS } from "../constants" import { FileFailureRegistry } from "../file-failure-registry" import { ProviderPacerRegistry } from "../provider-pacer" -import { EmbeddingError } from "../shared/indexing-errors" -import { CodeIndexStateManager } from "../state-manager" -import { codeIndexLog } from "../code-index-log" -import { IndexWriteGate } from "../index-write-gate" -import type { CacheManager } from "../cache-manager" -import type { CodeIndexConfigManager } from "../config-manager" -import type { GenerationAllocator } from "../generation-allocator" -import type { ParseWorkerClient } from "../parse-worker-client" -import type { IDirectoryScanner, IFileWatcher, IVectorStore } from "../interfaces" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" +import type { IDirectoryScanner } from "../interfaces" + import { clearAllMocks } from "../../../test-utils/reset" -vi.mock("../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), error: vi.fn() } })) vi.mock("vscode", () => ({ - workspace: { workspaceFolders: [{ uri: { fsPath: "/workspace" } }] }, - EventEmitter: class { - event = vi.fn() - fire = vi.fn() - dispose = vi.fn() - }, + workspace: { workspaceFolders: [{ uri: { fsPath: "/test/workspace" }, name: "test", index: 0 }] }, })) -vi.mock("../../../i18n", () => ({ t: (key: string) => key })) -const scanResult = { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } -function deferred() { - let resolve!: () => void - let reject!: (error: Error) => void - const promise = new Promise((complete, fail) => { - resolve = complete - reject = fail - }) - return { promise, resolve, reject } -} -async function drain() { - for (let iter = 0; iter < 50; iter++) await Promise.resolve() -} -function fixtures() { - const cache = { - getOpenFiles: vi.fn(() => ({})), - clearCacheFile: vi.fn().mockResolvedValue(undefined), - flush: vi.fn().mockResolvedValue(undefined), - invalidateFile: vi.fn().mockResolvedValue(undefined), +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { instance: { captureEvent: vi.fn() } }, +})) + +vi.mock("../../../i18n", () => ({ + t: (key: string, params?: { errorMessage?: string }) => + params?.errorMessage ? `${key}: ${params.errorMessage}` : key, +})) + +type ScanResult = Awaited> +const emptyScan: ScanResult = { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } + +function setup() { + let currentState = "Standby" + const stateManager = { + get state() { + return currentState + }, + setSystemState: vi.fn((state: string, _message?: string) => { + currentState = state + }), + reportFileQueueProgress: vi.fn(), + reportBlockIndexingProgress: vi.fn(), } - const store = { - initialize: vi.fn().mockResolvedValue(false), - hasIndexedData: vi.fn().mockResolvedValue(false), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - clearCollection: vi.fn().mockResolvedValue(undefined), - deleteCollection: vi.fn().mockResolvedValue(undefined), + const cacheManager = { + clearCacheFile: vi.fn().mockResolvedValue(undefined), + flush: vi.fn().mockResolvedValue(undefined), } - const scanner = { - hasIncompleteFiles: false, - scanDirectory: vi.fn().mockResolvedValue(scanResult), + const vectorStore = { + initialize: vi.fn().mockResolvedValue(false), + hasIndexedData: vi.fn().mockResolvedValue(false), + markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), + markIndexingComplete: vi.fn().mockResolvedValue(undefined), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), } - const watcher = { - isProcessing: false, - initialize: vi.fn().mockResolvedValue(undefined), + const scanner = { scanDirectory: vi.fn().mockResolvedValue(emptyScan) } + const fileWatcher = { + initialize: vi.fn().mockResolvedValue(undefined), + onDidStartBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), + onBatchProgressUpdate: vi.fn().mockReturnValue({ dispose: vi.fn() }), + onDidFinishBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), dispose: vi.fn(), - onDidStartBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), - onBatchProgressUpdate: vi.fn(() => ({ dispose: vi.fn() })), - onDidFinishBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), - onParserFatal: vi.fn>(() => ({ dispose: vi.fn() })), - processFile: vi.fn(), } - const worker = { stop: vi.fn(), stopped: Promise.resolve(), logStopTimeout: vi.fn() } - const gate = new IndexWriteGate() - vi.spyOn(gate, "run") - vi.spyOn(gate, "settled") - vi.spyOn(gate, "hasPending") - vi.spyOn(gate, "abandon") - const allocator = { initialize: vi.fn().mockResolvedValue(undefined) } - return { cache, store, scanner, watcher, worker, gate, allocator } + const failures = new FileFailureRegistry() + const pacers = new ProviderPacerRegistry() + // The doubles provide only the members the orchestrator calls on each collaborator. + type Arguments = ConstructorParameters + const orchestrator = new CodeIndexOrchestrator( + { isFeatureConfigured: true } as unknown as Arguments[0], + stateManager as unknown as Arguments[1], + "/test/workspace", + cacheManager as unknown as Arguments[3], + vectorStore as unknown as Arguments[4], + scanner as unknown as Arguments[5], + fileWatcher as unknown as Arguments[6], + failures, + pacers, + ) + return { orchestrator, stateManager, cacheManager, vectorStore, scanner, fileWatcher, failures, pacers } } -describe("CodeIndexOrchestrator lifecycle", () => { - let state: CodeIndexStateManager - let cache: ReturnType["cache"] - let store: ReturnType["store"] - let scanner: ReturnType["scanner"] - let watcher: ReturnType["watcher"] - let worker: ReturnType["worker"] - let gate: IndexWriteGate - let allocator: ReturnType["allocator"] - let current: boolean - let fatal: ReturnType void>> - let orchestrator: CodeIndexOrchestrator - let failures: FileFailureRegistry - let pacers: ProviderPacerRegistry - - beforeEach(() => { - vi.useFakeTimers() - clearAllMocks() - state = new CodeIndexStateManager() - state.setDiagnosticContext({ source: "manager", attempt: 57 }) - vi.spyOn(state, "setSystemState") - vi.spyOn(state, "reportBlockIndexingProgress") - vi.spyOn(state, "reportFileQueueProgress") - ;({ cache, store, scanner, watcher, worker, gate, allocator } = fixtures()) - current = true - fatal = vi.fn<(error: ParserFatalError, message: string) => void>() - failures = new FileFailureRegistry() - pacers = new ProviderPacerRegistry() - orchestrator = createOrchestrator() +/** A scanner that waits until its signal aborts, then returns normally. */ +function waitForAbort(...args: Parameters) { + const signal = args[4] + return new Promise((resolve) => { + if (signal?.aborted) return resolve(emptyScan) + signal?.addEventListener("abort", () => resolve(emptyScan), { once: true }) }) +} - function createOrchestrator(beforeScan?: (signal: AbortSignal) => Promise) { - // These structural doubles provide only the public lifecycle dependencies used by the owner. - return new CodeIndexOrchestrator( - { isFeatureConfigured: true } as CodeIndexConfigManager, - state, - "/workspace", - cache as unknown as CacheManager, - store as unknown as IVectorStore, - scanner, - watcher, - worker as unknown as ParseWorkerClient, - gate, - allocator as unknown as GenerationAllocator, - failures, - () => current, - fatal, - pacers, - beforeScan, - ) - } - - afterEach(() => vi.useRealTimers()) +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) - it("initializes the allocator after the store and before dispatch", async () => { - const order: string[] = [] - store.initialize.mockImplementation(async () => { - order.push("store") - return false - }) - allocator.initialize.mockImplementation(async () => { - order.push("allocator") - }) - scanner.scanDirectory.mockImplementation(async () => { - order.push("scan") - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }) - await orchestrator.startIndexing() - expect(order).toEqual(["store", "allocator", "scan"]) - expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() - expect(store.markIndexingComplete).toHaveBeenCalledOnce() - expect(store.markIndexingIncomplete).toHaveBeenCalledBefore(scanner.scanDirectory) - expect(store.markIndexingComplete).toHaveBeenCalledAfter(watcher.initialize) - expect(allocator.initialize).toHaveBeenCalledWith(store, expect.any(AbortSignal), gate) - expect(state.state).toBe("Indexed") - }) - - it("exposes startup completion before the scan settles", async () => { - const held = deferred() - scanner.scanDirectory.mockReturnValue(held.promise.then(() => scanResult)) - const scan = orchestrator.startIndexing() - await orchestrator.startupResult - expect(scanner.scanDirectory).toHaveBeenCalledOnce() - expect(state.state).toBe("Indexing") - held.resolve() - await scan - }) +describe("CodeIndexOrchestrator", () => { + beforeEach(() => clearAllMocks()) - it.each(["resolve", "reject"])("settles startup promptly on stop before initialization can %s", async (ending) => { - const held = deferred() - store.initialize.mockReturnValue(held.promise.then(() => false)) - const scan = orchestrator.startIndexing() - const startupSettled = vi.fn() - void orchestrator.startupResult.then(startupSettled, startupSettled) - await drain() - expect(startupSettled).not.toHaveBeenCalled() - const stop = orchestrator.stopIndexing() - await drain() - expect(startupSettled).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stop - expect(state.state).toBe("Standby") - if (ending === "resolve") held.resolve() - else held.reject(new Error("late initialization failure")) - await scan - expect(allocator.initialize).not.toHaveBeenCalled() - expect(scanner.scanDirectory).not.toHaveBeenCalled() - expect(startupSettled).toHaveBeenCalledOnce() - expect(state.state).toBe("Standby") - }) - - it("logs scan, changed-file batch and stop transitions with the original attempt", async () => { - scanner.scanDirectory.mockImplementation(async (_directory, _onError, onIndexed) => { - state.setSystemState("Standby", "Paused", "test") - onIndexed?.(1) - return { stats: { processed: 1, skipped: 0 }, totalBlockCount: 1 } + it.each([false, true])("starts the watcher and completes after a scan (incremental: %s)", async (incremental) => { + const context = setup() + const events: string[] = [] + context.vectorStore.hasIndexedData.mockResolvedValue(incremental) + context.vectorStore.markIndexingIncomplete.mockImplementation(async () => void events.push("incomplete")) + context.scanner.scanDirectory.mockImplementation(async () => { + events.push("scan") + return emptyScan }) - await orchestrator.startIndexing() - expect(codeIndexLog.info).toHaveBeenCalledWith( - "State transition", - expect.objectContaining({ newState: "Indexed", source: "scan-end", attempt: 57 }), - ) - watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) - state.setSystemState("Standby", "Paused", "test") - watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) - watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 2 }) - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) - for (const [newState, source] of [ - ["Indexing", "scan"], - ["Indexing", "changed-file-batch"], - ["Indexed", "changed-file-batch"], - ]) { - expect(codeIndexLog.info).toHaveBeenCalledWith( - "State transition", - expect.objectContaining({ newState, source, attempt: 57 }), - ) - } - watcher.isProcessing = true - await orchestrator.stopIndexing() - for (const newState of ["Stopping", "Standby"]) { - expect(codeIndexLog.info).toHaveBeenCalledWith( - "State transition", - expect.objectContaining({ newState, source: "stop", attempt: 57 }), - ) - } - }) - - it("returns to Indexed when all failed files recover without a new scan", async () => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - await orchestrator.startIndexing() - expect(state.state).toBe("Standby") - expect(store.markIndexingComplete).not.toHaveBeenCalled() - const completion = deferred() - store.markIndexingComplete.mockReturnValue(completion.promise) - failures.clear("bad.ts", "revision") - await drain() - expect(store.markIndexingComplete).toHaveBeenCalledOnce() - expect(state.state).toBe("Standby") - completion.resolve() - await drain() - expect(state.state).toBe("Indexed") - expect(scanner.scanDirectory).toHaveBeenCalledOnce() - }) - - it.each(["stop", "parser", "superseded", "batch"])( - "does not issue recovery completion after queued work becomes %s", - async (ending) => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - await orchestrator.startIndexing() - const queued = deferred() - vi.mocked(gate.run).mockImplementation( - async (_signal: AbortSignal, write: () => Result | Promise) => { - await queued.promise - return write() - }, - ) - failures.clear("bad.ts", "revision") - if (ending === "stop") { - const stopping = orchestrator.stopWatcher() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stopping - } - if (ending === "parser") - watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parser-abort", "bad.ts")) - if (ending === "superseded") current = false - if (ending === "batch") watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) - queued.resolve() - await drain() - expect(store.markIndexingComplete).not.toHaveBeenCalled() - if (ending === "batch") { - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) - await drain() - expect(store.markIndexingComplete).toHaveBeenCalledOnce() - expect(state.state).toBe("Indexed") - } - }, - ) + context.fileWatcher.initialize.mockImplementation(async () => void events.push("watcher")) + context.vectorStore.markIndexingComplete.mockImplementation(async () => void events.push("complete")) - it.each(["stop", "superseded", "failure"])("handles pending recovery completion on %s", async (ending) => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - await orchestrator.startIndexing() - const completion = deferred() - store.markIndexingComplete.mockReturnValue(completion.promise) - failures.clear("bad.ts", "revision") - await drain() - if (ending === "stop") { - const stopping = orchestrator.stopWatcher() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stopping - } - if (ending === "superseded") current = false - vi.mocked(state.setSystemState).mockClear() - if (ending === "failure") completion.reject(new Error("metadata failed")) - else completion.resolve() - await drain() - if (ending === "failure") { - expect(state.state).toBe("Error") - expect(codeIndexLog.error).toHaveBeenCalledWith("Recovery completion failed", { error: expect.any(Error) }) - } else expect(state.setSystemState).not.toHaveBeenCalled() - }) + await context.orchestrator.startIndexing() - it.each(["stop", "parser", "superseded"])("recovery never overrides %s", async (ending) => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - await orchestrator.startIndexing() - if (ending === "stop") { - const stopping = orchestrator.stopWatcher() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stopping - } - if (ending === "parser") watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parser-abort", "bad.ts")) - if (ending === "superseded") current = false - vi.mocked(state.setSystemState).mockClear() - failures.clear("bad.ts", "revision") - expect(state.setSystemState).not.toHaveBeenCalled() + expect(events).toEqual(["incomplete", "scan", "watcher", "complete"]) + expect(context.orchestrator.state).toBe("Indexed") + expect(context.fileWatcher.initialize).toHaveBeenCalledWith(expect.any(AbortSignal)) }) - it("distinguishes active retries, exhausted failures and a user stop", async () => { - await orchestrator.startIndexing() - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "embed", "failed", 1, "retrying") - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Indexing", - message: "embeddings:orchestrator.indexingFilesRetrying", - }) - failures.record("bad.ts", "revision", "embed", "failed", 3, "exhausted") - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Standby", - message: "embeddings:orchestrator.indexingFilesFailed", + it("completes a scan with file failures as idle partial (Standby), without marking it complete", async () => { + const context = setup() + context.scanner.scanDirectory.mockImplementation(async (_dir, onError) => { + context.failures.renew("a.ts", "h") + context.failures.record("a.ts", "h", "embed", "server down", 3, "exhausted") + onError?.(new Error("server down")) + return emptyScan }) - watcher.isProcessing = true - watcher.onDidStartBatchProcessing.mock.calls[0][0]([]) - await orchestrator.stopIndexing() - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Standby", - message: "embeddings:orchestrator.indexingStoppedPartial", - }) - }) - - it("keeps recovery Indexing while a watcher batch remains active", async () => { - await orchestrator.startIndexing() - watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "embed", "retrying", 1, "retrying") - failures.clear("bad.ts", "revision") - expect(state.state).toBe("Indexing") - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [] }) - await drain() - expect(state.state).toBe("Indexed") - }) - - it("refuses scanning after a durable reservation failure without destroying data", async () => { - allocator.initialize.mockRejectedValue(new Error("Storage reservation failed")) - const scan = orchestrator.startIndexing() - await expect(orchestrator.startupResult).rejects.toThrow("Storage reservation failed") - await scan - expect(scanner.scanDirectory).not.toHaveBeenCalled() - expect(state.state).toBe("Error") - expect(store.clearCollection).not.toHaveBeenCalled() - expect(cache.clearCacheFile).not.toHaveBeenCalled() - }) - - it.each(["batch error", "incomplete file", "missing blocks"])( - "does not complete a scan with %s", - async (reason) => { - scanner.scanDirectory.mockImplementation(async (_directory, onError, onIndexed, onParsed) => { - if (reason === "batch error") onError?.(new Error("save failed")) - if (reason === "missing blocks") { - onParsed?.(10) - onIndexed?.(9) - } - return { - stats: { processed: 1, skipped: 0 }, - totalBlockCount: 10, - incompleteFiles: reason === "incomplete file" ? ["bad.ts"] : [], - } - }) - await orchestrator.startIndexing() - expect(store.markIndexingComplete).not.toHaveBeenCalled() - expect(state.state).toBe("Standby") - expect(state.setSystemState).toHaveBeenLastCalledWith( + await context.orchestrator.startIndexing() + expect(context.orchestrator.state).toBe("Standby") + expect(context.vectorStore.markIndexingComplete).not.toHaveBeenCalled() + expect(context.fileWatcher.initialize).toHaveBeenCalled() + + context.failures.clear("a.ts", "h") + expect(context.orchestrator.state).toBe("Indexed") + }) + + it("does not clean up when initialize() fails and ends in Error", async () => { + const context = setup() + context.vectorStore.initialize.mockRejectedValue(new Error("Qdrant unreachable")) + await context.orchestrator.startIndexing() + expect(context.vectorStore.clearCollection).not.toHaveBeenCalled() + expect(context.cacheManager.clearCacheFile).not.toHaveBeenCalled() + expect(context.orchestrator.state).toBe("Error") + }) + + it.each([false, true])("only cleans up a failed scan when this run created the collection: %s", async (created) => { + const context = setup() + context.vectorStore.initialize.mockResolvedValue(created) + context.vectorStore.markIndexingIncomplete.mockRejectedValue(new Error("mark incomplete failure")) + await context.orchestrator.startIndexing() + expect(context.vectorStore.clearCollection).toHaveBeenCalledTimes(created ? 1 : 0) + expect(context.cacheManager.clearCacheFile).toHaveBeenCalledTimes(created ? 2 : 0) + expect(context.orchestrator.state).toBe("Error") + expect(context.fileWatcher.dispose).toHaveBeenCalled() + }) + + it("keeps private error details out of telemetry", async () => { + const context = setup() + const message = "Failed (Workspace: /Users/private-user/Secret Project)" + context.scanner.scanDirectory.mockRejectedValue(new Error(message)) + await context.orchestrator.startIndexing() + expect(vi.mocked(TelemetryService.instance.captureEvent).mock.calls).toEqual([ + [TelemetryEventName.CODE_INDEX_ERROR, { error: "Indexing failed", location: "startIndexing" }], + ]) + expect(context.stateManager.setSystemState).toHaveBeenLastCalledWith("Error", expect.stringContaining(message)) + }) + + describe("stop", () => { + it("CI-OR-01: repeated stops join one promise and end in Standby without clearing data", async () => { + const context = setup() + context.scanner.scanDirectory.mockImplementation(waitForAbort) + const start = context.orchestrator.startIndexing() + await tick() + + const first = context.orchestrator.stopIndexing() + const second = context.orchestrator.stopIndexing() + expect(second).toBe(first) + await Promise.all([first, start]) + + expect(context.orchestrator.state).toBe("Standby") + expect(context.stateManager.setSystemState).toHaveBeenLastCalledWith( "Standby", - "embeddings:orchestrator.indexingFilesFailed", - "scan-end", + "embeddings:orchestrator.indexingStoppedPartial", ) - }, - ) - - it("queue progress never declares a failed watcher batch Indexed", async () => { - await orchestrator.startIndexing() - vi.mocked(state.setSystemState).mockClear() - watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) - watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) - expect(state.state).toBe("Indexing") - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ - processedFiles: [{ path: "/workspace/bad.ts", status: "error" }], + expect(context.cacheManager.flush).toHaveBeenCalledOnce() + expect(context.vectorStore.clearCollection).not.toHaveBeenCalled() + expect(context.cacheManager.clearCacheFile).not.toHaveBeenCalled() + expect(context.fileWatcher.dispose).toHaveBeenCalled() }) - expect(state.state).toBe("Standby") - expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Indexed")).toBe(false) - }) - it.each([ - { - name: "recovers Indexed once a registered failed file clears", - summary: { processedFiles: [{ path: "/workspace/bad.ts", status: "error" as const }] }, - expected: "Indexed", - }, - { - name: "stays partial after a whole-batch error with no processed files", - summary: { processedFiles: [], batchError: new Error("batch failed") }, - expected: "Standby", - }, - ])("watcher batch $name", async ({ summary, expected }) => { - await orchestrator.startIndexing() - watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "read failed", 1, "exhausted") - watcher.onDidFinishBatchProcessing.mock.calls[0][0](summary) - failures.clear("bad.ts", "revision") - await drain() - expect(state.state).toBe(expected) - }) - - it("keeps an unregistered watcher failure partial while another file is registered", async () => { - await orchestrator.startIndexing() - failures.renew("other.ts", "revision") - failures.record("other.ts", "revision", "read", "read failed", 1, "exhausted") - watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ - processedFiles: [{ path: "/workspace/unregistered.ts", status: "local_error" }], - }) - failures.clear("other.ts", "revision") - await drain() - expect(state.state).toBe("Standby") - }) - - it("reports partial when cancellation interrupts a watcher before durable invalidation", async () => { - await orchestrator.startIndexing() - watcher.isProcessing = true - watcher.onDidStartBatchProcessing.mock.calls[0][0](["reading.ts"]) - await orchestrator.stopIndexing() - expect(state.setSystemState).toHaveBeenLastCalledWith( - "Standby", - "embeddings:orchestrator.indexingStoppedPartial", - "stop", - ) - }) - - it("stops the watcher session after a naturally ended scan", async () => { - await orchestrator.startIndexing() - expect(orchestrator.isScanning).toBe(false) - expect(orchestrator.isActive).toBe(true) - const stop = orchestrator.stopIndexing() - expect(orchestrator.stopWatcher()).toBe(stop) - expect(worker.stop).toHaveBeenCalledOnce() - expect(watcher.dispose).toHaveBeenCalledOnce() - await stop - expect(orchestrator.isActive).toBe(false) - }) - - it.each(["scan", "gate", "worker"])("bounds stopping when %s never settles", async (pending) => { - const held = deferred() - if (pending === "scan") scanner.scanDirectory.mockReturnValue(held.promise.then(() => scanResult)) - if (pending === "gate") { - vi.mocked(gate.settled).mockReturnValue(held.promise) - vi.mocked(gate.hasPending).mockReturnValue(true) - } - if (pending === "worker") worker.stopped = held.promise - const scan = orchestrator.startIndexing() - await drain() - watcher.isProcessing = true - const stop = orchestrator.stopIndexing() - expect(orchestrator.stopIndexing()).toBe(stop) - expect(orchestrator.stopWatcher()).toBe(stop) - expect(state.state).toBe("Stopping") - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) - expect(state.state).toBe("Stopping") - await vi.advanceTimersByTimeAsync(1) - await stop - expect(gate.abandon).toHaveBeenCalledOnce() - expect(state.setSystemState).toHaveBeenLastCalledWith( - "Standby", - "embeddings:orchestrator.indexingStoppedPartial", - "stop", - ) - expect(worker.logStopTimeout).toHaveBeenCalledTimes(pending === "worker" ? 1 : 0) - held.resolve() - await scan - expect(store.markIndexingComplete).toHaveBeenCalledTimes(pending === "scan" ? 0 : 1) - }) - - it("waits for all scan, writes and physical worker exit before a fully settled stop", async () => { - const scanFinished = deferred() - const writesFinished = deferred() - const workerFinished = deferred() - scanner.scanDirectory.mockReturnValue(scanFinished.promise.then(() => scanResult)) - vi.mocked(gate.settled).mockReturnValue(writesFinished.promise) - worker.stopped = workerFinished.promise - const scan = orchestrator.startIndexing() - await drain() - const stop = orchestrator.stopIndexing() - scanFinished.resolve() - writesFinished.resolve() - await drain() - expect(state.state).toBe("Stopping") - workerFinished.resolve() - await stop - await scan - expect(state.setSystemState).toHaveBeenLastCalledWith( - "Standby", - "embeddings:orchestrator.indexingStopped", - "stop", - ) - expect(gate.abandon).not.toHaveBeenCalled() - }) - - it("preserves incomplete-file status even when every stop wait settles", async () => { - const scanFinished = deferred() - scanner.scanDirectory.mockReturnValue(scanFinished.promise.then(() => scanResult)) - cache.getOpenFiles.mockReturnValue({ "bad.ts": true }) - void orchestrator.startIndexing() - await drain() - const stop = orchestrator.stopIndexing() - scanFinished.resolve() - await stop - expect(state.setSystemState).toHaveBeenLastCalledWith( - "Standby", - "embeddings:orchestrator.indexingStoppedPartial", - "stop", - ) - }) - - it("does not overwrite a newer attempt when the previous stop expires", async () => { - scanner.scanDirectory.mockReturnValue(deferred().promise.then(() => scanResult)) - void orchestrator.startIndexing() - await drain() - const stop = orchestrator.stopIndexing() - current = false - state.setSystemState("Indexing", "new attempt") - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stop - expect(state.setSystemState).toHaveBeenLastCalledWith("Indexing", "new attempt") - }) - - it.each(["scan", "watcher"])( - "latches fatal %s failure, preserves Error and rejects late callbacks", - async (source) => { - let lateProgress: (count: number) => void = () => {} - if (source === "scan") { - scanner.scanDirectory.mockImplementation(async (_directory, _onError, onProgress) => { - lateProgress = (count) => onProgress?.(count) - throw new ParserFatalError("parser-abort", "bad.ts") + it("CI-OR-01: a stop during a provider wait reaches Standby without waiting for the delay", async () => { + vi.useFakeTimers() + try { + const context = setup() + const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs: 3_600_000 }) + context.scanner.scanDirectory.mockImplementation(async (_dir, _onError, _onIndexed, _onParsed, signal) => { + const pacer = context.pacers.get("openai:1") + await pacer.run("a.ts", signal, () => Promise.reject(limited)).catch(() => {}) + return emptyScan }) + const start = context.orchestrator.startIndexing() + await vi.advanceTimersByTimeAsync(0) + expect(context.pacers.waitingPaths()).toEqual(["a.ts"]) + expect(context.orchestrator.state).toBe("Indexing") + + await context.orchestrator.stopIndexing() + await start + expect(context.orchestrator.state).toBe("Standby") + expect(context.pacers.waitingPaths()).toEqual([]) + expect(context.failures.list()).toEqual([]) + } finally { + vi.useRealTimers() } - await orchestrator.startIndexing() - if (source === "watcher") { - watcher.isProcessing = true - lateProgress = () => - watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) - watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parse-timeout", "bad.ts")) - } - await orchestrator.stopIndexing() - lateProgress(100) - expect(state.state).toBe("Error") - expect(codeIndexLog.info).toHaveBeenCalledWith( - "State transition", - expect.objectContaining({ newState: "Error", source: "error", attempt: 57 }), - ) - expect(vi.mocked(state.setSystemState).mock.calls.some(([next]) => next === "Stopping")).toBe(false) - expect(fatal).toHaveBeenCalledOnce() - expect(worker.stop).toHaveBeenCalledOnce() - expect(store.clearCollection).not.toHaveBeenCalled() - expect(cache.clearCacheFile).not.toHaveBeenCalled() - expect(state.reportBlockIndexingProgress).not.toHaveBeenCalled() - expect(state.reportFileQueueProgress).not.toHaveBeenCalled() - await orchestrator.startIndexing() - expect(scanner.scanDirectory).toHaveBeenCalledOnce() - }, - ) - - it("keeps ordinary scan errors and completed data after cancel", async () => { - scanner.scanDirectory.mockRejectedValue(new Error("network failed")) - await orchestrator.startIndexing() - await orchestrator.stopIndexing() - expect(state.state).toBe("Error") - expect(codeIndexLog.info).toHaveBeenCalledWith( - "State transition", - expect.objectContaining({ newState: "Error", source: "error", attempt: 57 }), - ) - expect(store.clearCollection).not.toHaveBeenCalled() - expect(cache.clearCacheFile).not.toHaveBeenCalled() - }) - it.each([false, true])("chooses the scan before exactly one incomplete mark: %s", async (incremental) => { - const order: string[] = [] - store.hasIndexedData.mockImplementation(async () => { - order.push("choose") - return incremental - }) - store.markIndexingIncomplete.mockImplementation(async () => { - order.push("mark") - }) - const held = deferred() - scanner.scanDirectory.mockImplementation(async () => { - order.push("scan") - await held.promise - return scanResult }) - const scan = orchestrator.startIndexing() - await orchestrator.startupResult - expect(order).toEqual(["choose", "mark", "scan"]) - expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() - held.resolve() - await scan - expect(watcher.initialize).toHaveBeenCalledWith(expect.any(AbortSignal)) - }) - - it.each([false, true])("only cleans up a failed newly created collection: %s", async (created) => { - store.initialize.mockResolvedValue(created) - store.markIndexingIncomplete.mockRejectedValue(new Error("metadata failed")) - await orchestrator.startIndexing() - expect(store.clearCollection).toHaveBeenCalledTimes(created ? 1 : 0) - expect(cache.clearCacheFile).toHaveBeenCalledTimes(created ? 2 : 0) - expect(state.state).toBe("Error") - }) - it("keeps partial summary until recovery and stopped text until a new start", async () => { - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "failed", 1, "exhausted") - await orchestrator.startIndexing() - expect(orchestrator.isNaturallySettledPartial).toBe(true) - await orchestrator.stopIndexing() - const stoppedMessage = state.getCurrentStatus().message - failures.clear("bad.ts") - await drain() - expect(orchestrator.isNaturallySettledPartial).toBe(false) - expect(state.getCurrentStatus().message).toBe(stoppedMessage) - }) - - it("abandons and persists uncertain paths at the stop deadline", async () => { - await orchestrator.startIndexing() - const held = deferred() - const signal = watcher.initialize.mock.calls[0][0]! - const pending = gate.run(signal, () => held.promise, ["uncertain.ts"]).catch(() => {}) - await drain() - const stop = orchestrator.stopIndexing() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stop - expect(cache.invalidateFile).toHaveBeenCalledWith("uncertain.ts") - expect(cache.flush).toHaveBeenCalled() - held.resolve() - await pending - }) - - it.each(["success", "delete", "cache"] as const)( - "settles clear with the caller-visible %s result", - async (result) => { - await orchestrator.startIndexing() - const failure = new Error(`${result} failed`) - if (result === "delete") store.deleteCollection.mockRejectedValue(failure) - if (result === "cache") cache.clearCacheFile.mockRejectedValue(failure) - const clearing = orchestrator.clearIndexData() - if (result === "success") { - await expect(clearing).resolves.toBeUndefined() - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Standby", - message: "Index data cleared successfully.", - }) - } else { - await expect(clearing).rejects.toBe(failure) - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Error", - message: `Failed to clear index data: ${String(failure)}`, - }) - } - expect(store.deleteCollection).toHaveBeenCalledOnce() - expect(cache.clearCacheFile).toHaveBeenCalledTimes(result === "delete" ? 0 : 1) - }, - ) - - it.each(["resolve", "reject"] as const)("quietly unwinds a stale clear that later %s", async (ending) => { - await orchestrator.startIndexing() - const held = deferred() - store.deleteCollection.mockReturnValue(held.promise) - const clearing = orchestrator.clearIndexData() - await drain() - current = false - vi.mocked(state.setSystemState).mockClear() - if (ending === "resolve") held.resolve() - else held.reject(new Error("old delete failed")) - await expect(clearing).resolves.toBeUndefined() - expect(cache.clearCacheFile).not.toHaveBeenCalled() - expect(state.setSystemState).not.toHaveBeenCalled() - }) - - it("keeps stop independent of a hung clear and fences its stale continuation", async () => { - await orchestrator.startIndexing() - const held = deferred() - store.deleteCollection.mockReturnValue(held.promise) - const clearing = orchestrator.clearIndexData() - await drain() - const stop = orchestrator.stopWatcher() - expect(orchestrator.stopIndexing()).toBe(stop) - await stop - current = false - vi.mocked(state.setSystemState).mockClear() - held.resolve() - await clearing - expect(cache.clearCacheFile).not.toHaveBeenCalled() - expect(state.setSystemState).not.toHaveBeenCalled() - }) - it.each(["stop", "supersede"])("fences new-collection cleanup after %s", async (ending) => { - store.initialize.mockResolvedValue(true) - const held = deferred() - scanner.scanDirectory.mockImplementation(async () => { - await held.promise - throw new Error("late scan failure") + it("CI-OR-02: no status write after the abort except the final Standby", async () => { + const context = setup() + let report: ((count: number) => void) | undefined + context.scanner.scanDirectory.mockImplementation(async (dir, onError, onIndexed, onParsed, signal) => { + report = onParsed + return waitForAbort(dir, onError, onIndexed, onParsed, signal) + }) + const start = context.orchestrator.startIndexing() + await tick() + + const stop = context.orchestrator.stopIndexing() + const writesAtAbort = context.stateManager.setSystemState.mock.calls.length + report?.(5) + context.failures.renew("late.ts", "h") + context.failures.record("late.ts", "h", "embed", "late", 1, "exhausted") + await Promise.all([stop, start]) + + const lateWrites = context.stateManager.setSystemState.mock.calls.slice(writesAtAbort).map(([state]) => state) + expect(lateWrites).toEqual(["Standby"]) + expect(context.stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() }) - const scan = orchestrator.startIndexing() - await orchestrator.startupResult - if (ending === "stop") { - const stop = orchestrator.stopWatcher() - held.resolve() - await stop - } else { - current = false - held.resolve() - } - await scan - expect(store.clearCollection).not.toHaveBeenCalled() - expect(cache.clearCacheFile).toHaveBeenCalledOnce() - }) - it("contains a fatal parser event during watcher initialization", async () => { - watcher.initialize.mockImplementation(async () => { - watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parse-timeout", "bad.ts")) + it("CI-OR-03: stop then start on the same orchestrator runs a new session and writes status again", async () => { + const context = setup() + context.scanner.scanDirectory.mockImplementationOnce(waitForAbort) + const firstStart = context.orchestrator.startIndexing() + await tick() + await context.orchestrator.stopIndexing() + await firstStart + expect(context.orchestrator.state).toBe("Standby") + + await context.orchestrator.startIndexing() + expect(context.scanner.scanDirectory).toHaveBeenCalledTimes(2) + expect(context.orchestrator.state).toBe("Indexed") }) - await orchestrator.startIndexing() - await orchestrator.stopWatcher() - expect(fatal).toHaveBeenCalledOnce() - expect(state.state).toBe("Error") - expect(store.markIndexingComplete).not.toHaveBeenCalled() - }) - it("bounds stop when cache persistence hangs", async () => { - await orchestrator.startIndexing() - const held = deferred() - cache.flush.mockReturnValue(held.promise) - const stop = orchestrator.stopWatcher() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stop - expect(state.state).toBe("Standby") - held.resolve() - await drain() - }) - - it.each(["success", "failure", "stop", "supersede"])( - "acknowledges deferred validation before initialization: %s", - async (ending) => { - const held = deferred() - orchestrator = createOrchestrator(async () => held.promise) - const scan = orchestrator.startIndexing() - await orchestrator.startupResult - expect(store.initialize).not.toHaveBeenCalled() - expect(store.markIndexingIncomplete).not.toHaveBeenCalled() - if (ending === "stop") { - const stopping = orchestrator.stopIndexing() - await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) - await stopping - } - if (ending === "supersede") current = false - if (ending === "failure") held.reject(new Error("bad key")) - else held.resolve() - await scan - if (ending === "success") { - expect(store.initialize).toHaveBeenCalledOnce() - expect(store.markIndexingComplete).toHaveBeenCalledOnce() - expect(state.state).toBe("Indexed") - } else { - expect(store.initialize).not.toHaveBeenCalled() - expect(scanner.scanDirectory).not.toHaveBeenCalled() - expect(store.clearCollection).not.toHaveBeenCalled() - expect(cache.clearCacheFile).not.toHaveBeenCalled() - expect(fatal).not.toHaveBeenCalled() - if (ending === "failure") expect(state.state).toBe("Error") - } - }, - ) - - it("keeps recovery and provider callbacks from replacing Stopping", async () => { - await orchestrator.startIndexing() - state.setSystemState("Stopping", "original") - vi.mocked(state.setSystemState).mockClear() - failures.renew("bad.ts", "revision") - failures.record("bad.ts", "revision", "read", "failed", 1, "exhausted") - const pacer = pacers.get("embedding:test") - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }), - pacer.markIssued(), - ) - const controller = new AbortController() - const waiting = pacer.run("waiting.ts", controller.signal, async () => {}).catch(() => {}) - failures.clear("bad.ts", "revision") - controller.abort() - await waiting - await drain() - expect(state.setSystemState).not.toHaveBeenCalled() - expect(state.getCurrentStatus()).toMatchObject({ systemStatus: "Stopping", message: "original" }) - }) - - it("publishes provider waiting and returns to Indexed when queued work resumes", async () => { - await orchestrator.startIndexing() - const pacer = pacers.get("embedding:test") - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 1000 }), - pacer.markIssued(), - ) - expect(state.state).toBe("Indexed") - const request = vi.fn(async () => {}) - const waiting = pacer.run("waiting.ts", undefined, request) - await drain() - expect(state.getCurrentStatus()).toMatchObject({ - systemStatus: "Indexing", - message: "embeddings:orchestrator.providerWaitingUntil", + it("a start while stopping waits for the stop, then starts", async () => { + const context = setup() + context.scanner.scanDirectory.mockImplementationOnce(waitForAbort) + const firstStart = context.orchestrator.startIndexing() + await tick() + const stop = context.orchestrator.stopIndexing() + const secondStart = context.orchestrator.startIndexing() + await Promise.all([stop, firstStart, secondStart]) + expect(context.scanner.scanDirectory).toHaveBeenCalledTimes(2) + expect(context.orchestrator.state).toBe("Indexed") }) - expect(failures.list()).toEqual([]) - expect(request).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1000) - await waiting - expect(state.state).toBe("Indexed") - expect(store.markIndexingComplete).toHaveBeenCalledOnce() - }) - it("disposes the provider subscription on stop", async () => { - const subscribe = vi.spyOn(pacers, "onDidChange") - await orchestrator.startIndexing() - const subscription = subscribe.mock.results[0].value - const dispose = vi.spyOn(subscription, "dispose") - await orchestrator.stopIndexing() - expect(dispose).toHaveBeenCalledOnce() - }) + it("a repeated start while scanning returns the running scan", async () => { + const context = setup() + context.scanner.scanDirectory.mockImplementationOnce(waitForAbort) + const first = context.orchestrator.startIndexing() + await tick() + const second = context.orchestrator.startIndexing() + await tick() + expect(context.scanner.scanDirectory).toHaveBeenCalledOnce() + await context.orchestrator.stopIndexing() + await Promise.all([first, second]) + }) - it("leaves a naturally settled partial scan idle and incomplete", async () => { - scanner.scanDirectory.mockResolvedValue({ ...scanResult, incompleteFiles: ["bad.ts"] }) - await orchestrator.startIndexing() - expect(orchestrator.isScanning).toBe(false) - expect(orchestrator.isNaturallySettledPartial).toBe(true) - expect(state.state).toBe("Standby") - expect(store.markIndexingIncomplete).toHaveBeenCalledOnce() - expect(store.markIndexingComplete).not.toHaveBeenCalled() + it("preserves Error across a stop", async () => { + const context = setup() + context.vectorStore.initialize.mockRejectedValue(new Error("down")) + await context.orchestrator.startIndexing() + await context.orchestrator.stopIndexing() + expect(context.orchestrator.state).toBe("Error") + }) }) }) diff --git a/src/services/code-index/__tests__/service-factory.spec.ts b/src/services/code-index/__tests__/service-factory.spec.ts index 3b22f267a5..bc1d619af2 100644 --- a/src/services/code-index/__tests__/service-factory.spec.ts +++ b/src/services/code-index/__tests__/service-factory.spec.ts @@ -13,36 +13,8 @@ import { BedrockEmbedder } from "../embedders/bedrock" import { OpenRouterEmbedder } from "../embedders/openrouter" import type { CodeIndexConfig } from "../interfaces/config" -import { CodeIndexConfigManager } from "../config-manager" -import { CacheManager } from "../cache-manager" -import { ParseWorkerClient } from "../parse-worker-client" -import { IndexWriteGate } from "../index-write-gate" -import { GenerationAllocator } from "../generation-allocator" -import { FileFailureRegistry } from "../file-failure-registry" -import { ProviderPacerRegistry } from "../provider-pacer" -import { DirectoryScanner } from "../processors/scanner" -import { FileWatcher } from "../processors/file-watcher" -import { makeExtensionContext } from "../../../test-utils/vscode" -import ignore from "ignore" - import { clearAllMocks } from "../../../test-utils/reset" -vitest.mock("../config-manager") -vitest.mock("../cache-manager") -vitest.mock("../parse-worker-client", () => ({ - ParseWorkerClient: vitest.fn( - class { - parseFile = vitest.fn().mockResolvedValue([]) - stop = vitest.fn() - stopped = Promise.resolve() - logStopTimeout = vitest.fn() - dispose = vitest.fn() - }, - ), -})) -vitest.mock("../processors/scanner") -vitest.mock("../processors/file-watcher") - // Mock the embedders and vector store vitest.mock("../embedders/openai") vitest.mock("../embedders/ollama") @@ -85,166 +57,6 @@ import { providerIdentifiers } from "@roo-code/types/provider-identifiers" const mockGetDefaultModelId = getDefaultModelId as MockedFunction const mockGetModelDimension = getModelDimension as MockedFunction -describe("workspace service integration", () => { - beforeEach(() => clearAllMocks()) - afterEach(() => vitest.restoreAllMocks()) - - function setup(isConfigured = true) { - const context = makeExtensionContext() - const cache = new CacheManager(context, "/workspace") - const watcherCache = new CacheManager(context, "/workspace") - const configManager = Object.create(CodeIndexConfigManager.prototype) as CodeIndexConfigManager - const config: CodeIndexConfig = { isConfigured, embedderProvider: "openai-compatible" } - Object.defineProperty(configManager, "isFeatureConfigured", { value: isConfigured }) - vitest.mocked(configManager.getConfig).mockReturnValue(config) - const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) - const vectorStore = new QdrantVectorStore("/workspace", "http://localhost:6333", 512) - const createEmbedder = vitest.spyOn(EmbedderFactory.prototype, "create").mockReturnValue(embedder) - const createVectorStore = vitest.spyOn(VectorStoreFactory.prototype, "create").mockReturnValue(vectorStore) - const gate = new IndexWriteGate() - const generations = new GenerationAllocator(cache) - const failures = new FileFailureRegistry() - const providerPacers = new ProviderPacerRegistry() - const parser = new ParseWorkerClient() - const isCurrentAttempt = vitest.fn(() => true) - const ignored = ignore() - const createFactory = () => - new CodeIndexServiceFactory( - configManager, - "/workspace", - cache, - gate, - generations, - failures, - parser, - isCurrentAttempt, - providerPacers, - ) - return { - context, - cache, - watcherCache, - configManager, - config, - embedder, - vectorStore, - createEmbedder, - createVectorStore, - gate, - generations, - failures, - parser, - isCurrentAttempt, - providerPacers, - ignored, - createFactory, - } - } - - it("injects the same workspace dependencies into both processors across service and factory recreation", () => { - const options = setup() - options.failures.renew("file.ts", "revision") - const budget = options.failures.getRetryBudget("file.ts", "revision", "embed") - budget.retriesUsed = 2 - options.failures.record("file.ts", "revision", "embed", "exhausted", 3, "exhausted", "server") - const reset = vitest.spyOn(options.failures, "resetAllBudgets") - const renew = vitest.spyOn(options.failures, "renew") - const clear = vitest.spyOn(options.failures, "clear") - const initialize = vitest.spyOn(options.generations, "initialize") - const factory = options.createFactory() - - for (const currentFactory of [factory, factory, options.createFactory()]) { - const services = currentFactory.createServices(options.context, options.watcherCache, options.ignored) - const scannerArguments = vitest.mocked(DirectoryScanner).mock.lastCall! - const watcherArguments = vitest.mocked(FileWatcher).mock.lastCall! - expect(services).toEqual({ - embedder: options.embedder, - vectorStore: options.vectorStore, - parser: options.parser, - scanner: vitest.mocked(DirectoryScanner).mock.instances.at(-1), - fileWatcher: vitest.mocked(FileWatcher).mock.instances.at(-1), - }) - expect(scannerArguments[2]).toBe(options.parser) - expect(watcherArguments[8]).toBe(options.parser) - expect(scannerArguments[3]).toBe(options.cache) - expect(watcherArguments[2]).toBe(options.watcherCache) - expect(scannerArguments[6]).toBe(options.gate) - expect(watcherArguments[9]).toBe(options.gate) - expect(scannerArguments[7]).toBe(options.generations) - expect(watcherArguments[10]).toBe(options.generations) - expect(scannerArguments[8]).toBe(options.failures) - expect(watcherArguments[11]).toBe(options.failures) - expect(scannerArguments[9]).toBe("/workspace") - expect(scannerArguments[10]).toBe(options.isCurrentAttempt) - expect(watcherArguments[12]).toBe(options.isCurrentAttempt) - expect(scannerArguments[11]).toBe(options.providerPacers) - expect(watcherArguments[13]).toBe(options.providerPacers) - for (const current of [true, false]) { - options.isCurrentAttempt.mockReturnValue(current) - expect(scannerArguments[10]?.()).toBe(current) - expect(watcherArguments[12]?.()).toBe(current) - } - } - - expect(options.configManager.getConfig).toHaveBeenCalledTimes(3) - expect(options.createEmbedder).toHaveBeenCalledWith(options.config) - expect(options.createVectorStore).toHaveBeenCalledWith(options.config, "/workspace") - expect(options.failures.getRetryBudget("file.ts", "revision", "embed")).toBe(budget) - expect(budget.retriesUsed).toBe(2) - expect(options.failures.isExhausted("file.ts", "revision", "embed")).toBe(true) - expect(reset).not.toHaveBeenCalled() - expect(renew).not.toHaveBeenCalled() - expect(clear).not.toHaveBeenCalled() - expect(initialize).not.toHaveBeenCalled() - expect(ParseWorkerClient).toHaveBeenCalledTimes(1) - expect(options.parser.stop).not.toHaveBeenCalled() - }) - - it("preserves the first three arguments and shares defaults within one factory, not across workspaces", async () => { - const options = setup() - const firstFactory = new CodeIndexServiceFactory(options.configManager, "/workspace", options.cache) - const otherFactory = new CodeIndexServiceFactory(options.configManager, "/other", options.cache) - const first = firstFactory.createServices(options.context, options.cache, options.ignored) - const second = firstFactory.createServices(options.context, options.cache, options.ignored) - const other = otherFactory.createServices(options.context, options.cache, options.ignored) - expect(first.parser).toBeInstanceOf(ParseWorkerClient) - expect(first.parser).toBe(second.parser) - expect(first.parser).not.toBe(other.parser) - const scannerCalls = vitest.mocked(DirectoryScanner).mock.calls - const watcherCalls = vitest.mocked(FileWatcher).mock.calls - for (const [scannerPosition, watcherPosition] of [ - [6, 9], - [7, 10], - [8, 11], - [11, 13], - ] as const) { - expect(scannerCalls[0][scannerPosition]).toBe(scannerCalls[1][scannerPosition]) - expect(scannerCalls[0][scannerPosition]).toBe(watcherCalls[0][watcherPosition]) - expect(scannerCalls[0][scannerPosition]).not.toBe(scannerCalls[2][scannerPosition]) - } - first.parser.stop() - expect(first.parser.stop).toHaveBeenCalledOnce() - await first.parser.stopped - first.parser.logStopTimeout() - first.parser.dispose() - expect(first.parser.logStopTimeout).toHaveBeenCalledOnce() - expect(first.parser.dispose).toHaveBeenCalledOnce() - }) - - it("rejects unconfigured indexing before constructing processors or reading configuration", () => { - const options = setup(false) - const factory = options.createFactory() - expect(() => factory.createServices(options.context, options.cache, options.ignored)).toThrow( - "serviceFactory.codeIndexingNotConfigured", - ) - expect(options.configManager.getConfig).not.toHaveBeenCalled() - expect(options.createEmbedder).not.toHaveBeenCalled() - expect(options.createVectorStore).not.toHaveBeenCalled() - expect(DirectoryScanner).not.toHaveBeenCalled() - expect(FileWatcher).not.toHaveBeenCalled() - }) -}) - describe("CodeIndexServiceFactory", () => { let factory: CodeIndexServiceFactory let embedderFactory: EmbedderFactory diff --git a/src/services/code-index/cache-manager.ts b/src/services/code-index/cache-manager.ts index 0a3d7d10c2..e3fd706525 100644 --- a/src/services/code-index/cache-manager.ts +++ b/src/services/code-index/cache-manager.ts @@ -1,153 +1,157 @@ import * as vscode from "vscode" import { createHash } from "crypto" -import type { ICacheManager } from "./interfaces/cache" +import { ICacheManager } from "./interfaces/cache" import debounce from "lodash.debounce" import { safeWriteJson } from "../../utils/safeWriteJson" -import { codeIndexLog } from "./code-index-log" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" +import { t } from "../../i18n" +import { isSourceMissing } from "./shared/indexing-errors" -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === "object" && !Array.isArray(value) +/** The cache exists but could not be read; treating it as empty would erase the index on the next scan. */ +export class CacheReadError extends Error { + constructor(cause: unknown) { + super(t("embeddings:orchestrator.cacheReadFailed"), { cause }) + this.name = "CacheReadError" + } +} + +function isHashMap(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + Object.values(value).every((hash) => typeof hash === "string") + ) } +/** + * Manages the cache for code indexing + */ export class CacheManager implements ICacheManager { - private readonly cachePath: vscode.Uri + private cachePath: vscode.Uri private fileHashes: Record = {} - private openFiles: Record = {} - private watermark?: number - private saveTail: Promise = Promise.resolve() - private readonly debouncedSave - public loadSucceeded = false - public loadedAt = 0 + private _debouncedSaveCache: () => void + /** + * Creates a new cache manager + * @param context VS Code extension context + * @param workspacePath Path to the workspace + */ constructor(context: vscode.ExtensionContext, workspacePath: string) { this.cachePath = vscode.Uri.joinPath( context.globalStorageUri, `roo-index-cache-${createHash("sha256").update(workspacePath).digest("hex")}.json`, ) - this.debouncedSave = debounce(() => { - void this.saveNow().catch((error: unknown) => codeIndexLog.error("Failed to save cache:", error)) + this._debouncedSaveCache = debounce(async () => { + await this._performSave() }, 1500) } + /** + * Initializes the cache manager by loading the cache file + */ async initialize(): Promise { - this.fileHashes = {} - this.openFiles = {} - this.watermark = undefined - this.loadSucceeded = false + let cacheData: Uint8Array try { - const buffer = await vscode.workspace.fs.readFile(this.cachePath) - const data: unknown = JSON.parse(Buffer.from(buffer).toString()) - if (!isRecord(data)) throw new Error("Invalid indexing cache") - if (data.version === 2) { - if (!isRecord(data.fileHashes) || !isRecord(data.openFiles)) { - throw new Error("Invalid indexing cache records") - } - for (const [filePath, hash] of Object.entries(data.fileHashes)) { - if (typeof hash !== "string") throw new Error("Invalid indexing hash") - this.fileHashes[filePath] = hash - } - for (const [filePath, open] of Object.entries(data.openFiles)) { - if (open !== true) throw new Error("Invalid open-file record") - this.openFiles[filePath] = true - delete this.fileHashes[filePath] - } - if ( - typeof data.generationWatermark === "number" && - Number.isSafeInteger(data.generationWatermark) && - data.generationWatermark >= 0 - ) { - this.watermark = data.generationWatermark - } - } else { - for (const [filePath, hash] of Object.entries(data)) { - if (typeof hash !== "string") throw new Error("Invalid legacy indexing cache") - this.fileHashes[filePath] = hash - } + cacheData = await vscode.workspace.fs.readFile(this.cachePath) + } catch (error) { + if (isSourceMissing(error)) { + this.fileHashes = {} + return } - this.loadSucceeded = true - } catch { + this.captureError(error, "initialize") + throw new CacheReadError(error) + } + try { + const parsed: unknown = JSON.parse(Buffer.from(cacheData).toString()) + this.fileHashes = isHashMap(parsed) ? parsed : {} + } catch (error) { this.fileHashes = {} - this.openFiles = {} - this.watermark = undefined + this.captureError(error, "initialize") } - this.loadedAt = Date.now() - } - - get generationWatermark(): number | undefined { - return this.watermark } - async reserveGenerations(watermark: number): Promise { - this.watermark = watermark - await this.saveNow() + private captureError(error: unknown, location: string): void { + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location, + }) } - async saveNow(): Promise { - this.debouncedSave.cancel() - const data = { - version: 2, - fileHashes: { ...this.fileHashes }, - openFiles: { ...this.openFiles }, - generationWatermark: this.watermark, + /** + * Saves the cache to disk + */ + private async _performSave(): Promise { + try { + await safeWriteJson(this.cachePath.fsPath, this.fileHashes) + } catch (error) { + console.error("Failed to save cache:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "_performSave", + }) } - // An older debounced snapshot must never overwrite durable invalidation. - const pending = this.saveTail.catch(() => {}).then(() => safeWriteJson(this.cachePath.fsPath, data)) - this.saveTail = pending - await pending } + /** + * Clears the cache file by writing an empty object to it + */ async clearCacheFile(): Promise { - this.fileHashes = {} - this.openFiles = {} - await this.saveNow() - } - - async invalidateFile(filePath: string): Promise { - delete this.fileHashes[filePath] - this.openFiles[filePath] = true - await this.saveNow() - } - - completeFile(filePath: string, hash: string): void { - this.fileHashes[filePath] = hash - delete this.openFiles[filePath] - this.debouncedSave() + try { + await safeWriteJson(this.cachePath.fsPath, {}) + this.fileHashes = {} + } catch (error) { + console.error("Failed to clear cache file:", error, this.cachePath) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "clearCacheFile", + }) + } } - completeRemoval(filePath: string): void { - delete this.fileHashes[filePath] - delete this.openFiles[filePath] - this.debouncedSave() + /** + * Gets the hash for a file path + * @param filePath Path to the file + * @returns The hash for the file or undefined if not found + */ + getHash(filePath: string): string | undefined { + return this.fileHashes[filePath] } + /** + * Updates the hash for a file path + * @param filePath Path to the file + * @param hash New hash value + */ updateHash(filePath: string, hash: string): void { - this.completeFile(filePath, hash) + this.fileHashes[filePath] = hash + this._debouncedSaveCache() } + /** + * Deletes the hash for a file path + * @param filePath Path to the file + */ deleteHash(filePath: string): void { - this.completeRemoval(filePath) - } - - getOpenFiles(): Record { - return { ...this.openFiles } - } - - getTrackedPaths(): string[] { - return [...new Set([...Object.keys(this.fileHashes), ...Object.keys(this.openFiles)])] - } - - isTracked(filePath: string): boolean { - return Object.hasOwn(this.fileHashes, filePath) || Object.hasOwn(this.openFiles, filePath) - } - - getHash(filePath: string): string | undefined { - return Object.hasOwn(this.fileHashes, filePath) ? this.fileHashes[filePath] : undefined + delete this.fileHashes[filePath] + this._debouncedSaveCache() } + /** + * Flushes any pending debounced cache writes to disk immediately. + */ async flush(): Promise { - await this.saveNow() + await this._performSave() } + /** + * Gets a copy of all file hashes + * @returns A copy of the file hashes record + */ getAllHashes(): Record { return { ...this.fileHashes } } diff --git a/src/services/code-index/code-index-scan-executor.ts b/src/services/code-index/code-index-scan-executor.ts index 616be1e1a7..e0f38b4fff 100644 --- a/src/services/code-index/code-index-scan-executor.ts +++ b/src/services/code-index/code-index-scan-executor.ts @@ -1,75 +1,64 @@ -import type { IDirectoryScanner } from "./interfaces" +import type { IDirectoryScanner, IVectorStore } from "./interfaces" import type { CodeIndexStateManager } from "./state-manager" -import { ParserFatalError } from "./parser-errors" -import { EmbeddingError, IndexingRetryError, getIndexingErrorStatusCode } from "./shared/indexing-errors" -import { codeIndexLog } from "./code-index-log" - -function isEmbeddingConfigurationError(error: Error): boolean { - let current: unknown = error - let embedding = false - const visited = new Set() - while (current instanceof Error && !visited.has(current)) { - visited.add(current) - if ( - current instanceof EmbeddingError || - (current instanceof IndexingRetryError && current.activity === "embed") - ) { - embedding = true - } - current = current.cause - } - const status = getIndexingErrorStatusCode(error) - return embedding && status !== undefined && [400, 401, 403, 404, 422].includes(status) -} +/** Executes scans; cancellation cleanup, watcher startup and completion remain with the orchestrator. */ export class CodeIndexScanExecutor { + /** True when the last scan finished with file-level failures; those files stay in the failure registry. */ public partial = false constructor( private readonly workspacePath: string, private readonly scanner: IDirectoryScanner, + private readonly vectorStore: Pick, private readonly stateManager: Pick, - private readonly isCurrentAttempt: () => boolean = () => true, ) {} - runIncrementalScan(signal: AbortSignal): Promise { + /** Returns false when the scanner returns after cancellation, preserving the owner's normal-return path. */ + async runIncrementalScan(signal: AbortSignal): Promise { + console.log( + "[CodeIndexOrchestrator] Collection already has indexed data. Running incremental scan for new/changed files...", + ) + this.stateManager.setSystemState("Indexing", "Checking for new or modified files...") return this.scanWorkspace(signal, "incremental") } - runFullScan(signal: AbortSignal): Promise { + /** Returns false on returned cancellation; scanner rejections propagate unchanged. */ + async runFullScan(signal: AbortSignal): Promise { + this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") return this.scanWorkspace(signal, "full") } + // File-level failures complete the scan with a partial index instead of failing the whole attempt. private async scanWorkspace(signal: AbortSignal, kind: "full" | "incremental"): Promise { - const isLive = () => !signal.aborted && this.isCurrentAttempt() - if (!isLive()) return false this.partial = false - this.stateManager.setSystemState( - "Indexing", - kind === "full" ? "Services ready. Starting workspace scan..." : "Checking for new or modified files...", - "scan", - ) + await this.vectorStore.markIndexingIncomplete() let indexed = 0 let found = 0 - const batchErrors: Error[] = [] + let failureCount = 0 const result = await this.scanner.scanDirectory( this.workspacePath, - (error) => { - if (isLive()) batchErrors.push(error) + (fileError: Error) => { + console.error( + `[CodeIndexOrchestrator] Error during ${kind === "full" ? "initial" : "incremental"} scan: ${fileError.message}`, + fileError, + ) + failureCount++ }, - (count) => { - if (!isLive()) return + (count: number) => { + if (signal.aborted) return indexed += count - this.stateManager.reportBlockIndexingProgress(indexed, found, "scan") + this.stateManager.reportBlockIndexingProgress(indexed, found) }, - (count) => { - if (!isLive()) return + (count: number) => { + if (signal.aborted) return found += count - this.stateManager.reportBlockIndexingProgress(indexed, found, "scan") + this.stateManager.reportBlockIndexingProgress(indexed, found) }, signal, ) - if (!isLive()) return false + + // Cancellation precedes missing-result checks, just as in the orchestrator. + if (signal.aborted) return false if (!result) { throw new Error( kind === "full" @@ -77,24 +66,8 @@ export class CodeIndexScanExecutor { : "Incremental scan failed, is scanner initialized?", ) } - const fatal = batchErrors.find((error) => error instanceof ParserFatalError) - if (fatal) throw fatal - if ( - kind === "full" && - indexed === 0 && - found > 0 && - batchErrors.length > 0 && - batchErrors.every(isEmbeddingConfigurationError) - ) { - throw batchErrors[0] - } - this.partial = batchErrors.length > 0 || (result.incompleteFiles?.length ?? 0) > 0 || indexed < found - codeIndexLog.info("Scan summary", { - blocksIndexed: indexed, - blocksFound: found, - failureCount: batchErrors.length, - incompleteFiles: result.incompleteFiles, - }) + this.partial = failureCount > 0 || indexed < found + console.log(`[CodeIndexOrchestrator] ${kind} scan completed: ${indexed} of ${found} blocks indexed`) return true } } diff --git a/src/services/code-index/interfaces/directory-scanner-factory.ts b/src/services/code-index/interfaces/directory-scanner-factory.ts index 94abdd235a..707398b6be 100644 --- a/src/services/code-index/interfaces/directory-scanner-factory.ts +++ b/src/services/code-index/interfaces/directory-scanner-factory.ts @@ -1,10 +1,8 @@ import type { Ignore } from "ignore" import type { CacheManager } from "../cache-manager" -import type { ICodeParser, IDirectoryScanner, IEmbedder, IVectorStore } from "./index" import type { FileFailureRegistry } from "../file-failure-registry" -import type { IndexWriteGate } from "../index-write-gate" -import type { GenerationAllocator } from "../generation-allocator" import type { ProviderPacerRegistry } from "../provider-pacer" +import type { ICodeParser, IDirectoryScanner, IEmbedder, IVectorStore } from "./index" export interface DirectoryScannerFactoryOptions { embedder: IEmbedder @@ -12,12 +10,8 @@ export interface DirectoryScannerFactoryOptions { parser: ICodeParser cacheManager: CacheManager ignoreInstance: Ignore - failures?: FileFailureRegistry - writeGate?: IndexWriteGate - generations?: GenerationAllocator - workspacePath?: string - isCurrentAttempt?: () => boolean - providerPacers?: ProviderPacerRegistry + failureRegistry: FileFailureRegistry + pacers: ProviderPacerRegistry } export interface IDirectoryScannerFactory { diff --git a/src/services/code-index/interfaces/file-watcher-factory.ts b/src/services/code-index/interfaces/file-watcher-factory.ts index 97861fa2c0..05aa844602 100644 --- a/src/services/code-index/interfaces/file-watcher-factory.ts +++ b/src/services/code-index/interfaces/file-watcher-factory.ts @@ -2,11 +2,9 @@ import type * as vscode from "vscode" import type { Ignore } from "ignore" import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import type { CacheManager } from "../cache-manager" -import type { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./index" import type { FileFailureRegistry } from "../file-failure-registry" -import type { IndexWriteGate } from "../index-write-gate" -import type { GenerationAllocator } from "../generation-allocator" import type { ProviderPacerRegistry } from "../provider-pacer" +import type { IEmbedder, IFileWatcher, IVectorStore } from "./index" export interface FileWatcherFactoryOptions { workspacePath: string @@ -16,12 +14,8 @@ export interface FileWatcherFactoryOptions { cacheManager: CacheManager ignoreInstance: Ignore rooIgnoreController?: RooIgnoreController - parser?: ICodeParser - failures?: FileFailureRegistry - writeGate?: IndexWriteGate - generations?: GenerationAllocator - isCurrentAttempt?: () => boolean - providerPacers?: ProviderPacerRegistry + failureRegistry: FileFailureRegistry + pacers: ProviderPacerRegistry } export interface IFileWatcherFactory { diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 9cf3e21ff0..ac6bdaaf8b 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -1,99 +1,78 @@ import * as vscode from "vscode" import { ContextProxy } from "../../core/config/ContextProxy" -import type { IndexingState } from "./interfaces/manager" +import { IndexingState } from "./interfaces/manager" import { CodeIndexConfigManager } from "./config-manager" -import { CodeIndexStateManager } from "./state-manager" +import type { CodeIndexStateManager } from "./state-manager" import { CodeIndexServiceFactory } from "./service-factory" import { CodeIndexSearchService, type CodeIndexSearchResponse } from "./search-service" import { CodeIndexOrchestrator } from "./orchestrator" import { CacheManager } from "./cache-manager" import { SembleProvider } from "./semble" +import { FileFailureRegistry } from "./file-failure-registry" +import { ProviderPacerRegistry } from "./provider-pacer" +import { formatProviderWaitMessage } from "./shared/provider-wait-message" import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import fs from "fs/promises" import ignore from "ignore" import path from "path" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" import { t } from "../../i18n" -import { IndexWriteGate } from "./index-write-gate" -import { GenerationAllocator } from "./generation-allocator" -import { FileFailureRegistry, type FailedFile } from "./file-failure-registry" -import { ParseWorkerClient } from "./parse-worker-client" -import { codeIndexLog } from "./code-index-log" -import { initializeCodeIndexHostLogging } from "./code-index-log-host" -import { ProviderPacerRegistry } from "./provider-pacer" -import { classifyIndexingError } from "./shared/indexing-errors" -import { formatProviderWaitMessage } from "./shared/provider-wait-message" -type FailedFileStatus = Pick -type FailedFileSummary = FailedFileStatus & { count: number } export class CodeIndexManager { - private static readonly diagnosticOwners = new Set() - private _configManager?: CodeIndexConfigManager - private _serviceFactory?: CodeIndexServiceFactory - private _orchestrator?: CodeIndexOrchestrator - private _searchService?: CodeIndexSearchService - private _cacheManager?: CacheManager - private cacheInitialization?: Promise - private _sembleProvider?: SembleProvider - private sembleState?: CodeIndexStateManager - private sembleSubscription?: vscode.Disposable - private parser?: ParseWorkerClient - private _isRecoveringFromError = false - private parserFailed = false - private parserFailureMessage = "" - private servicesStopped = false - private disabledSinceFailure = false - private readonly writeGate = new IndexWriteGate() - private generationAllocator?: GenerationAllocator - private currentAttempt = 0 - private readonly idleStop = Promise.resolve() - private clearing: Promise = Promise.resolve() + // Specialized class instances + private _configManager: CodeIndexConfigManager | undefined + private readonly _stateManager: CodeIndexStateManager + private _serviceFactory: CodeIndexServiceFactory | undefined + private _orchestrator: CodeIndexOrchestrator | undefined + private _searchService: CodeIndexSearchService | undefined + private _cacheManager: CacheManager | undefined + private _sembleProvider: SembleProvider | undefined + + // One registry and pacer set per workspace: recreating services must not reset retry budgets or provider waits. + private readonly failureRegistry = new FileFailureRegistry() + private readonly pacers = new ProviderPacerRegistry() private startInFlight?: Promise - private servicesRestartPending = false - private startupFailureMessage?: string - private startupEpoch = 0 - public readonly fileFailureRegistry = new FileFailureRegistry() - public readonly providerPacerRegistry = new ProviderPacerRegistry() - private readonly progressEmitter = new vscode.EventEmitter>() - private readonly statusSubscriptions: vscode.Disposable[] = [] - - /** @internal — construct only via CodeIndexManagerRegistry. */ + + // Flag to prevent race conditions during error recovery + private _isRecoveringFromError = false + + private readonly workspacePath: string + private readonly _folderUri: vscode.Uri + private readonly context: vscode.ExtensionContext + + /** @internal — construct only via {@link CodeIndexManagerRegistry} */ public constructor( - private readonly workspacePath: string, - private readonly _folderUri: vscode.Uri, - private readonly context: vscode.ExtensionContext, - private readonly _stateManager: CodeIndexStateManager, + workspacePath: string, + folderUri: vscode.Uri, + context: vscode.ExtensionContext, + stateManager: CodeIndexStateManager, ) { - initializeCodeIndexHostLogging() - CodeIndexManager.diagnosticOwners.add(this) - this.configureDiagnostics() - this.statusSubscriptions.push( - this._stateManager.onProgressUpdate(() => this.progressEmitter.fire(this.getCurrentStatus())), - this.fileFailureRegistry.onDidChange(() => this.progressEmitter.fire(this.getCurrentStatus())), - this.providerPacerRegistry.onDidChange(() => this.progressEmitter.fire(this.getCurrentStatus())), - ) + this.workspacePath = workspacePath + this._folderUri = folderUri + this.context = context + this._stateManager = stateManager } + // --- Public API --- + + /** + * Returns the workspaceState key for per-folder indexing enablement, + * keyed by the real workspace folder URI so local/remote schemes cannot collide. + */ private _workspaceEnabledKey(): string { return "codeIndexWorkspaceEnabled:" + this._folderUri.toString(true) } public get isWorkspaceEnabled(): boolean { - return ( - this.context.workspaceState.get(this._workspaceEnabledKey(), undefined) ?? - this.autoEnableDefault - ) + const explicit = this.context.workspaceState.get(this._workspaceEnabledKey(), undefined) + if (explicit !== undefined) return explicit + return this.autoEnableDefault } public async setWorkspaceEnabled(enabled: boolean): Promise { - const wasEnabled = this.isWorkspaceEnabled await this.context.workspaceState.update(this._workspaceEnabledKey(), enabled) - if (!enabled) { - this.disabledSinceFailure = true - void this.stopWatcher() - } else if (!wasEnabled) { - this.renewAfterDisable() - } - this.progressEmitter.fire(this.getCurrentStatus()) + if (enabled) this.failureRegistry.resetAllBudgets() } public get autoEnableDefault(): boolean { @@ -105,18 +84,28 @@ export class CodeIndexManager { } public get onProgressUpdate() { - return this.progressEmitter.event + return this._stateManager.onProgressUpdate } - private assertInitialized(): void { - if (this._sembleProvider) return + private assertInitialized() { + if (this._sembleProvider) { + // When semble is active, we don't need orchestrator/searchService + return + } if (!this._configManager || !this._orchestrator || !this._searchService || !this._cacheManager) { throw new Error("CodeIndexManager not initialized. Call initialize() first.") } } public get state(): IndexingState { - return this.getCurrentStatus().systemStatus + if (!this.isFeatureEnabled) { + return "Standby" + } + if (this._sembleProvider) { + return this._sembleProvider.state + } + this.assertInitialized() + return this._orchestrator!.state } public get isFeatureEnabled(): boolean { @@ -127,6 +116,7 @@ export class CodeIndexManager { return this._configManager?.isFeatureConfigured ?? false } + /** Whether configuration loading has succeeded, independently of service or indexing readiness. */ public get isConfigurationLoaded(): boolean { return this._configManager?.isConfigurationLoaded ?? false } @@ -140,283 +130,262 @@ export class CodeIndexManager { } } - private renewAfterDisable(): void { - this.parserFailed = false - this.startupFailureMessage = undefined - this.disabledSinceFailure = false - this.fileFailureRegistry.resetAllBudgets() - } - - private async initializeCache(): Promise { - if (!this.cacheInitialization) { - const cache = (this._cacheManager ??= new CacheManager(this.context, this.workspacePath)) - this.cacheInitialization = cache - .initialize() - .then(() => { - this.writeGate.recover(Object.keys(cache.getOpenFiles()), cache.loadedAt) - this.generationAllocator = new GenerationAllocator(cache) - }) - .catch((error: unknown) => { - this.cacheInitialization = undefined - throw error - }) + /** + * Initializes the manager with configuration and dependent services. + * Must be called before using any other methods. + * @returns Object indicating if a restart is needed + */ + public async initialize(contextProxy: ContextProxy): Promise<{ requiresRestart: boolean }> { + // 1. ConfigManager Initialization and Configuration Loading + if (!this._configManager) { + this._configManager = new CodeIndexConfigManager(contextProxy) } - await this.cacheInitialization - } + // Load configuration once to get current state and restart requirements + const { requiresRestart } = await this._configManager.loadConfiguration() - public async initialize(contextProxy: ContextProxy, autoStart = true): Promise<{ requiresRestart: boolean }> { - const epoch = this.startupEpoch - const configuration = (this._configManager ??= new CodeIndexConfigManager(contextProxy)) - const { requiresRestart } = await configuration.loadConfiguration() - if (configuration !== this._configManager || epoch !== this.startupEpoch) return { requiresRestart } - this.configureDiagnostics() - if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { - this.disabledSinceFailure = true - void this.stopWatcher() + // 2. Check if feature is enabled + if (!this.isFeatureEnabled) { + if (this._orchestrator) { + await this._orchestrator.stopIndexing() + } + if (this._sembleProvider) { + this._sembleProvider.stopIndexing() + } return { requiresRestart } } - if (!this.workspacePath) { + + // 3. Check if workspace is available + const workspacePath = this.workspacePath + if (!workspacePath) { this._stateManager.setSystemState("Standby", "No workspace folder open") return { requiresRestart } } - if (this.disabledSinceFailure) this.renewAfterDisable() - if (autoStart && (this.parserFailed || this.startupFailureMessage)) return { requiresRestart } - if (!autoStart) { - await this.initializeCache() + + // 4. Check workspace-level enablement (before creating expensive services) + if (!this.isWorkspaceEnabled) { + this._stateManager.setSystemState("Standby", "Indexing not enabled for this workspace") return { requiresRestart } } - if (this.startInFlight) { - this.servicesRestartPending ||= requiresRestart - await this.startInFlight - } else if ((!this._serviceFactory && !this._sembleProvider) || requiresRestart || this.servicesStopped) { - await this.runStartup(async (startupEpoch) => { - await this.initializeCache() - if (startupEpoch === this.startupEpoch) await this._recreateServices() - }) + + // 5. CacheManager Initialization; an unreadable cache ends here so existing index data is kept. + if (!(await this.ensureCacheManager())) return { requiresRestart } + + // 6. Determine if Core Services Need Recreation + const needsServiceRecreation = (!this._serviceFactory && !this._sembleProvider) || requiresRestart + + if (needsServiceRecreation) { + await this._recreateServices() + } + + // 7. Handle Indexing Start/Restart + if (this._sembleProvider) { + // For semble, start indexing if needed + const shouldStartIndexing = requiresRestart || needsServiceRecreation + if (shouldStartIndexing) { + await this._sembleProvider.startIndexing() + } + } else { + const shouldStartOrRestartIndexing = + requiresRestart || + (needsServiceRecreation && (!this._orchestrator || this._orchestrator.state !== "Indexing")) + + if (shouldStartOrRestartIndexing) { + this.startIndexingInBackground() + } } + return { requiresRestart } } - /** Resolves after startup, not after the background scan. */ - public startIndexing(contextProxy?: ContextProxy): Promise { - if (this.startInFlight) return this.startInFlight - if (this._sembleProvider) { - return this.runStartup(async (epoch) => { - await this.clearing - if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return - this.startupFailureMessage = undefined - this.servicesStopped = false - this.subscribeSembleState() - }) + /** Returns false when the cache could not be read; status is then Error and Start Indexing retries. */ + private async ensureCacheManager(): Promise { + if (this._cacheManager) return true + const cacheManager = new CacheManager(this.context, this.workspacePath) + try { + await cacheManager.initialize() + } catch (error) { + console.error("[CodeIndexManager] Failed to read the index cache:", error) + this._stateManager.setSystemState( + "Error", + error instanceof Error ? error.message : t("embeddings:orchestrator.cacheReadFailed"), + ) + return false } - if ( - this.isFeatureEnabled && - this.isWorkspaceEnabled && - this._orchestrator?.isScanning && - !this._orchestrator.stopped - ) { - return this._orchestrator.startupResult - } - return this.runStartup(async (epoch) => { - if (contextProxy) await this.initialize(contextProxy, false) - if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return - this.parserFailed = false - this.startupFailureMessage = undefined - await this.initializeCache() - if (epoch !== this.startupEpoch) return - await this._recreateServices() - if (epoch !== this.startupEpoch) return - this.fileFailureRegistry.resetAllBudgets() - this.assertInitialized() + this._cacheManager = cacheManager + return true + } + + private startIndexingInBackground(): void { + void this.startOrchestrator().catch((error) => { + console.error("[CodeIndexManager] Background indexing failed:", error) }) } - private runStartup(operation: (epoch: number) => Promise): Promise { + private startOrchestrator(): Promise { if (this.startInFlight) return this.startInFlight - const epoch = this.startupEpoch - const startup = Promise.resolve().then(async () => { - try { - if (epoch !== this.startupEpoch) return - await operation(epoch) - while ( - epoch === this.startupEpoch && - !this.parserFailed && - this.isFeatureEnabled && - this.isWorkspaceEnabled - ) { - if (this.servicesRestartPending) { - this.servicesRestartPending = false - await this._recreateServices() - continue - } - await this.startIndexingInBackground() - if (!this.servicesRestartPending) break - } - } catch (error) { - if (epoch !== this.startupEpoch) return - this.startupFailureMessage = codeIndexLog.redact(error instanceof Error ? error.message : String(error)) - this._stateManager.setSystemState("Error", this.startupFailureMessage) - codeIndexLog.error("Indexing startup failed", { error, attempt: this.currentAttempt }) - throw error - } - }) - this.startInFlight = startup.finally(() => { + const orchestrator = this._orchestrator + if (!orchestrator) return Promise.resolve() + this.startInFlight = orchestrator.startIndexing().finally(() => { this.startInFlight = undefined - this.servicesRestartPending = false }) return this.startInFlight } - private async startIndexingInBackground(): Promise { - if (this.servicesStopped || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return + /** + * Initiates the indexing process (initial scan and starts watcher). + * Automatically recovers from error state if needed before starting. + * A repeated call while a start or scan is running returns that start's promise (R13). + */ + public async startIndexing(): Promise { + if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { + return + } + + // Delegate to semble provider if active if (this._sembleProvider) { await this._sembleProvider.startIndexing() return } - const orchestrator = this._orchestrator - if (!orchestrator) return - const attempt = this.currentAttempt - void orchestrator.startIndexing().catch((error: unknown) => { - if (attempt !== this.currentAttempt || orchestrator.stopped) return - codeIndexLog.error("Background scan failed", { error, attempt }) - }) - await orchestrator.startupResult + + if (this.startInFlight) return this.startInFlight + this.failureRegistry.resetAllBudgets() + + // Check if we're in error state and recover if needed + const currentStatus = this.getCurrentStatus() + if (currentStatus.systemStatus === "Error") { + await this.recoverFromError() + + // After recovery, we need to reinitialize since recoverFromError clears all services + // This will be handled by the caller (webviewMessageHandler) checking isInitialized + return + } + + this.assertInitialized() + await this.startOrchestrator() } + /** + * Stops any in-progress indexing operation and the file watcher. + * Repeated calls while stopping return the same promise. + */ public stopIndexing(): Promise { - this.startupEpoch++ - this.servicesStopped = true - this._sembleProvider?.stopIndexing() - this.parser?.stop() - if (this._orchestrator) return this._orchestrator.stopIndexing() - this.currentAttempt++ - return this.idleStop + if (this._sembleProvider) { + this._sembleProvider.stopIndexing() + return Promise.resolve() + } + return this._orchestrator?.stopIndexing() ?? Promise.resolve() } + /** + * Stops the file watcher and potentially cleans up resources. + */ public stopWatcher(): Promise { - this.startupEpoch++ - this.servicesStopped = true - this._sembleProvider?.stopIndexing() - this.parser?.stop() - if (this._orchestrator) return this._orchestrator.stopWatcher() - this.currentAttempt++ - return this.idleStop + if (!this.isFeatureEnabled) { + return Promise.resolve() + } + return this._orchestrator?.stopWatcher() ?? Promise.resolve() } - /** Drops services without clearing the fatal-parser latch or retained workspace data. */ + /** + * Recovers from error state by stopping the current scan and watcher, clearing the error and + * dropping service instances so the next initialize() recreates them. + * + * @remarks + * - Safe to call even when not in error state (idempotent) + * - Does not restart indexing automatically - call initialize() after recovery + * - Prevents race conditions from multiple concurrent recovery attempts + */ public async recoverFromError(): Promise { - if (this._isRecoveringFromError) return + // Prevent race conditions from multiple rapid recovery attempts + if (this._isRecoveringFromError) { + return + } + this._isRecoveringFromError = true try { - void this.stopWatcher() - this.currentAttempt++ + // The old scan and watcher must stop before replacement work starts (R37). + await this._orchestrator?.stopIndexing() this._stateManager.setSystemState("Standby", "") } catch (error) { - codeIndexLog.error("Failed to clear error state during recovery", { error }) + // Log error but continue with recovery - clearing service instances is more important + console.error("Failed to clear error state during recovery:", error) } finally { + // Force re-initialization by clearing service instances + // This ensures a clean slate even if state update failed this._configManager = undefined this._serviceFactory = undefined this._orchestrator = undefined this._searchService = undefined - this.disposeSemble() + this._sembleProvider = undefined + this._cacheManager = undefined + + // Reset the flag after recovery is complete this._isRecoveringFromError = false } } - private subscribeSembleState(): void { - this.sembleSubscription?.dispose() - const state = this.sembleState - const epoch = this.startupEpoch - const attempt = this.currentAttempt - this.sembleSubscription = state?.onProgressUpdate(() => { - if (epoch !== this.startupEpoch || attempt !== this.currentAttempt) return - const status = state.getCurrentStatus() - this._stateManager.setSystemState(status.systemStatus, status.message) - }) - } - - private disposeSemble(): void { - this.sembleSubscription?.dispose() - this.sembleSubscription = undefined - this._sembleProvider?.dispose() - this._sembleProvider = undefined - this.sembleState?.dispose() - this.sembleState = undefined - } - + /** + * Cleans up the manager instance. + */ public dispose(): void { - void this.stopWatcher() - this.currentAttempt++ - this.disposeSemble() - this.statusSubscriptions.forEach((subscription) => subscription.dispose()) - this.progressEmitter.dispose() + void this.stopIndexing() + if (this._sembleProvider) { + this._sembleProvider.dispose() + this._sembleProvider = undefined + } this._stateManager.dispose() - CodeIndexManager.diagnosticOwners.delete(this) } + /** + * Clears all index data by stopping the watcher, clearing the Qdrant collection, + * and deleting the cache file. + */ public async clearIndexData(): Promise { - if (!this.isFeatureEnabled) return + if (!this.isFeatureEnabled) { + return + } + if (this._sembleProvider) { + await this._sembleProvider.clearIndexData() + return + } this.assertInitialized() - this.startupEpoch++ - this.servicesStopped = true - const clearing = this._sembleProvider - ? this._sembleProvider.clearIndexData() - : this._orchestrator!.clearIndexData() - this.clearing = clearing.catch(() => {}) - await clearing + await this._orchestrator!.clearIndexData() + await this._cacheManager!.clearCacheFile() } + // --- Private Helpers --- + + /** + * Status precedence: Error > Stopping > active indexing or provider wait > idle partial (Standby with + * failures) > Indexed > Standby. + */ public getCurrentStatus() { const status = this._stateManager.getCurrentStatus() - const stopped = - this.servicesStopped || !!this._orchestrator?.stopped || !this.isWorkspaceEnabled || !this.isFeatureEnabled - const failedFiles: FailedFileStatus[] = this.fileFailureRegistry.list().map((failure) => ({ - path: codeIndexLog.redact( - path.isAbsolute(failure.path) ? path.relative(this.workspacePath, failure.path) : failure.path, - ), + const failedFiles = this.failureRegistry.list().map((failure) => ({ + path: failure.path, activity: failure.activity, - reason: codeIndexLog.redact(failure.reason), - state: stopped ? "exhausted" : failure.state, + reason: failure.reason, + state: failure.state, })) - const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles[0] - const failedFileSummary: FailedFileSummary | undefined = representative - ? { ...representative, count: failedFiles.length } - : undefined - const providerWaits = this.providerPacerRegistry.providerWaits().map((wait) => ({ - ...wait, - waitingFiles: wait.waitingFiles.map((file) => - codeIndexLog.redact(path.isAbsolute(file) ? path.relative(this.workspacePath, file) : file), - ), - })) - const waiting = !stopped && this.providerPacerRegistry.hasWaitingRequests() + const providerWaits = this.pacers.providerWaits() let systemStatus: IndexingState = status.systemStatus let message = status.message - if (this.parserFailed || this.startupFailureMessage) { - systemStatus = "Error" - message = this.parserFailed ? this.parserFailureMessage : this.startupFailureMessage! - } else if (systemStatus !== "Error" && systemStatus !== "Stopping") { - const retrying = failedFiles.some((failure) => failure.state === "retrying") - if (stopped) { - systemStatus = "Standby" - } else if (waiting) { + if (systemStatus !== "Error" && systemStatus !== "Stopping") { + if (this.pacers.hasWaitingRequests()) { systemStatus = "Indexing" message = formatProviderWaitMessage(providerWaits) - } else if (this._orchestrator?.isScanning || retrying || systemStatus === "Indexing") { + } else if (this._orchestrator?.isScanning) { systemStatus = "Indexing" - if (retrying) message = t("embeddings:orchestrator.indexingFilesRetrying") - } else if (this._orchestrator?.isNaturallySettledPartial) { + } else if (systemStatus !== "Indexing" && failedFiles.length > 0) { systemStatus = "Standby" message = t("embeddings:orchestrator.indexingFilesFailed") - } else if (systemStatus !== "Indexed") { - systemStatus = "Standby" } } return { ...status, systemStatus, - message: codeIndexLog.redact(message), + message, failedFiles, - failedFileSummary, providerWaits, workspacePath: this.workspacePath, workspaceEnabled: this.isWorkspaceEnabled, @@ -424,127 +393,89 @@ export class CodeIndexManager { } } - public async searchIndex( - query: string, - directoryPrefix?: string, - signal?: AbortSignal, - ): Promise { + public async searchIndex(query: string, directoryPrefix?: string): Promise { + const empty: CodeIndexSearchResponse = { results: [], incomplete: false, failedPaths: [], waitingPaths: [] } if (!this.isFeatureEnabled) { - const failedPaths = this.fileFailureRegistry.list().map((failure) => failure.path) - const waitingPaths = this.providerPacerRegistry.waitingPaths() - return { - results: [], - incomplete: failedPaths.length > 0 || waitingPaths.length > 0, - failedPaths, - waitingPaths, - } + return empty } if (this._sembleProvider) { - return { - results: await this._sembleProvider.searchIndex(query, directoryPrefix), - incomplete: false, - failedPaths: [], - waitingPaths: [], - } + return { ...empty, results: await this._sembleProvider.searchIndex(query, directoryPrefix) } } this.assertInitialized() - return this._searchService!.searchIndex(query, directoryPrefix, signal) + return this._searchService!.searchIndex(query, directoryPrefix) } + /** + * Private helper method to recreate services with current configuration. + * Used by both initialize() and handleSettingsChange(). + */ private async _recreateServices(): Promise { - const epoch = this.startupEpoch - // Keep the clearing owner's attempt valid until its deletion has finished. - await this.clearing - if (epoch !== this.startupEpoch || !this.isFeatureEnabled || !this.isWorkspaceEnabled) return - const attempt = ++this.currentAttempt - const isCurrentAttempt = () => attempt === this.currentAttempt - const isCurrentStartup = () => isCurrentAttempt() && epoch === this.startupEpoch - this._stateManager.setDiagnosticContext({ source: "manager", attempt }) - void this._orchestrator?.stopWatcher() - this.parser?.stop() - this.parser = undefined + // The old scan and watcher must stop before replacement services start (R37). + if (this._orchestrator) { + await this._orchestrator.stopIndexing() + } + // Dispose existing semble provider if switching away + if (this._sembleProvider) { + this._sembleProvider.dispose() + this._sembleProvider = undefined + } + // Clear existing services to ensure clean state this._orchestrator = undefined this._searchService = undefined - this._serviceFactory = undefined - this.disposeSemble() + + // Branch: if provider is "semble", create SembleProvider instead of external services if (this._configManager!.currentEmbedderProvider === "semble") { - const state = new CodeIndexStateManager() - this.sembleState = state - this.subscribeSembleState() - const provider = new SembleProvider(this.workspacePath, this.context, state) - this._sembleProvider = provider - await provider.initialize() - if (!isCurrentStartup()) { - if (this._sembleProvider === provider) this.disposeSemble() - else provider.dispose() - return - } - this.servicesStopped = false + this._sembleProvider = new SembleProvider(this.workspacePath, this.context, this._stateManager) + await this._sembleProvider.initialize() return } + + // (Re)Initialize service factory for external providers + this._serviceFactory = new CodeIndexServiceFactory( + this._configManager!, + this.workspacePath, + this._cacheManager!, + this.failureRegistry, + this.pacers, + ) + const ignoreInstance = ignore() + const workspacePath = this.workspacePath + + if (!workspacePath) { + this._stateManager.setSystemState("Standby", "") + return + } + + // Create .gitignore instance + const ignorePath = path.join(workspacePath, ".gitignore") try { - ignoreInstance.add(await fs.readFile(path.join(this.workspacePath, ".gitignore"), "utf8")) + const content = await fs.readFile(ignorePath, "utf8") + ignoreInstance.add(content) ignoreInstance.add(".gitignore") } catch (error) { - codeIndexLog.debug("Could not load .gitignore", { error }) + // Should never happen: reading file failed even though it exists + console.error("Unexpected error loading .gitignore:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "_recreateServices", + }) } - if (!isCurrentStartup()) return - const rooIgnoreController = new RooIgnoreController(this.workspacePath) + + // Create RooIgnoreController instance + const rooIgnoreController = new RooIgnoreController(workspacePath) await rooIgnoreController.initialize() - if (!isCurrentStartup()) return - const parser = new ParseWorkerClient() - this.parser = parser - const factory = new CodeIndexServiceFactory( - this._configManager!, - this.workspacePath, - this._cacheManager!, - this.writeGate, - this.generationAllocator!, - this.fileFailureRegistry, - parser, - isCurrentAttempt, - this.providerPacerRegistry, - ) - this._serviceFactory = factory - const { embedder, vectorStore, scanner, fileWatcher } = factory.createServices( + + // (Re)Create shared service instances + const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices( this.context, this._cacheManager!, ignoreInstance, rooIgnoreController, ) - const pacer = this.providerPacerRegistry.get(embedder.rateLimitScope) - let deferredValidation = pacer.state().pausedUntil !== undefined - let validationResult: { valid: boolean; error?: string } = { valid: true } - if (!deferredValidation) { - const issueSeq = pacer.markIssued() - try { - validationResult = await factory.validateEmbedder(embedder) - } catch (error) { - if (!isCurrentStartup()) { - parser.stop() - fileWatcher.dispose() - return - } - if (classifyIndexingError(error) !== "rate-limit") { - parser.stop() - fileWatcher.dispose() - throw error - } - pacer.reportRateLimit(error, issueSeq) - deferredValidation = true - } - } - if (!isCurrentStartup()) { - parser.stop() - fileWatcher.dispose() - return - } - if (!validationResult.valid) { - parser.stop() - fileWatcher.dispose() - throw new Error(validationResult.error || "Embedder configuration validation failed") - } + + // (Re)Initialize orchestrator this._orchestrator = new CodeIndexOrchestrator( this._configManager!, this._stateManager, @@ -553,82 +484,67 @@ export class CodeIndexManager { vectorStore, scanner, fileWatcher, - parser, - this.writeGate, - this.generationAllocator!, - this.fileFailureRegistry, - isCurrentAttempt, - (_error, message) => { - if (!isCurrentAttempt()) return - this.parserFailed = true - this.parserFailureMessage = codeIndexLog.redact(message) - this.servicesStopped = true - this.progressEmitter.fire(this.getCurrentStatus()) - }, - this.providerPacerRegistry, - deferredValidation - ? async (signal) => { - const result = await pacer.run(undefined, signal, () => factory.validateEmbedder(embedder)) - if (!result.valid) throw new Error(result.error || "Embedder configuration validation failed") - } - : undefined, + this.failureRegistry, + this.pacers, ) + + // (Re)Initialize search service this._searchService = new CodeIndexSearchService( this._configManager!, this._stateManager, embedder, vectorStore, - this.fileFailureRegistry, - this.providerPacerRegistry, - () => this.getCurrentStatus(), + this.failureRegistry, + this.pacers, ) - this.servicesStopped = false - this._stateManager.setSystemState("Standby", "") - } - private configureDiagnostics(): void { - const owners = [...CodeIndexManager.diagnosticOwners] - codeIndexLog.configure({ - workspacePaths: owners.map((manager) => manager.workspacePath), - secrets: owners.flatMap((manager) => { - const config = manager._configManager?.getConfig() - return [ - config?.openAiOptions?.openAiNativeApiKey, - config?.openAiCompatibleOptions?.apiKey, - config?.geminiOptions?.apiKey, - config?.mistralOptions?.apiKey, - config?.vercelAiGatewayOptions?.apiKey, - config?.openRouterOptions?.apiKey, - config?.qdrantApiKey, - ].filter((secret): secret is string => !!secret) - }), - }) + // Clear any error state after successful recreation + this._stateManager.setSystemState("Standby", "") } + /** + * Handle code index settings changes. + * This method should be called when code index settings are updated + * to ensure the CodeIndexConfigManager picks up the new configuration. + * If the configuration changes require a restart, the service will be restarted. + */ public async handleSettingsChange(): Promise { - const configuration = this._configManager - if (!configuration) return - const epoch = this.startupEpoch - const { requiresRestart } = await configuration.loadConfiguration() - if (configuration !== this._configManager || epoch !== this.startupEpoch) return - this.configureDiagnostics() - if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) { - this.disabledSinceFailure = true - void this.stopWatcher() - return - } - if (this.disabledSinceFailure) this.renewAfterDisable() - if (this.parserFailed) return - if (this.startInFlight) { - this.servicesRestartPending ||= requiresRestart - return this.startInFlight - } - if ((requiresRestart || this.servicesStopped || this.startupFailureMessage) && this.isFeatureConfigured) { - return this.runStartup(async (startupEpoch) => { - this.startupFailureMessage = undefined - await this.initializeCache() - if (startupEpoch === this.startupEpoch) await this._recreateServices() - }) + if (this._configManager) { + const { requiresRestart } = await this._configManager.loadConfiguration() + + const isFeatureEnabled = this.isFeatureEnabled + const isFeatureConfigured = this.isFeatureConfigured + + // If feature is disabled, stop the service (including any active scan) + if (!isFeatureEnabled) { + await this.stopIndexing() + this._stateManager.setSystemState("Standby", "Code indexing is disabled") + return + } + + if (requiresRestart && isFeatureEnabled && isFeatureConfigured) { + try { + // Ensure cacheManager is initialized before recreating services + if (!(await this.ensureCacheManager())) return + + // Recreate services with new configuration + await this._recreateServices() + } catch (error) { + console.error("Failed to recreate services:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "handleSettingsChange", + }) + // Replacement failures leave indexing restartable from Error. + this._stateManager.setSystemState( + "Error", + error instanceof Error ? error.message : t("embeddings:orchestrator.unknownError"), + ) + // Re-throw the error so the caller knows validation failed + throw error + } + } } } } diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 1f070d101a..585364157b 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -1,426 +1,325 @@ import * as vscode from "vscode" import * as path from "path" -import type { CodeIndexConfigManager } from "./config-manager" -import type { CodeIndexStateManager, IndexingState } from "./state-manager" -import type { IFileWatcher, IVectorStore, IDirectoryScanner } from "./interfaces" -import type { CacheManager } from "./cache-manager" +import { CodeIndexConfigManager } from "./config-manager" +import { CodeIndexStateManager, IndexingState } from "./state-manager" +import { IFileWatcher, IVectorStore, BatchProcessingSummary } from "./interfaces" +import { DirectoryScanner } from "./processors" +import { CacheManager } from "./cache-manager" import { CodeIndexScanExecutor } from "./code-index-scan-executor" -import { t } from "../../i18n" -import { ParserFatalError, abortError } from "./parser-errors" -import { ParseWorkerClient } from "./parse-worker-client" -import { IndexWriteGate } from "./index-write-gate" -import { GenerationAllocator } from "./generation-allocator" import { FileFailureRegistry } from "./file-failure-registry" import { ProviderPacerRegistry } from "./provider-pacer" -import { INDEX_STOP_LIMIT_MS } from "./constants" -import { codeIndexLog } from "./code-index-log" import { formatProviderWaitMessage } from "./shared/provider-wait-message" -import { generateRelativeFilePath } from "./shared/get-relative-path" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" +import { t } from "../../i18n" +/** + * Manages the code indexing workflow, coordinating between different services and managers. + * + * One `AbortController` covers a whole session (scan and watcher). Every status write except the final + * Standby in `stop()` first checks that controller's signal, so a stopped session cannot overwrite a later one. + */ export class CodeIndexOrchestrator { - private watcherSubscriptions: vscode.Disposable[] = [] - private scanning = false - private controller?: AbortController + private _fileWatcherSubscriptions: vscode.Disposable[] = [] + private _isProcessing: boolean = false + private _abortController: AbortController | null = null private scanPromise?: Promise private stopPromise?: Promise - private readonly idleStop = Promise.resolve() - public startupResult: Promise = Promise.resolve() - private resolveStartup?: () => void - private rejectStartup?: (error: unknown) => void - private failureSubscription?: vscode.Disposable - private pacerSubscription?: vscode.Disposable + private scanDone = false private watcherBatchPending = false - private scanPartial = false - private batchPartial = false - private naturallySettled = false - public stopped = false - private parserFailure?: ParserFatalError - private collectionIncomplete = false - private recoveryCompletion?: Promise + private recoverySubscriptions: vscode.Disposable[] = [] private readonly scanExecutor: CodeIndexScanExecutor constructor( private readonly configManager: CodeIndexConfigManager, private readonly stateManager: CodeIndexStateManager, - private readonly workspacePath: string, + workspacePath: string, private readonly cacheManager: CacheManager, private readonly vectorStore: IVectorStore, - private readonly scanner: IDirectoryScanner, + scanner: DirectoryScanner, private readonly fileWatcher: IFileWatcher, - private readonly parser: ParseWorkerClient = new ParseWorkerClient(), - private readonly writeGate = new IndexWriteGate(), - private readonly generations = new GenerationAllocator(cacheManager), - private readonly failures = new FileFailureRegistry(), - private readonly isCurrentAttempt: () => boolean = () => true, - private readonly onFatal?: (error: ParserFatalError, message: string) => void, - private readonly providerPacers = new ProviderPacerRegistry(), - private readonly beforeScan?: (signal: AbortSignal) => Promise, + private readonly failures: FileFailureRegistry = new FileFailureRegistry(), + private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), ) { - this.scanExecutor = new CodeIndexScanExecutor(workspacePath, scanner, stateManager, isCurrentAttempt) - } - - private isLive(signal: AbortSignal): boolean { - return !signal.aborted && this.isCurrentAttempt() + this.scanExecutor = new CodeIndexScanExecutor(workspacePath, scanner, vectorStore, stateManager) } - private checkLive(signal: AbortSignal): void { - if (!this.isLive(signal)) throw abortError() + private setState(signal: AbortSignal, state: IndexingState, message?: string): void { + if (!signal.aborted) this.stateManager.setSystemState(state, message) } - private write(signal: AbortSignal, operation: () => Result | Promise): Promise { - return this.writeGate.run(signal, () => { - this.checkLive(signal) - return operation() - }) + private hasOutstandingWork(): boolean { + return this.failures.list().length > 0 || this.pacers.hasWaitingRequests() } - private failStop(error: ParserFatalError): void { - const signal = this.controller?.signal - if (!signal || !this.isLive(signal) || this.parserFailure) return - this.parserFailure = error - const message = t("embeddings:orchestrator.parserFailed", { - filePath: error.filePath, - category: error.category, - }) - this.stateManager.setSystemState("Error", message, "error") - if (this.isLive(signal)) this.onFatal?.(error, message) - void this.stop(true) - } - - private hasIncompleteFiles(): boolean { - return ( - this.failures.list().length > 0 || - this.providerPacers.waitingPaths().length > 0 || - this.watcherBatchPending || - this.scanPartial || - this.batchPartial || - (this.stopped && !!this.scanner.hasIncompleteFiles) || - Object.keys(this.cacheManager.getOpenFiles()).length > 0 - ) - } - - private updateRecoveryStatus(signal: AbortSignal, source = "changed-file-batch"): void { - if ( - !this.isLive(signal) || - this.stopped || - this.stateManager.state === "Error" || - this.stateManager.state === "Stopping" - ) + /** Reflects failures and provider waits in the status; never replaces Stopping or Error. */ + private updateRecoveryStatus(signal: AbortSignal): void { + const state = this.stateManager.state + if (signal.aborted || state === "Stopping" || state === "Error") return + if (this.pacers.hasWaitingRequests()) { + this.stateManager.setSystemState("Indexing", formatProviderWaitMessage(this.pacers.providerWaits())) + return + } + if (!this.scanDone || this.watcherBatchPending) return + if (this.failures.list().length > 0) { + this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingFilesFailed")) return - const failures = this.failures.list() - const retrying = failures.some((failure) => failure.state === "retrying") - const message = t( - retrying ? "embeddings:orchestrator.indexingFilesRetrying" : "embeddings:orchestrator.indexingFilesFailed", - ) - if (this.providerPacers.hasWaitingRequests()) { - this.stateManager.setSystemState( - "Indexing", - formatProviderWaitMessage(this.providerPacers.providerWaits()), - source, - ) - } else if (this.scanning || this.watcherBatchPending || retrying) { - this.stateManager.setSystemState("Indexing", failures.length > 0 ? message : undefined, source) - } else if (this.hasIncompleteFiles()) { - if (this.naturallySettled) this.stateManager.setSystemState("Standby", message, source) - } else if (this.collectionIncomplete) { - this.recoveryCompletion ??= this.completeRecovery(signal, source).finally(() => { - this.recoveryCompletion = undefined - }) - } else { - this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted"), source) + } + if (state !== "Indexed") { + this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) } } - private async completeRecovery(signal: AbortSignal, source: string): Promise { - try { - await this.write(signal, async () => { - if ( - this.stopped || - this.stateManager.state === "Error" || - this.stateManager.state === "Stopping" || - this.hasIncompleteFiles() || - this.scanning - ) - return - await this.vectorStore.markIndexingComplete() - if (this.isLive(signal)) this.collectionIncomplete = false - }) - if (this.isLive(signal) && !this.collectionIncomplete) this.updateRecoveryStatus(signal, source) - } catch (error) { - if (!this.isLive(signal) || this.stopped) return - codeIndexLog.error("Recovery completion failed", { error }) - this.stateManager.setSystemState("Error", String(error), "error") - void this.stop(true) + private async _startWatcher(signal: AbortSignal): Promise { + if (!this.configManager.isFeatureConfigured) { + throw new Error("Cannot start watcher: Service not configured.") } - } - private async startWatcher(signal: AbortSignal): Promise { - this.checkLive(signal) - this.watcherSubscriptions = [ - this.fileWatcher.onDidStartBatchProcessing(() => { - if (!this.isLive(signal)) return - this.watcherBatchPending = true - this.naturallySettled = false - this.stateManager.setSystemState("Indexing", "Processing file changes...", "changed-file-batch") - }), - this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { - if (!this.isLive(signal)) return - this.stateManager.reportFileQueueProgress( - processedInBatch, - totalInBatch, - currentFile ? path.basename(currentFile) : undefined, - "changed-file-batch", - ) - }), - this.fileWatcher.onDidFinishBatchProcessing((summary) => { - if (!this.isLive(signal)) return - this.watcherBatchPending = false - this.naturallySettled = true - // Registry-backed failures recover through the registry, not a sticky batch flag. - const registered = new Set(this.failures.list().map((failure) => failure.path)) - this.batchPartial = - (!!summary.batchError && summary.processedFiles.length === 0) || - summary.processedFiles.some( - (file) => - (file.status === "error" || file.status === "local_error") && - !registered.has(generateRelativeFilePath(file.path, this.workspacePath)), + this.setState(signal, "Indexing", "Initializing file watcher...") + + try { + await this.fileWatcher.initialize(signal) + + this._fileWatcherSubscriptions = [ + this.fileWatcher.onDidStartBatchProcessing(() => { + this.watcherBatchPending = true + }), + this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { + if (signal.aborted || this.stateManager.state === "Stopping") return + if (totalInBatch > 0 && this.stateManager.state !== "Indexing") { + this.stateManager.setSystemState("Indexing", "Processing file changes...") + } + this.stateManager.reportFileQueueProgress( + processedInBatch, + totalInBatch, + currentFile ? path.basename(currentFile) : undefined, ) - this.updateRecoveryStatus(signal) - }), - ] - if (this.fileWatcher.onParserFatal) { - this.watcherSubscriptions.push(this.fileWatcher.onParserFatal((error) => this.failStop(error))) + }), + this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { + if (summary.batchError) { + console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) + } + this.watcherBatchPending = false + this.updateRecoveryStatus(signal) + }), + ] + } catch (error) { + console.error("[CodeIndexOrchestrator] Failed to start file watcher:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "_startWatcher", + }) + throw error } - await this.fileWatcher.initialize(signal) - this.checkLive(signal) } - public startIndexing(): Promise { - if (this.stopped || !this.isCurrentAttempt()) return Promise.resolve() - if (this.scanning) return this.scanPromise ?? Promise.resolve() - if (this.controller) return Promise.resolve() - if (!vscode.workspace.workspaceFolders?.length) { - const message = t("embeddings:orchestrator.indexingRequiresWorkspace") - this.stateManager.setSystemState("Error", message, "error") - this.startupResult = Promise.reject(new Error(message)) - void this.startupResult.catch(() => {}) - return Promise.resolve() + /** + * Starts one indexing session (scan, then watcher). A repeated call while a scan runs returns the running + * scan; a call while stopping waits for the stop to finish first. + */ + public async startIndexing(): Promise { + if (this.stopPromise) await this.stopPromise + if (this.scanPromise) return this.scanPromise + + if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) { + this.stateManager.setSystemState("Error", t("embeddings:orchestrator.indexingRequiresWorkspace")) + console.warn("[CodeIndexOrchestrator] Start rejected: No workspace folder open.") + return } + if (!this.configManager.isFeatureConfigured) { - this.stateManager.setSystemState( - "Standby", - "Missing configuration. Save your settings to start indexing.", - "start-request", - ) - return Promise.resolve() + this.stateManager.setSystemState("Standby", "Missing configuration. Save your settings to start indexing.") + console.warn("[CodeIndexOrchestrator] Start rejected: Missing configuration.") + return } - this.scanning = true - this.controller = new AbortController() - this.startupResult = new Promise((resolve, reject) => { - this.resolveStartup = resolve - this.rejectStartup = reject + + if (this._abortController && !this._abortController.signal.aborted) { + console.warn("[CodeIndexOrchestrator] Start rejected: indexing session already active.") + return + } + + this._isProcessing = true + this.scanDone = false + this._abortController = new AbortController() + const signal = this._abortController.signal + this.recoverySubscriptions = [ + this.failures.onDidChange(() => this.updateRecoveryStatus(signal)), + this.pacers.onDidChange(() => this.updateRecoveryStatus(signal)), + ] + this.stateManager.setSystemState("Indexing", "Initializing services...") + this.scanPromise = this.runScan(signal).finally(() => { + this.scanPromise = undefined }) - void this.startupResult.catch(() => {}) - const signal = this.controller.signal - this.failureSubscription = this.failures.onDidChange(() => - this.updateRecoveryStatus(signal, this.scanning ? "scan" : "changed-file-batch"), - ) - this.pacerSubscription = this.providerPacers.onDidChange(() => - this.updateRecoveryStatus(signal, this.scanning ? "scan" : "changed-file-batch"), - ) - this.stateManager.setSystemState("Indexing", "Initializing services...", "start-request") - this.scanPromise = this.runScan(signal) return this.scanPromise } private async runScan(signal: AbortSignal): Promise { + // Only clean up collections created by this run; existing data must survive failed retries. let clearIndexOnError = false + try { - if (this.beforeScan) { - this.resolveStartup?.() - await this.beforeScan(signal) - this.checkLive(signal) + const collectionCreated = await this.vectorStore.initialize(signal) + clearIndexOnError = collectionCreated + + if (collectionCreated && !signal.aborted) { + await this.cacheManager.clearCacheFile() } - clearIndexOnError = await this.write(signal, () => this.vectorStore.initialize()) - this.checkLive(signal) - if (clearIndexOnError) await this.write(signal, () => this.cacheManager.clearCacheFile()) - this.checkLive(signal) - await this.generations.initialize(this.vectorStore, signal, this.writeGate) - this.checkLive(signal) + const hasExistingData = await this.vectorStore.hasIndexedData() - this.checkLive(signal) - await this.write(signal, () => this.vectorStore.markIndexingIncomplete()) - this.checkLive(signal) - this.collectionIncomplete = true - this.resolveStartup?.() const completed = - hasExistingData && !clearIndexOnError + hasExistingData && !collectionCreated ? await this.scanExecutor.runIncrementalScan(signal) : await this.scanExecutor.runFullScan(signal) - this.checkLive(signal) - if (!completed) { - void this.stop(true) - return - } - this.scanPartial = this.scanExecutor.partial && this.failures.list().length === 0 - await this.startWatcher(signal) - this.checkLive(signal) - this.naturallySettled = true - } catch (error) { - this.rejectStartup?.(error) - if (!this.isLive(signal)) return - if (error instanceof ParserFatalError) { - this.failStop(error) - return + if (!completed || signal.aborted) return + + await this._startWatcher(signal) + if (signal.aborted) return + + this.scanDone = true + if (!this.scanExecutor.partial && !this.hasOutstandingWork()) { + await this.vectorStore.markIndexingComplete() } - if (error instanceof Error && error.name === "AbortError") { - void this.stop(true) + this.updateRecoveryStatus(signal) + } catch (error: any) { + // A stop aborts in-flight work; that is not an indexing failure. + if (error?.name === "AbortError" || signal.aborted) { + console.log("[CodeIndexOrchestrator] Indexing aborted by user.") return } - codeIndexLog.error("Initial scan failed", { error }) + + console.error("[CodeIndexOrchestrator] Error during indexing:", error) + // Scanner/provider errors and stacks may contain local paths or other private data. + // Keep details local; send only a fixed failure category across the telemetry boundary. + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: "Indexing failed", + location: "startIndexing", + }) if (clearIndexOnError) { try { - await this.write(signal, () => this.vectorStore.clearCollection()) - await this.write(signal, () => this.cacheManager.clearCacheFile()) + await this.vectorStore.clearCollection() } catch (cleanupError) { - if (this.isLive(signal)) codeIndexLog.error("Index cleanup failed", { error: cleanupError }) + console.error("[CodeIndexOrchestrator] Failed to clean up after error:", cleanupError) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: "Index cleanup failed", + location: "startIndexing.cleanup", + }) } + // Indexing started but failed mid-way - clear cache to avoid cache-Qdrant mismatch + await this.cacheManager.clearCacheFile() + } else { + console.log("[CodeIndexOrchestrator] Preserving existing index and cache for a retry.") } - if (!this.isLive(signal)) return - this.stateManager.setSystemState( + + this.setState( + signal, "Error", t("embeddings:orchestrator.failedDuringInitialScan", { - errorMessage: error instanceof Error ? error.message : t("embeddings:orchestrator.unknownError"), + errorMessage: error?.message || t("embeddings:orchestrator.unknownError"), }), - "error", ) - void this.stop(true) + this.disposeWatcher() } finally { - this.resolveStartup?.() - this.scanning = false - this.updateRecoveryStatus(signal, "scan-end") - if (this.recoveryCompletion) await this.recoveryCompletion + this._isProcessing = false } } + /** + * Stops the session: aborts the scan, provider waits and retry waits at once, disposes the watcher, + * then writes Standby. Repeated calls while stopping join the same promise. + */ public stopIndexing(): Promise { - return this.stop(false) + if (this.stopPromise) return this.stopPromise + const controller = this._abortController + if (!controller || controller.signal.aborted) { + this.disposeWatcher() + return Promise.resolve() + } + const preserveError = this.stateManager.state === "Error" + if (!preserveError) this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStopping")) + controller.abort() + this.disposeWatcher() + const scan = this.scanPromise + this.stopPromise = (async () => { + await scan?.catch(() => {}) + await this.cacheManager.flush() + if (!preserveError) { + this.stateManager.setSystemState( + "Standby", + t( + this.hasOutstandingWork() || !this.scanDone + ? "embeddings:orchestrator.indexingStoppedPartial" + : "embeddings:orchestrator.indexingStopped", + ), + ) + } + })().finally(() => { + this.stopPromise = undefined + }) + return this.stopPromise } + /** Stops scan and watcher; kept for callers that only need the watcher gone. */ public stopWatcher(): Promise { - return this.stop(true) - } - - private persistUncertain(signal?: AbortSignal): Promise { - if (!this.isCurrentAttempt()) return Promise.resolve() - const paths = signal ? this.writeGate.uncertainFiles(signal) : [] - return Promise.all(paths.map((filePath) => this.cacheManager.invalidateFile(filePath))) - .then(() => (this.isCurrentAttempt() ? this.cacheManager.flush() : undefined)) - .catch((error: unknown) => { - if (this.isCurrentAttempt()) codeIndexLog.error("Stopped index cache persistence failed", { error }) - }) + return this.stopIndexing() } - private stop(force: boolean): Promise { - if (this.stopPromise) return this.stopPromise - const signal = this.controller?.signal - const active = !!signal && !signal.aborted - if (!active && !force && !this.parserFailure) return this.idleStop - const preserveError = this.stateManager.state === "Error" || !!this.parserFailure - const expiresAt = Date.now() + INDEX_STOP_LIMIT_MS - if (signal) this.writeGate.stopAt(signal, expiresAt) - this.stopPromise = Promise.resolve().then(async () => { - let timer: ReturnType | undefined - let workerStopped = false - const workerExit = this.parser.stopped.then(() => { - workerStopped = true - }) - const settled = Promise.allSettled([ - this.scanPromise, - signal ? this.writeGate.settled(signal) : undefined, - workerExit, - ]).then(async () => { - await this.persistUncertain(signal) - return false - }) - const expired = await Promise.race([ - settled, - new Promise((resolve) => { - timer = setTimeout(() => resolve(true), Math.max(0, expiresAt - Date.now())) - }), - ]) - clearTimeout(timer) - if (expired && signal) { - this.writeGate.abandon(signal, expiresAt) - // Invalidation mutates the retained cache synchronously; disk I/O cannot extend the stop deadline. - void this.persistUncertain(signal) - } - if (expired && !workerStopped) this.parser.logStopTimeout() - if (!active || !this.isCurrentAttempt() || preserveError || this.state === "Error") return - this.stateManager.setSystemState( - "Standby", - t( - expired || this.hasIncompleteFiles() - ? "embeddings:orchestrator.indexingStoppedPartial" - : "embeddings:orchestrator.indexingStopped", - ), - "stop", - ) - }) - this.stopped = true - this.naturallySettled = false - this.controller?.abort() - this.resolveStartup?.() - this.failureSubscription?.dispose() - this.failureSubscription = undefined - this.pacerSubscription?.dispose() - this.pacerSubscription = undefined - this.parser.stop() + private disposeWatcher(): void { this.fileWatcher.dispose() - this.watcherSubscriptions.forEach((subscription) => subscription.dispose()) - this.watcherSubscriptions = [] - if (active && !preserveError && this.isCurrentAttempt()) { - this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStopping"), "stop") - } - return this.stopPromise + this._fileWatcherSubscriptions.forEach((sub) => sub.dispose()) + this._fileWatcherSubscriptions = [] + this.recoverySubscriptions.forEach((sub) => sub.dispose()) + this.recoverySubscriptions = [] + this.watcherBatchPending = false } + /** + * Clears all index data by stopping the watcher, clearing the vector store, + * and resetting the cache file. + */ public async clearIndexData(): Promise { - await this.stopWatcher() - const signal = new AbortController().signal + this._isProcessing = true + try { - this.checkLive(signal) - if (this.configManager.isFeatureConfigured) { - await this.write(signal, () => this.vectorStore.deleteCollection()) + await this.stopIndexing() + + try { + if (this.configManager.isFeatureConfigured) { + await this.vectorStore.deleteCollection() + } else { + console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") + } + } catch (error: any) { + console.error("[CodeIndexOrchestrator] Failed to clear vector collection:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "clearIndexData", + }) + this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) } - await this.write(signal, () => this.cacheManager.clearCacheFile()) - if (this.isLive(signal)) - this.stateManager.setSystemState("Standby", "Index data cleared successfully.", "rebuild") - } catch (error) { - if (!this.isLive(signal)) return - this.stateManager.setSystemState("Error", `Failed to clear index data: ${String(error)}`, "error") - throw error + + await this.cacheManager.clearCacheFile() + + if (this.stateManager.state !== "Error") { + this.stateManager.setSystemState("Standby", "Index data cleared successfully.") + } + } finally { + this._isProcessing = false } } public get isScanning(): boolean { - return this.scanning - } - - public get isActive(): boolean { - return !!this.controller && this.isLive(this.controller.signal) && !this.stopped + return this.scanPromise !== undefined } - public get isNaturallySettledPartial(): boolean { - return ( - this.isActive && - this.naturallySettled && - !this.scanning && - !this.watcherBatchPending && - this.hasIncompleteFiles() - ) + public get isStopping(): boolean { + return this.stopPromise !== undefined } + /** + * Gets the current state of the indexing system. + */ public get state(): IndexingState { return this.stateManager.state } diff --git a/src/services/code-index/processors/__tests__/file-preparation.spec.ts b/src/services/code-index/processors/__tests__/file-preparation.spec.ts index 22020a9fdd..96c585bb05 100644 --- a/src/services/code-index/processors/__tests__/file-preparation.spec.ts +++ b/src/services/code-index/processors/__tests__/file-preparation.spec.ts @@ -1,9 +1,9 @@ import { createHash } from "crypto" import path from "path" import { FileType, Uri } from "vscode" -import { v5 as uuidv5 } from "uuid" -import type { CodeBlock, ICodeParser, IEmbedder } from "../../interfaces" -import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../../constants" +import type { CodeBlock, ICodeParser } from "../../interfaces" +import { MAX_FILE_SIZE_BYTES } from "../../constants" +import { ParserLoadError } from "../parser" import { FilePreparation } from "../file-preparation" import type { FilePreparationDependencies } from "../file-preparation-dependencies" @@ -24,7 +24,6 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo const filePath = "/workspace/src/file.ts" const relativeFilePath = path[platform].join("src", "file.ts") - const normalizedFilePath = path[platform].resolve(filePath) const content = "test content" const hash = createHash("sha256").update(content).digest("hex") const block: CodeBlock = { @@ -57,11 +56,6 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo }, cacheManager: { getHash: vi.fn() }, parser: { parseFile: vi.fn().mockResolvedValue([{ ...block }]) }, - embedder: { - createEmbeddings: vi - .fn() - .mockResolvedValue({ embeddings: [[0.1, 0.2]] }), - }, } satisfies FilePreparationDependencies } @@ -81,7 +75,6 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }, ) @@ -104,7 +97,6 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }) it("skips oversized files without reading them", async () => { @@ -123,10 +115,9 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }) - it("skips unchanged content before parsing or embedding", async () => { + it("skips unchanged content before parsing", async () => { const dependencies = setup() dependencies.cacheManager.getHash.mockReturnValue(hash) expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ @@ -136,37 +127,28 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo }) expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }) - it("returns the hash and empty points when parsing produces no blocks", async () => { + it("returns the hash and no blocks when parsing produces none", async () => { const dependencies = setup() dependencies.parser.parseFile.mockResolvedValue([]) expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ path: filePath, status: "processed_for_batching", newHash: hash, - pointsToUpsert: [], + blocks: [], }) - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }) - it("still parses and returns the hash without an embedder or gitignore", async () => { + it("still parses and returns the hash without gitignore", async () => { const dependencies = setup() expect( - await new FilePreparation({ ...dependencies, embedder: undefined, ignoreInstance: undefined }).prepareFile( - filePath, - ), - ).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: hash, - pointsToUpsert: [], - }) + (await new FilePreparation({ ...dependencies, ignoreInstance: undefined }).prepareFile(filePath)).status, + ).toBe("processed_for_batching") expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) }) - it("preserves normalized paths, stable IDs and embedding order at the size limit", async () => { + it("CI-FP-03: returns parsed blocks and the hash at the size limit and never embeds", async () => { const dependencies = setup() dependencies.fileSystem.stat.mockResolvedValue({ type: FileType.File, @@ -174,37 +156,27 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo mtime: 0, size: MAX_FILE_SIZE_BYTES, }) - dependencies.parser.parseFile.mockResolvedValue([ - { ...block, file_path: "src/../src/file.ts" }, - { ...block, start_line: 8, end_line: 10, content: "second" }, - ]) - dependencies.embedder.createEmbeddings.mockResolvedValue({ - embeddings: [ - [0.1, 0.2], - [0.3, 0.4], - ], - }) - expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ - path: filePath, - status: "processed_for_batching", - newHash: hash, - pointsToUpsert: [ - { - id: uuidv5(`${normalizedFilePath}:2`, QDRANT_CODE_BLOCK_NAMESPACE), - vector: [0.1, 0.2], - payload: { filePath: relativeFilePath, codeChunk: content, startLine: 2, endLine: 5 }, - }, - { - id: uuidv5(`${normalizedFilePath}:8`, QDRANT_CODE_BLOCK_NAMESPACE), - vector: [0.3, 0.4], - payload: { filePath: relativeFilePath, codeChunk: "second", startLine: 8, endLine: 10 }, - }, - ], - }) + const blocks = [block, { ...block, start_line: 8, end_line: 10, content: "second" }] + dependencies.parser.parseFile.mockResolvedValue(blocks) + const result = await new FilePreparation(dependencies).prepareFile(filePath) + expect(result).toEqual({ path: filePath, status: "processed_for_batching", newHash: hash, blocks }) + expect(Object.keys(dependencies)).not.toContain("embedder") expect(dependencies.fileSystem.stat).toHaveBeenCalledWith(Uri.file(filePath)) expect(dependencies.fileSystem.readFile).toHaveBeenCalledWith(Uri.file(filePath)) - expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) - expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledWith([content, "second"]) + }) + + it.each(["stat", "read"] as const)("CI-FP-01: missing source from %s returns missing", async (stage) => { + for (const code of ["ENOENT", "FileNotFound"]) { + const dependencies = setup() + const missing = Object.assign(new Error("gone"), { code }) + if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(missing) + else dependencies.fileSystem.readFile.mockRejectedValue(missing) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "missing", + }) + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + } }) it("calls service methods with their original receivers", async () => { @@ -222,8 +194,6 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo expect(dependencies.cacheManager.getHash.mock.contexts[0]).toBe(dependencies.cacheManager) expect(dependencies.parser.parseFile).toHaveBeenCalledOnce() expect(dependencies.parser.parseFile.mock.contexts[0]).toBe(dependencies.parser) - expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(dependencies.embedder.createEmbeddings.mock.contexts[0]).toBe(dependencies.embedder) }) it("does not treat a hidden workspace ancestor as an excluded directory", async () => { @@ -274,7 +244,7 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo path: filePath, status: "processed_for_batching", newHash: expectedHash, - pointsToUpsert: [], + blocks: [], }) expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content: expectedContent, @@ -290,30 +260,33 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo }) expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) expect(dependencies.parser.parseFile).not.toHaveBeenCalled() - expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() }) }) - it.each(["access", "ignore", "stat", "read", "cache", "parse", "embed"])( - "returns the original %s error as a local error", - async (stage) => { - const dependencies = setup() - const error = new Error(`${stage} failed`) - const fail = () => { - throw error - } - if (stage === "access") dependencies.ignoreController.validateAccess.mockImplementation(fail) - if (stage === "ignore") dependencies.ignoreInstance.ignores.mockImplementation(fail) - if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(error) - if (stage === "read") dependencies.fileSystem.readFile.mockRejectedValue(error) - if (stage === "cache") dependencies.cacheManager.getHash.mockImplementation(fail) - if (stage === "parse") dependencies.parser.parseFile.mockRejectedValue(error) - if (stage === "embed") dependencies.embedder.createEmbeddings.mockRejectedValue(error) - const result = await new FilePreparation(dependencies).prepareFile(filePath) - expect(result).toEqual({ path: filePath, status: "local_error", error }) - expect(result.error).toBe(error) - }, - ) + it.each([ + ["access", "read"], + ["ignore", "read"], + ["stat", "read"], + ["read", "read"], + ["cache", "parse"], + ["parse", "parse"], + ["parser load", "parse"], + ] as const)("CI-FP-02: a %s error is a local error with activity %s", async (stage, activity) => { + const dependencies = setup() + const error = stage === "parser load" ? new ParserLoadError(".ts", new Error("wasm")) : new Error(`${stage} failed`) + const fail = () => { + throw error + } + if (stage === "access") dependencies.ignoreController.validateAccess.mockImplementation(fail) + if (stage === "ignore") dependencies.ignoreInstance.ignores.mockImplementation(fail) + if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(Object.assign(error, { code: "EACCES" })) + if (stage === "read") dependencies.fileSystem.readFile.mockRejectedValue(error) + if (stage === "cache") dependencies.cacheManager.getHash.mockImplementation(fail) + if (stage === "parse" || stage === "parser load") dependencies.parser.parseFile.mockRejectedValue(error) + const result = await new FilePreparation(dependencies).prepareFile(filePath) + expect(result).toEqual({ path: filePath, status: "local_error", activity, error }) + expect(result.error).toBe(error) + }) it("does not wrap non-Error rejections", async () => { const dependencies = setup() @@ -321,6 +294,7 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ path: filePath, status: "local_error", + activity: "read", error: "read failed", }) }) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 4959926e14..f2968fb81b 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -1,894 +1,464 @@ -import * as path from "path" +// npx vitest services/code-index/processors/__tests__/file-watcher.spec.ts + import * as vscode from "vscode" -import { createHash } from "crypto" -import ignore from "ignore" + import { FileWatcher } from "../file-watcher" -import type { CacheManager } from "../../cache-manager" -import type { - BatchProcessingSummary, - CodeBlock, - ICodeParser, - IEmbedder, - IVectorStore, - PointStruct, -} from "../../interfaces" +import { FilePreparation } from "../file-preparation" import { FileFailureRegistry } from "../../file-failure-registry" -import { GenerationAllocator } from "../../generation-allocator" -import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" -import { ParserFatalError } from "../../parser-errors" -import { codeIndexLog } from "../../code-index-log" -import { generateRelativeFilePath } from "../../shared/get-relative-path" -import { MAX_FILE_SIZE_BYTES } from "../../constants" -import { RooIgnoreController } from "../../../../core/ignore/RooIgnoreController" -import { makeExtensionContext, makeUri } from "../../../../test-utils/vscode" -import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" -import { ProviderPacerRegistry } from "../../provider-pacer" - -vi.mock("vscode", async () => { - const { makeEventEmitter, makeUri } = await import("../../../../test-utils/vscode") - return { - workspace: { fs: { readFile: vi.fn(), stat: vi.fn() }, createFileSystemWatcher: vi.fn() }, - Uri: { file: makeUri }, - RelativePattern: class { - constructor( - public base: string, - public pattern: string, - ) {} - }, - EventEmitter: class { - constructor() { - return makeEventEmitter() - } + +import { clearAllMocks } from "../../../../test-utils/reset" + +// Mock TelemetryService +vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), }, - } -}) -vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: class { - validateAccess = vi.fn(() => true) }, })) -vi.mock("../../code-index-log", () => ({ - codeIndexLog: { - info: vi.fn(), - debug: vi.fn(), - warn: vi.fn(), - redact: (message: string) => message, - createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), + +// Mock dependencies +vi.mock("../../cache-manager") +vi.mock("../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: vi.fn().mockImplementation(function () { + return { + validateAccess: vi.fn().mockReturnValue(true), + } + }), +})) +vi.mock("ignore") +vi.mock("../parser", () => ({ + codeParser: { + parseFile: vi.fn().mockImplementation(async (filePath: string) => [ + { + file_path: filePath, + content: "test content", + start_line: 1, + end_line: 1, + }, + ]), }, })) -const workspacePath = path.resolve("watcher-workspace") -const fileA = path.join(workspacePath, "a.ts") -const fileB = path.join(workspacePath, "b.ts") -const hash = (content: string) => createHash("sha256").update(content).digest("hex") -const watchers: FileWatcher[] = [] -let handlers: Record<"create" | "change" | "delete", (uri: vscode.Uri) => unknown> -const nativeDispose = vi.fn() - -function deferred() { - let resolve!: (value: Value) => void - let reject!: (error: Error) => void - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - return { promise, resolve, reject } -} +const createMockEventEmitter = () => { + const listeners = new Set<(event: any) => void>() -function blocks(filePath: string, count = 1): CodeBlock[] { - return Array.from({ length: count }, (_, blockNumber) => ({ - file_path: filePath, - content: `function block${blockNumber}() { return ${blockNumber} }`, - start_line: blockNumber + 1, - end_line: blockNumber + 1, - identifier: null, - type: "function", - fileHash: hash("current content"), - segmentHash: `${filePath}:${blockNumber}`, - })) -} - -function setup() { - const hashes = new Map() - const openFiles = new Set() - const points = new Map() - const cache = { - generationWatermark: 1024, - reserveGenerations: vi.fn().mockResolvedValue(undefined), - getHash: vi.fn((filePath: string) => hashes.get(filePath)), - isTracked: vi.fn((filePath: string) => hashes.has(filePath) || openFiles.has(filePath)), - getTrackedPaths: vi.fn(() => [...new Set([...hashes.keys(), ...openFiles])]), - invalidateFile: vi.fn(async (filePath: string) => { - hashes.delete(filePath) - openFiles.add(filePath) - }), - completeFile: vi.fn((filePath: string, revision: string) => { - hashes.set(filePath, revision) - openFiles.delete(filePath) + return { + event: vi.fn((listener: (event: any) => void) => { + listeners.add(listener) + return { + dispose: () => listeners.delete(listener), + } }), - completeRemoval: vi.fn((filePath: string) => { - hashes.delete(filePath) - openFiles.delete(filePath) + fire: vi.fn((event: any) => { + for (const listener of listeners) { + listener(event) + } }), - } - const store = { - rateLimitScope: "qdrant:watcher-test", - initialize: vi.fn().mockResolvedValue(false), - upsertPoints: vi.fn().mockImplementation(async (batch) => { - for (const point of batch) points.set(point.id, point) + dispose: vi.fn(() => { + listeners.clear() }), - deletePointsByFilePath: vi - .fn() - .mockImplementation(async (filePath, cutoff) => { - for (const point of points.values()) { - const payload: Record = point.payload - if ( - payload.filePath === generateRelativeFilePath(filePath, workspacePath) && - (typeof payload.generation !== "number" || payload.generation < cutoff) - ) - points.delete(point.id) - } - }), - deletePointsByMultipleFilePaths: vi - .fn() - .mockResolvedValue(undefined), - countPointsForFile: vi - .fn() - .mockImplementation(async (filePath, generation) => { - const selected = [...points.values()] - .map((point): Record => point.payload) - .filter((payload) => payload.filePath === generateRelativeFilePath(filePath, workspacePath)) - return { - current: selected.filter((payload) => payload.generation === generation).length, - other: selected.filter((payload) => payload.generation !== generation).length, - } - }), - search: vi.fn().mockResolvedValue([]), - clearCollection: vi.fn().mockResolvedValue(undefined), - deleteCollection: vi.fn().mockResolvedValue(undefined), - collectionExists: vi.fn().mockResolvedValue(true), - hasIndexedData: vi.fn().mockResolvedValue(true), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - } satisfies IVectorStore - const embedder = { - rateLimitScope: "openai:watcher-test", - createEmbeddings: vi - .fn() - .mockImplementation(async (texts) => ({ embeddings: texts.map(() => [1]) })), - validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), - embedderInfo: { name: "openai" }, - } satisfies IEmbedder - const parser = { - parseFile: vi.fn().mockImplementation(async (filePath) => blocks(filePath)), - } - const failures = new FileFailureRegistry() - const gate = new IndexWriteGate() - // The double supplies only the cache lookup, generation reservation, and completion methods used by watcher/writer. - const cacheManager = cache as unknown as CacheManager - const generations = new GenerationAllocator(cacheManager) - const ignored = ignore() - const access = new RooIgnoreController(workspacePath) - const pacers = new ProviderPacerRegistry() - let currentAttempt = true - const createWatcher = () => { - const created = new FileWatcher( - workspacePath, - makeExtensionContext(), - cacheManager, - embedder, - store, - ignored, - access, - 1, - parser, - gate, - generations, - failures, - () => currentAttempt, - pacers, - ) - watchers.push(created) - return created - } - return { - access, - cache, - hashes, - openFiles, - points, - store, - embedder, - parser, - failures, - gate, - ignored, - pacers, - createWatcher, - watcher: createWatcher(), - supersede: () => { - currentAttempt = false - }, } } -beforeEach(() => { - clearAllMocks() - vi.useFakeTimers() - handlers = { create: () => {}, change: () => {}, delete: () => {} } - vi.mocked(vscode.workspace.fs.readFile).mockReset().mockResolvedValue(Buffer.from("current content")) - vi.mocked(vscode.workspace.fs.stat).mockReset().mockResolvedValue({ size: 100, type: 1, ctime: 0, mtime: 0 }) - vi.mocked(vscode.workspace.createFileSystemWatcher).mockImplementation(() => ({ - ignoreCreateEvents: false, - ignoreChangeEvents: false, - ignoreDeleteEvents: false, - onDidCreate: (handler) => { - handlers.create = handler - return { dispose() {} } +// Mock vscode module +vi.mock("vscode", () => ({ + workspace: { + createFileSystemWatcher: vi.fn(), + workspaceFolders: [ + { + uri: { + fsPath: "/mock/workspace", + }, + }, + ], + fs: { + stat: vi.fn().mockResolvedValue({ size: 1000 }), + readFile: vi.fn().mockResolvedValue(Buffer.from("test content")), }, - onDidChange: (handler) => { - handlers.change = handler - return { dispose() {} } - }, - onDidDelete: (handler) => { - handlers.delete = handler - return { dispose() {} } - }, - dispose: nativeDispose, - })) -}) -afterEach(() => { - for (const watcher of watchers.splice(0)) watcher.dispose() - restoreGlobals() - vi.useRealTimers() -}) -async function queue(paths = [fileA], type: "create" | "change" | "delete" = "change") { - for (const filePath of paths) handlers[type](makeUri(filePath)) - await vi.advanceTimersByTimeAsync(500) -} -async function drain() { - for (let turn = 0; turn < 150; turn++) await Promise.resolve() -} - -describe("FileWatcher reliability", () => { - it("completes public processFile only after every batch, cleanup and exact verification", async () => { - const { watcher, parser, embedder, cache, store, hashes } = setup() - const held = deferred>>() - parser.parseFile.mockResolvedValue(blocks(fileA, 3)) - embedder.createEmbeddings.mockImplementationOnce(() => held.promise) - const pending = watcher.processFile(fileA) - await drain() - expect(watcher.isProcessing).toBe(true) - expect(cache.invalidateFile).toHaveBeenCalledOnce() - expect(hashes.has(fileA)).toBe(false) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - held.resolve({ embeddings: [[1]] }) - expect((await pending).status).toBe("success") - expect(store.upsertPoints).toHaveBeenCalledTimes(3) - expect(store.countPointsForFile).toHaveBeenCalledOnce() - expect(hashes.get(fileA)).toBe(hash("current content")) - expect(watcher.isProcessing).toBe(false) - }) + }, + RelativePattern: vi.fn().mockImplementation(function (base, pattern) { + return { base, pattern } + }), + Uri: { + file: vi.fn().mockImplementation((path) => ({ fsPath: path })), + }, + EventEmitter: vi.fn().mockImplementation(function () { + return createMockEventEmitter() + }), + ExtensionContext: vi.fn(), +})) - it("sets processing before start listeners and stops dispatch when a listener disposes", async () => { - const { watcher, parser } = setup() - await watcher.initialize() - watcher.onDidStartBatchProcessing(() => { - expect(watcher.isProcessing).toBe(true) - watcher.dispose() +describe("FileWatcher", () => { + let fileWatcher: FileWatcher + let mockWatcher: any + let mockOnDidCreate: any + let mockOnDidChange: any + let mockOnDidDelete: any + let mockContext: any + let mockCacheManager: any + let mockEmbedder: any + let mockVectorStore: any + let mockIgnoreInstance: any + + const waitForNextBatch = () => + new Promise((resolve) => { + const disposable = fileWatcher.onDidFinishBatchProcessing((summary) => { + disposable.dispose() + resolve(summary) + }) }) - await queue() - expect(parser.parseFile).not.toHaveBeenCalled() - expect(nativeDispose).toHaveBeenCalledOnce() - }) - it("initializes once and rejects pre-aborted, disposed and stale work before shared mutations", async () => { - const { watcher, failures, supersede } = setup() - await watcher.initialize() - await watcher.initialize() - expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledOnce() - supersede() - await expect(watcher.processFile(fileA)).rejects.toMatchObject({ name: "AbortError" }) - expect(failures.getRevision("a.ts")).toBeUndefined() - const other = setup().watcher - await expect(other.initialize(AbortSignal.abort())).rejects.toMatchObject({ name: "AbortError" }) - other.dispose() - await expect(other.initialize()).rejects.toMatchObject({ name: "AbortError" }) - expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledOnce() - }) + const flushBatch = async () => { + await vi.advanceTimersByTimeAsync(500) + } - it.each(["read", "parse"] as const)( - "records %s once without automatic retry and continues unrelated files", - async (activity) => { - const { watcher, parser, failures, hashes } = setup() - if (activity === "read") - vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(new Error("read failed")) - else parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) - const finished = vi.fn() - watcher.onDidFinishBatchProcessing(finished) - await watcher.initialize() - await queue([fileA, fileB]) - await vi.advanceTimersByTimeAsync(60_000) - expect(failures.list()).toEqual([ - expect.objectContaining({ path: "a.ts", activity, attempt: 1, state: "exhausted" }), - ]) - expect(hashes.has(fileB)).toBe(true) - expect(finished).toHaveBeenCalledOnce() - expect(vscode.workspace.fs.readFile).toHaveBeenCalledTimes(2) - const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value - expect(batch.record).toHaveBeenCalledOnce() - expect(batch.flush).toHaveBeenCalledOnce() - }, - ) - - it("does not reparse an exhausted revision until content changes or explicit retry resets it", async () => { - const { watcher, parser, failures } = setup() - parser.parseFile.mockRejectedValue(new Error("bad parse")) - await watcher.processFile(fileA) - await watcher.processFile(fileA) - expect(parser.parseFile).toHaveBeenCalledOnce() - vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("changed")) - await watcher.processFile(fileA) - expect(parser.parseFile).toHaveBeenCalledTimes(2) - failures.resetAllBudgets() - parser.parseFile.mockResolvedValue(blocks(fileA)) - expect((await watcher.processFile(fileA)).status).toBe("success") - expect(failures.list()).toEqual([]) + beforeEach(() => { + // Reset all mocks + clearAllMocks() + vi.useFakeTimers() + + // Create mock event handlers + mockOnDidCreate = vi.fn() + mockOnDidChange = vi.fn() + mockOnDidDelete = vi.fn() + + // Create mock watcher + mockWatcher = { + onDidCreate: vi.fn().mockImplementation((handler) => { + mockOnDidCreate = handler + return { dispose: vi.fn() } + }), + onDidChange: vi.fn().mockImplementation((handler) => { + mockOnDidChange = handler + return { dispose: vi.fn() } + }), + onDidDelete: vi.fn().mockImplementation((handler) => { + mockOnDidDelete = handler + return { dispose: vi.fn() } + }), + dispose: vi.fn(), + } + + // Mock createFileSystemWatcher to return our mock watcher + vi.mocked(vscode.workspace.createFileSystemWatcher).mockReturnValue(mockWatcher) + + // Create mock dependencies + mockContext = { + subscriptions: [], + } + + mockCacheManager = { + getHash: vi.fn(), + updateHash: vi.fn(), + deleteHash: vi.fn(), + } + + mockEmbedder = { + createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), + } + + mockVectorStore = { + upsertPoints: vi.fn().mockResolvedValue(undefined), + deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), + deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), + } + + mockIgnoreInstance = { + ignores: vi.fn().mockReturnValue(false), + } + + fileWatcher = new FileWatcher( + "/mock/workspace", + mockContext, + mockCacheManager, + mockEmbedder, + mockVectorStore, + mockIgnoreInstance, + ) }) - it.each(["embed", "upsert"] as const)( - "shares the %s retry allowance across batches and watcher recreation", - async (activity) => { - const { watcher, createWatcher, parser, embedder, store, failures } = setup() - parser.parseFile.mockResolvedValue(blocks(fileA, 2)) - const failure = Object.assign(new Error("temporary"), { status: 503 }) - const target = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints - target.mockRejectedValueOnce(failure).mockRejectedValueOnce(failure) - if (activity === "embed") - embedder.createEmbeddings.mockResolvedValueOnce({ embeddings: [[1]] }).mockRejectedValue(failure) - else store.upsertPoints.mockResolvedValueOnce(undefined).mockRejectedValue(failure) - const pending = watcher.processFile(fileA) - await vi.advanceTimersByTimeAsync(1500) - expect((await pending).status).toBe("local_error") - expect(target).toHaveBeenCalledTimes(4) - expect(failures.getRetryBudget("a.ts", hash("current content"), activity).retriesUsed).toBe(2) - await createWatcher().processFile(fileA) - expect(target).toHaveBeenCalledTimes(4) - }, - ) - - it.each([401, 400, 408, 409])("never retries permanent HTTP %s", async (status) => { - const { watcher, embedder, failures } = setup() - embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("permanent"), { status })) - expect((await watcher.processFile(fileA)).status).toBe("local_error") - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(failures.get("a.ts")).toMatchObject({ - activity: "embed", - state: "exhausted", - classification: "permanent", - }) + afterEach(async () => { + fileWatcher?.dispose() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() }) - it("retries a store request without reembedding and logs every grouped failure", async () => { - const { watcher, store, embedder, hashes, failures } = setup() - store.upsertPoints.mockRejectedValueOnce(Object.assign(new Error("temporary"), { status: 503 })) - await watcher.initialize() - await queue() - await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) - expect(store.upsertPoints).toHaveBeenCalledTimes(2) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(hashes.has(fileA)).toBe(true) - expect(failures.list()).toEqual([]) - const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value - expect(batch.record).toHaveBeenCalledOnce() - expect(batch.flush).toHaveBeenCalledOnce() + it("reuses constructor-created preparation for public processFile without writing points or cache", async () => { + const prepare = vi.spyOn(FilePreparation.prototype, "prepareFile") + try { + const preparation = fileWatcher["filePreparation"] + expect(preparation).toBeInstanceOf(FilePreparation) + expect(preparation["dependencies"].fileSystem).toBe(vscode.workspace.fs) + expect(preparation["dependencies"].cacheManager).toBe(mockCacheManager) + expect(preparation["dependencies"].ignoreController).toBe(fileWatcher["ignoreController"]) + const path = "/mock/workspace/src/file.ts" + const result = await fileWatcher.processFile(path) + expect(prepare).toHaveBeenNthCalledWith(1, path) + expect(result).toBe(await prepare.mock.results[0].value) + expect(result.status).toBe("processed_for_batching") + expect(result.blocks).toHaveLength(1) + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + expect(vscode.workspace.fs.stat).toHaveBeenCalledWith(vscode.Uri.file(path)) + expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(vscode.Uri.file(path)) + const secondPath = "/mock/workspace/src/second.ts" + const secondResult = await fileWatcher.processFile(secondPath) + expect(prepare).toHaveBeenCalledTimes(2) + expect(prepare).toHaveBeenNthCalledWith(2, secondPath) + expect(prepare.mock.contexts[0]).toBe(preparation) + expect(prepare.mock.contexts[1]).toBe(preparation) + expect(secondResult).toBe(await prepare.mock.results[1].value) + expect(secondResult.status).toBe("processed_for_batching") + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + expect(mockCacheManager.deleteHash).not.toHaveBeenCalled() + } finally { + prepare.mockRestore() + } }) - it.each(["create", "change", "delete"] as const)( - "filters untracked hidden paths on %s and processes eligible paths", - async (type) => { - const { watcher, parser, store, hashes } = setup() - await watcher.initialize() - await queue([path.join(workspacePath, ".hidden/file.ts")], type) - expect(parser.parseFile).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - if (type !== "delete") { - await queue([fileA], type) - expect(hashes.has(fileA)).toBe(true) + describe("file filtering", () => { + it("should ignore files in hidden directories on create events", async () => { + // Initialize the file watcher + await fileWatcher.initialize() + + const batchPromise = waitForNextBatch() + + // Simulate file creation events + const testCases = [ + { path: "/mock/workspace/src/file.ts", shouldProcess: true }, + { path: "/mock/workspace/.git/config", shouldProcess: false }, + { path: "/mock/workspace/.hidden/file.ts", shouldProcess: false }, + { path: "/mock/workspace/src/.next/static/file.js", shouldProcess: false }, + { path: "/mock/workspace/node_modules/package/index.js", shouldProcess: false }, + { path: "/mock/workspace/normal/file.js", shouldProcess: true }, + ] + + // Trigger file creation events + for (const { path } of testCases) { + await mockOnDidCreate({ fsPath: path }) } - }, - ) - - it.each(["delete", "ignore", "rooignore", "size", "extension"] as const)( - "removes tracked paths with a fenced generation on %s", - async (kind) => { - const { watcher, cache, hashes, store, ignored, access, parser } = setup() - const filePath = kind === "extension" ? path.join(workspacePath, "a.unsupported") : fileA - hashes.set(filePath, "previous") - if (kind === "ignore") ignored.add("a.ts") - if (kind === "rooignore") vi.mocked(access.validateAccess).mockReturnValue(false) - if (kind === "size") - vi.mocked(vscode.workspace.fs.stat).mockResolvedValue({ - size: MAX_FILE_SIZE_BYTES + 1, - type: 1, - mtime: 0, - ctime: 0, - }) - await watcher.initialize() - await queue([filePath], kind === "delete" ? "delete" : "change") - expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) - expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(filePath, expect.any(Number)) - expect(cache.completeRemoval).toHaveBeenCalledWith(filePath) - expect(parser.parseFile).not.toHaveBeenCalled() - expect(hashes.has(filePath)).toBe(false) - }, - ) - - it("removes a registry-only deleted source and retains failed removal until explicit retry", async () => { - const { watcher, failures, store } = setup() - failures.renew("a.ts", "unread") - failures.record("a.ts", "unread", "read", "read failed", 1, "exhausted") - store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) - await watcher.initialize() - await queue([fileA], "delete") - await queue([fileA], "delete") - expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() - failures.resetAllBudgets() - await queue([fileA], "delete") - expect(store.deletePointsByFilePath).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - }) - it.each(["success", "failure", "fatal"] as const)( - "suppresses stale parse %s continuations and events", - async (outcome) => { - const { watcher, parser, supersede, cache, failures } = setup() - const held = deferred() - parser.parseFile.mockImplementationOnce(() => held.promise) - const fatal = vi.fn() - const finished = vi.fn() - watcher.onParserFatal(fatal) - watcher.onDidFinishBatchProcessing(finished) - await watcher.initialize() - const pending = watcher.processFile(fileA) - const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) - await drain() - supersede() - if (outcome === "success") held.resolve(blocks(fileA)) - else held.reject(outcome === "fatal" ? new ParserFatalError("parser-abort", fileA) : new Error("late")) - await rejected - await queue([fileB]) - expect(parser.parseFile).toHaveBeenCalledOnce() - expect(cache.invalidateFile).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - expect(fatal).not.toHaveBeenCalled() - expect(finished).not.toHaveBeenCalled() - }, - ) - - it("propagates fatal identity immediately, stops queued dispatch and preserves prior data", async () => { - const { watcher, parser, hashes, store, failures } = setup() - const failure = new ParserFatalError("parse-timeout", fileA) - hashes.set(fileB, "completed") - parser.parseFile.mockRejectedValueOnce(failure) - const fatal = vi.fn() - watcher.onParserFatal(fatal) - await expect(watcher.processFile(fileA)).rejects.toBe(failure) - expect(fatal).toHaveBeenCalledExactlyOnceWith(failure) - expect(failures.get("a.ts")).toMatchObject({ activity: "parse", attempt: 1, state: "exhausted" }) - expect(hashes.get(fileB)).toBe("completed") - expect(store.clearCollection).not.toHaveBeenCalled() - await expect(watcher.processFile(fileB)).rejects.toMatchObject({ name: "AbortError" }) - }) + await flushBatch() + + const batchSummary = await batchPromise + const successPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "success") + .map((result: any) => result.path) + const skippedPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "skipped") + .map((result: any) => result.path) + + // Check that files in hidden directories were not processed + expect(successPaths).toContain("/mock/workspace/src/file.ts") + expect(successPaths).toContain("/mock/workspace/normal/file.js") + expect(skippedPaths).toContain("/mock/workspace/.git/config") + expect(skippedPaths).toContain("/mock/workspace/.hidden/file.ts") + expect(skippedPaths).toContain("/mock/workspace/src/.next/static/file.js") + expect(skippedPaths).toContain("/mock/workspace/node_modules/package/index.js") + }) - it.each(["dispose", "abort"] as const)( - "cancels the actual provider signal on %s and drops late results", - async (action) => { - const { watcher, embedder, store, failures, cache } = setup() - const held = deferred>>() - embedder.createEmbeddings.mockImplementationOnce(() => held.promise) - const owner = new AbortController() - await watcher.initialize(owner.signal) - const pending = watcher.processFile(fileA) - const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) - await drain() - const signal = embedder.createEmbeddings.mock.calls[0][2] - if (action === "dispose") watcher.dispose() - else owner.abort() - expect(signal?.aborted).toBe(true) - held.resolve({ embeddings: [[1]] }) - await rejected - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }, - ) - - it("tracks a late store result under the session owner and never completes an aborted file", async () => { - const { watcher, store, gate, cache, failures } = setup() - const held = deferred() - store.upsertPoints.mockImplementationOnce(() => held.promise) - const owner = new AbortController() - await watcher.initialize(owner.signal) - const pending = watcher.processFile(fileA) - const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) - await drain() - expect(store.upsertPoints).toHaveBeenCalledOnce() - expect(gate.hasPending(owner.signal)).toBe(true) - owner.abort() - held.resolve(undefined) - await rejected - await gate.settled(owner.signal) - expect(gate.hasPending(owner.signal)).toBe(false) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(store.countPointsForFile).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }) + it("should ignore files in hidden directories on change events", async () => { + // Initialize the file watcher + await fileWatcher.initialize() - it.each([ - { kind: "empty", parsed: [] }, - { kind: "whitespace", parsed: [{ ...blocks(fileA)[0], content: " \n\t " }] }, - ])("completes $kind revisions without embedding", async ({ parsed }) => { - const { watcher, parser, embedder, store, hashes } = setup() - parser.parseFile.mockResolvedValue(parsed) - expect((await watcher.processFile(fileA)).status).toBe("success") - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() - expect(store.countPointsForFile).toHaveBeenCalledOnce() - expect(hashes.get(fileA)).toBe(hash("current content")) - }) + const batchPromise = waitForNextBatch() - it.each(["abort", "dispose", "stale"] as const)( - "does not classify a session %s during delete as per-file supersession", - async (action) => { - const { watcher, hashes, store, failures, cache, supersede } = setup() - const held = deferred() - hashes.set(fileA, "previous") - store.deletePointsByFilePath.mockImplementationOnce(() => held.promise) - const owner = new AbortController() - await watcher.initialize(owner.signal) - const pending = watcher["removeFile"](fileA) - const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) - await drain() - if (action === "dispose") watcher.dispose() - else { - owner.abort() - if (action === "stale") supersede() - } - held.resolve(undefined) - await rejected - expect(cache.completeRemoval).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }, - ) - - it.each(["read", "parse", "embed", "delete", "excluded"] as const)( - "reports superseded %s work as skipped without a false batch error or completion", - async (stage) => { - const { watcher, parser, embedder, store, cache, hashes, openFiles, failures, ignored } = setup() - const held = deferred() - hashes.set(fileA, "previous") - if (stage === "read") - vi.mocked(vscode.workspace.fs.readFile).mockImplementationOnce(async () => { - await held.promise - return Buffer.from("current content") - }) - else if (stage === "parse") - parser.parseFile.mockImplementationOnce(async () => { - await held.promise - return blocks(fileA) - }) - else if (stage === "embed") - embedder.createEmbeddings.mockImplementationOnce(async () => { - await held.promise - return { embeddings: [[1]] } - }) - else { - if (stage === "excluded") ignored.add("a.ts") - store.deletePointsByFilePath.mockImplementationOnce(() => held.promise) + // Simulate file change events + const testCases = [ + { path: "/mock/workspace/src/file.ts", shouldProcess: true }, + { path: "/mock/workspace/.vscode/settings.json", shouldProcess: false }, + { path: "/mock/workspace/src/.cache/data.json", shouldProcess: false }, + { path: "/mock/workspace/dist/bundle.js", shouldProcess: false }, + ] + + // Trigger file change events + for (const { path } of testCases) { + await mockOnDidChange({ fsPath: path }) } - const finished = vi.fn<(summary: BatchProcessingSummary) => void>() - watcher.onDidFinishBatchProcessing(finished) - await watcher.initialize() - await queue([fileA], stage === "delete" ? "delete" : "change") - expect(finished).not.toHaveBeenCalled() - handlers.change(makeUri(fileA)) - held.resolve(undefined) - await drain() - expect(finished).toHaveBeenCalledExactlyOnceWith({ - processedFiles: [{ path: fileA, status: "skipped", reason: "File change was superseded" }], - batchError: undefined, + + await flushBatch() + + const batchSummary = await batchPromise + const successPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "success") + .map((result: any) => result.path) + const skippedPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "skipped") + .map((result: any) => result.path) + + // Check that files in hidden directories were not processed + expect(successPaths).toContain("/mock/workspace/src/file.ts") + expect(skippedPaths).toContain("/mock/workspace/.vscode/settings.json") + expect(skippedPaths).toContain("/mock/workspace/src/.cache/data.json") + expect(skippedPaths).toContain("/mock/workspace/dist/bundle.js") + }) + + it("should remove deleted files after the debounce window", async () => { + // Initialize the file watcher + await fileWatcher.initialize() + + const deletedFiles: string[] = [] + mockVectorStore.deletePointsByFilePath.mockImplementation(async (filePath: string) => { + deletedFiles.push(filePath) }) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(cache.completeRemoval).not.toHaveBeenCalled() - if (stage === "embed" || stage === "delete" || stage === "excluded") { - expect(openFiles.has(fileA)).toBe(true) - expect(hashes.has(fileA)).toBe(false) + const batchPromise = waitForNextBatch() + + // Simulate file deletion events + const testCases = [ + { path: "/mock/workspace/src/file.ts", shouldProcess: true }, + { path: "/mock/workspace/.git/objects/abc123", shouldProcess: false }, + { path: "/mock/workspace/.DS_Store", shouldProcess: false }, + { path: "/mock/workspace/build/.cache/temp.js", shouldProcess: false }, + ] + + // Trigger file deletion events + for (const { path } of testCases) { + await mockOnDidDelete({ fsPath: path }) } - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) - expect(finished).toHaveBeenCalledTimes(2) - expect(finished.mock.calls[1][0].batchError).toBeUndefined() - expect(finished.mock.calls[1][0].processedFiles).toEqual([{ path: fileA, status: "success" }]) - }, - ) - - it("keeps parser cancellation scoped to the session when a file change is superseded", async () => { - const { watcher, parser, cache } = setup() - await watcher.initialize() - const held = deferred() - parser.parseFile.mockReturnValueOnce(held.promise) - const pending = watcher.processFile(fileA) - await drain() - const parseSignal = parser.parseFile.mock.calls[0][1]?.signal - expect(parseSignal).toBeDefined() - handlers.change(makeUri(fileA)) - expect(parseSignal?.aborted).toBe(false) - held.resolve(blocks(fileA)) - await expect(pending).resolves.toMatchObject({ status: "skipped" }) - expect(cache.completeFile).not.toHaveBeenCalled() - await expect(watcher.processFile(fileA)).resolves.toMatchObject({ status: "success" }) - watcher.dispose() - expect(parseSignal?.aborted).toBe(true) - }) - it("preserves a fatal parser failure after the active file change was superseded", async () => { - const { watcher, parser } = setup() - await watcher.initialize() - const reported = vi.fn() - watcher.onParserFatal(reported) - const held = deferred() - const fatal = new ParserFatalError("parse-timeout", fileA) - parser.parseFile.mockReturnValueOnce(held.promise) - const pending = watcher.processFile(fileA).catch((error: unknown) => error) - await drain() - handlers.change(makeUri(fileA)) - held.reject(fatal) - expect(await pending).toBe(fatal) - expect(reported).toHaveBeenCalledExactlyOnceWith(fatal) - }) + await flushBatch() + await batchPromise - it("preserves a non-superseded public AbortError instead of treating it as skipped", async () => { - const { watcher, parser, failures } = setup() - const failure = new DOMException("parser cancelled", "AbortError") - parser.parseFile.mockRejectedValueOnce(failure) - await expect(watcher.processFile(fileA)).rejects.toBe(failure) - expect(failures.list()).toEqual([]) - }) + expect(deletedFiles).toEqual(testCases.map(({ path }) => path)) + }) - it("cancels a long retry wait on revision change without renewing another file", async () => { - const { watcher, embedder, failures, hashes } = setup() - embedder.createEmbeddings.mockRejectedValueOnce( - Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 }), - ) - await watcher.initialize() - await queue() - const oldSignal = embedder.createEmbeddings.mock.calls[0][2] - failures.renew("b.ts", "other") - const otherBudget = failures.getRetryBudget("b.ts", "other", "embed") - otherBudget.retriesUsed = 2 - vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("new revision")) - handlers.change(makeUri(fileA)) - expect(oldSignal?.aborted).toBe(true) - await vi.advanceTimersByTimeAsync(1000) - // The shared provider pause outlives the cancelled revision. - expect(hashes.get(fileA)).toBeUndefined() - await vi.advanceTimersByTimeAsync(28_000) - expect(hashes.get(fileA)).toBeUndefined() - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(2000) - expect(hashes.get(fileA)).toBe(hash("new revision")) - expect(failures.getRetryBudget("b.ts", "other", "embed")).toBe(otherBudget) - await vi.advanceTimersByTimeAsync(60_000) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) - }) + it("should handle nested hidden directories correctly", async () => { + // Initialize the file watcher + await fileWatcher.initialize() - it("cancels an active revision on delete before generation-scoped removal", async () => { - const { watcher, embedder, hashes, store, failures } = setup() - embedder.createEmbeddings.mockRejectedValueOnce(Object.assign(new Error("temporary"), { status: 503 })) - await watcher.initialize() - await queue() - const oldSignal = embedder.createEmbeddings.mock.calls[0][2] - handlers.delete(makeUri(fileA)) - expect(oldSignal?.aborted).toBe(true) - await vi.advanceTimersByTimeAsync(1000) - expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() - expect(hashes.has(fileA)).toBe(false) - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync(60_000) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - }) + const batchPromise = waitForNextBatch() - it("keeps verification mismatch and uncertain unchanged content incomplete", async () => { - const { watcher, hashes, gate, store, parser, cache } = setup() - hashes.set(fileA, hash("current content")) - expect((await watcher.processFile(fileA)).status).toBe("skipped") - gate.recover([fileA]) - store.countPointsForFile.mockResolvedValue({ current: 0, other: 1 }) - expect((await watcher.processFile(fileA)).status).toBe("local_error") - expect(parser.parseFile).toHaveBeenCalledOnce() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(hashes.has(fileA)).toBe(false) - }) + // Test deeply nested hidden directories + const testCases = [ + { path: "/mock/workspace/src/components/Button.tsx", shouldProcess: true }, + { path: "/mock/workspace/src/.hidden/components/Button.tsx", shouldProcess: false }, + { path: "/mock/workspace/.hidden/src/components/Button.tsx", shouldProcess: false }, + { path: "/mock/workspace/src/components/.hidden/Button.tsx", shouldProcess: false }, + ] - it("bounds dispatch while allowing an unrelated file to finish independently", async () => { - const { watcher, parser, hashes } = setup() - const held = deferred() - parser.parseFile.mockImplementation(async (filePath) => (filePath === fileA ? blocks(fileA) : held.promise)) - await watcher.initialize() - const paths = [ - fileA, - ...Array.from({ length: 11 }, (_, fileNumber) => path.join(workspacePath, `${fileNumber}.ts`)), - ] - await queue(paths) - expect(hashes.has(fileA)).toBe(true) - expect(parser.parseFile).toHaveBeenCalledTimes(11) - watcher.dispose() - held.resolve([]) - await drain() - expect(parser.parseFile).toHaveBeenCalledTimes(11) - }) -}) + // Trigger file creation events + for (const { path } of testCases) { + await mockOnDidCreate({ fsPath: path }) + } -describe("T43 watcher source missing at read is a removal", () => { - const errorWithCode = (code: string) => Object.assign(new Error("gone"), { code }) - const failAt = (stage: "stat" | "read", error: Error) => { - if (stage === "stat") vi.mocked(vscode.workspace.fs.stat).mockRejectedValueOnce(error) - else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(error) - } - const pointsFor = (points: Map, filePath: string) => - [...points.values()].filter((point) => { - const payload: Record = point.payload - return payload.filePath === path.relative(workspacePath, filePath) + await flushBatch() + + const batchSummary = await batchPromise + const successPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "success") + .map((result: any) => result.path) + const skippedPaths = batchSummary.processedFiles + .filter((result: any) => result.status === "skipped") + .map((result: any) => result.path) + + // Check that files in hidden directories were not processed + expect(successPaths).toContain("/mock/workspace/src/components/Button.tsx") + expect(skippedPaths).toContain("/mock/workspace/src/.hidden/components/Button.tsx") + expect(skippedPaths).toContain("/mock/workspace/.hidden/src/components/Button.tsx") + expect(skippedPaths).toContain("/mock/workspace/src/components/.hidden/Button.tsx") }) - const fileStat = (size: number): vscode.FileStat => ({ size, type: 1, ctime: 0, mtime: 0 }) - - it.each(["stat", "read"] as const)("never-indexed file missing at %s is skipped", async (stage) => { - const { watcher, store, failures } = setup() - failAt(stage, errorWithCode("FileNotFound")) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "skipped" }) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }) - - it.each(["stat", "read"] as const)("previously indexed file missing at %s is removed", async (stage) => { - const { watcher, points, hashes, failures } = setup() - expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) - expect(pointsFor(points, fileA)).not.toEqual([]) - failAt(stage, errorWithCode(stage === "stat" ? "FileNotFound" : "ENOENT")) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) - expect(pointsFor(points, fileA)).toEqual([]) - expect(hashes.get(fileA)).toBeUndefined() - expect(failures.list()).toEqual([]) }) - it("clears a prior failure for a path found missing", async () => { - const { watcher, parser, failures } = setup() - parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) - await watcher.processFile(fileA) - expect(failures.list()).toHaveLength(1) - failAt("stat", errorWithCode("FileNotFound")) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) - expect(failures.list()).toEqual([]) - }) + describe("failures and removals", () => { + const path = "/mock/workspace/src/file.ts" + + function watcherWithRegistry() { + const failures = new FileFailureRegistry() + fileWatcher.dispose() + fileWatcher = new FileWatcher( + "/mock/workspace", + mockContext, + mockCacheManager, + mockEmbedder, + mockVectorStore, + mockIgnoreInstance, + undefined, + undefined, + failures, + ) + return failures + } + + async function runChange() { + await fileWatcher.initialize() + const batchPromise = waitForNextBatch() + await mockOnDidChange({ fsPath: path }) + await flushBatch() + return batchPromise + } + + it("CI-FW-01: a missing result routes to removeFile", async () => { + const failures = watcherWithRegistry() + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ path, status: "missing" }) + const summary = await runChange() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith(path) + expect(mockCacheManager.deleteHash).toHaveBeenCalledWith(path) + expect(summary.processedFiles).toEqual([{ path, status: "success" }]) + expect(failures.list()).toEqual([]) + }) - it("keeps a cleanup failure as a delete failure", async () => { - const { watcher, store, failures } = setup() - await watcher.processFile(fileA) - failAt("read", errorWithCode("FileNotFound")) - store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) - expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "delete" })]) - }) + it.each(["read", "parse"] as const)( + "CI-FW-02: a local_error is recorded with the result's activity %s", + async (activity) => { + const failures = watcherWithRegistry() + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ + path, + status: "local_error", + activity, + error: new Error("bad"), + }) + await runChange() + expect(failures.list()).toEqual([expect.objectContaining({ path: "src/file.ts", activity, reason: "bad" })]) + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) - it.each([ - ["stat", "NoPermissions"], - ["read", "EACCES"], - ["read", "Unknown"], - ] as const)("keeps denied access at %s (%s) as a read failure", async (stage, code) => { - const { watcher, failures } = setup() - failAt(stage, errorWithCode(code)) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) - expect(failures.list()).toEqual([ - expect.objectContaining({ path: "a.ts", activity: "read", state: "exhausted" }), - ]) - }) + it("CI-FW-02: prepared blocks are passed to the writer, which embeds and stores the hash", async () => { + watcherWithRegistry() + const blocks = [ + { + file_path: path, + identifier: null, + type: "function", + start_line: 1, + end_line: 2, + content: "code", + fileHash: "h", + segmentHash: "s", + }, + ] + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ + path, + status: "processed_for_batching", + newHash: "h", + blocks, + }) + const summary = await runChange() + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledWith(["code"]) + expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1) + expect(mockCacheManager.updateHash).toHaveBeenCalledWith(path, "h") + expect(summary.processedFiles).toEqual([{ path, status: "success" }]) + }) - it("keeps a not-found error from setup, outside stat and read, as a failure", async () => { - const { watcher, access, failures } = setup() - vi.mocked(access.validateAccess).mockImplementationOnce(() => { - throw errorWithCode("ENOENT") + it("writes nothing after the session signal aborts", async () => { + watcherWithRegistry() + const controller = new AbortController() + await fileWatcher.initialize(controller.signal) + controller.abort() + await mockOnDidChange({ fsPath: path }) + await flushBatch() + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() }) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "local_error" }) - expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "read" })]) }) - it.each( - (["stat", "read"] as const).flatMap((stage) => - (["new", "indexed", "failed"] as const).flatMap((history) => - (["during", "after"] as const).map((timing) => ({ stage, history, timing })), - ), - ), - )( - "removes $history source missing at $stage $timing a concurrent provider wait", - async ({ stage, history, timing }) => { - const { watcher, points, hashes, store, embedder, failures, pacers } = setup() - const finished = vi.fn<(summary: BatchProcessingSummary) => void>() - watcher.onDidFinishBatchProcessing(finished) - await watcher.initialize() - if (history === "indexed") await queue() - if (history === "failed") { - points.set("obsolete", { id: "obsolete", vector: [0], payload: { filePath: "a.ts", generation: 0 } }) - failures.renew("a.ts", "old") - failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") - } - const metadata = deferred() - vi.mocked(vscode.workspace.fs.stat).mockImplementation(async (uri) => - uri.fsPath === fileA ? metadata.promise : fileStat(100), - ) - const readFile = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! - const missingRead = vi.fn() - vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => { - if (uri.fsPath === fileA) { - missingRead() - throw errorWithCode("FileNotFound") - } - return readFile(uri) - }) - embedder.createEmbeddings - .mockClear() - .mockRejectedValueOnce(Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 })) - finished.mockClear() - await queue([fileA, fileB]) - expect(pacers.waitingPaths()).toContain("b.ts") - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(missingRead).not.toHaveBeenCalled() - expect(finished).not.toHaveBeenCalled() - const resumeAt = pacers.providerWaits()[0].resumeAt! - await vi.advanceTimersByTimeAsync(resumeAt - Date.now() - (timing === "during" ? 1 : 0)) - if (stage === "stat") metadata.reject(errorWithCode("FileNotFound")) - else metadata.resolve(fileStat(100)) - await drain() - expect(missingRead).toHaveBeenCalledTimes(stage === "read" ? 1 : 0) - if (timing === "during") { - expect(pacers.waitingPaths()).toContain("b.ts") - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(finished).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - } - await drain() - expect(finished).toHaveBeenCalledExactlyOnceWith({ - processedFiles: expect.arrayContaining([ - { - path: fileA, - status: history === "new" ? "skipped" : "success", - ...(history === "new" ? { reason: "File is not tracked" } : {}), - }, - { path: fileB, status: "success" }, - ]), - batchError: undefined, - }) - expect(pointsFor(points, fileA)).toEqual([]) - expect(hashes.get(fileA)).toBeUndefined() - expect(pointsFor(points, fileB)).toHaveLength(1) - expect(hashes.get(fileB)).toBe(hash("current content")) - expect(failures.list()).toEqual([]) - expect(pacers.waitingPaths()).toEqual([]) - expect(watcher.isProcessing).toBe(false) - if (history === "new") - expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) - else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) - }, - ) - - it("indexes a recreated file normally", async () => { - const { watcher, points, hashes } = setup() - await watcher.processFile(fileA) - failAt("read", errorWithCode("FileNotFound")) - await watcher.processFile(fileA) - expect(pointsFor(points, fileA)).toEqual([]) - expect(await watcher.processFile(fileA)).toMatchObject({ status: "success" }) - expect(pointsFor(points, fileA)).not.toEqual([]) - expect(hashes.get(fileA)).toBe(hash("current content")) + describe("dispose", () => { + it("should dispose of the watcher when disposed", async () => { + await fileWatcher.initialize() + fileWatcher.dispose() + + expect(mockWatcher.dispose).toHaveBeenCalled() + }) }) }) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 2ed46b50f1..89be4e7495 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -1,6 +1,6 @@ // npx vitest services/code-index/processors/__tests__/parser.spec.ts -import { CodeParser } from "../parser" +import { CodeParser, codeParser, ParserLoadError } from "../parser" import { loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser" import { parseMarkdown } from "../../../tree-sitter/markdownParser" import { readFile } from "fs/promises" @@ -8,8 +8,13 @@ import { Node } from "web-tree-sitter" import { clearAllMocks } from "../../../../test-utils/reset" -vi.mock("../../code-index-log", () => ({ - codeIndexLog: { error: vi.fn(), warn: vi.fn() }, +// Mock TelemetryService +vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), + }, + }, })) // Override Jest-based fs/promises mock with vitest-compatible version @@ -36,7 +41,6 @@ const mockLanguageParser = { js: { parser: { parse: vi.fn((content: string) => ({ - delete: vi.fn(), rootNode: { text: content, startPosition: { row: 0 }, @@ -161,16 +165,17 @@ describe("CodeParser", () => { expect(result).toBeDefined() }) - it("should handle parser load errors", async () => { - ;(loadRequiredLanguageParsers as any).mockRejectedValue(new Error("Load failed")) - const result = await parser["parseContent"]("test.js", "const test = 123", "hash") - expect(result).toEqual([]) - }) - - it("should return empty array when no parser is available", async () => { - ;(loadRequiredLanguageParsers as any).mockResolvedValue({} as any) + it.each([ + ["a rejected load", () => vi.mocked(loadRequiredLanguageParsers).mockRejectedValueOnce(new Error("Load failed"))], + ["no parser for the extension", () => vi.mocked(loadRequiredLanguageParsers).mockResolvedValueOnce({})], + ])("CI-PA-01: %s throws ParserLoadError and the next call reloads", async (_label, failLoad) => { + failLoad() + await expect(parser["parseContent"]("test.js", "const test = 123", "hash")).rejects.toBeInstanceOf( + ParserLoadError, + ) const result = await parser["parseContent"]("test.js", "const test = 123", "hash") - expect(result).toEqual([]) + expect(Array.isArray(result)).toBe(true) + expect(loadRequiredLanguageParsers).toHaveBeenCalledTimes(2) }) }) @@ -312,6 +317,15 @@ describe("CodeParser", () => { }) }) + describe("singleton instance", () => { + it("should maintain parser state across calls", async () => { + const result1 = await codeParser.parseFile("test.js", { content: "const a = 1" }) + const result2 = await codeParser.parseFile("test.js", { content: "const b = 2" }) + expect(result1).toBeDefined() + expect(result2).toBeDefined() + }) + }) + describe("Markdown Support", () => { beforeEach(() => { clearAllMocks() diff --git a/src/services/code-index/processors/__tests__/parser.txt.spec.ts b/src/services/code-index/processors/__tests__/parser.txt.spec.ts index eca53e8351..0fa9526916 100644 --- a/src/services/code-index/processors/__tests__/parser.txt.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.txt.spec.ts @@ -1,6 +1,5 @@ import { CodeParser } from "../parser" -import { scannerExtensions } from "../../shared/supported-extensions" -import { isFallbackExtension as shouldUseFallbackChunking } from "../../../shared/fallback-extensions" +import { scannerExtensions, shouldUseFallbackChunking } from "../../shared/supported-extensions" vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ TelemetryService: { diff --git a/src/services/code-index/processors/__tests__/parser.vb.spec.ts b/src/services/code-index/processors/__tests__/parser.vb.spec.ts index 9124a8464e..95fd73cc9a 100644 --- a/src/services/code-index/processors/__tests__/parser.vb.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.vb.spec.ts @@ -9,7 +9,7 @@ vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ }, })) -import { isFallbackExtension as shouldUseFallbackChunking } from "../../../shared/fallback-extensions" +import { shouldUseFallbackChunking } from "../../shared/supported-extensions" describe("CodeParser - VB.NET and Fallback Extensions Support", () => { let parser: CodeParser diff --git a/src/services/code-index/processors/__tests__/processor-factories.spec.ts b/src/services/code-index/processors/__tests__/processor-factories.spec.ts index 1b222fb997..a3aaf211b9 100644 --- a/src/services/code-index/processors/__tests__/processor-factories.spec.ts +++ b/src/services/code-index/processors/__tests__/processor-factories.spec.ts @@ -18,6 +18,8 @@ import { getEmbeddingBatchSize } from "../get-embedding-batch-size" import { codeParser } from "../parser" import { DirectoryScanner } from "../scanner" import { FileWatcher } from "../file-watcher" +import { FileFailureRegistry } from "../../file-failure-registry" +import { ProviderPacerRegistry } from "../../provider-pacer" vi.mock("vscode", () => ({ workspace: { getConfiguration: vi.fn(() => ({ get: vi.fn() })) }, @@ -44,6 +46,8 @@ describe("processor factories", () => { vectorStore: new QdrantVectorStore("/workspace", "http://localhost:6333", 512), parser: codeParser, ignoreInstance: ignore(), + failureRegistry: new FileFailureRegistry(), + pacers: new ProviderPacerRegistry(), } } @@ -67,6 +71,8 @@ describe("processor factories", () => { options.cacheManager, options.ignoreInstance, batchSize, + options.failureRegistry, + options.pacers, ) expect(FileWatcher).toHaveBeenCalledWith( options.workspacePath, @@ -77,6 +83,8 @@ describe("processor factories", () => { options.ignoreInstance, undefined, batchSize, + options.failureRegistry, + options.pacers, ) expect(vscode.workspace.getConfiguration).toHaveBeenCalledWith(Package.name) expect(configuration.get).toHaveBeenCalledWith("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) @@ -148,13 +156,19 @@ describe("processor factories", () => { vi.mocked(configManager.getConfig).mockReturnValue(config) const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create").mockReturnValue(options.embedder) const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create").mockReturnValue(options.vectorStore) - const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) + const factory = new CodeIndexServiceFactory( + configManager, + options.workspacePath, + options.cacheManager, + options.failureRegistry, + options.pacers, + ) const services = factory.createServices(options.context, watcherCache, options.ignoreInstance) expect(configManager.getConfig).toHaveBeenCalledTimes(1) - expect(createEmbedder).toHaveBeenCalledExactlyOnceWith(config) - expect(createVectorStore).toHaveBeenCalledExactlyOnceWith(config, options.workspacePath) + expect(createEmbedder).toHaveBeenCalledExactlyOnceWith(config, options.pacers) + expect(createVectorStore).toHaveBeenCalledExactlyOnceWith(config, options.workspacePath, options.pacers) expect(services).toEqual({ embedder: options.embedder, vectorStore: options.vectorStore, @@ -165,6 +179,8 @@ describe("processor factories", () => { expect(vi.mocked(DirectoryScanner).mock.calls[0][3]).toBe(options.cacheManager) expect(vi.mocked(FileWatcher).mock.calls[0][2]).toBe(watcherCache) expect(vi.mocked(FileWatcher).mock.calls[0][0]).toBe(options.workspacePath) + expect(vi.mocked(DirectoryScanner).mock.calls[0].slice(6)).toEqual([options.failureRegistry, options.pacers]) + expect(vi.mocked(FileWatcher).mock.calls[0].slice(8)).toEqual([options.failureRegistry, options.pacers]) }) it("does not create services when indexing is not configured", () => { diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 5705460092..6275c031d6 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -1,792 +1,582 @@ -import * as path from "path" -import * as vscode from "vscode" -import { Stats } from "fs" -import { stat } from "fs/promises" -import { createHash } from "crypto" -import ignore from "ignore" +// npx vitest services/code-index/processors/__tests__/scanner.spec.ts + import { DirectoryScanner } from "../scanner" -import type { CacheManager } from "../../cache-manager" -import type { CodeBlock, ICodeParser, IEmbedder, IVectorStore, PointStruct } from "../../interfaces" +import { ParserLoadError } from "../parser" import { FileFailureRegistry } from "../../file-failure-registry" -import { GenerationAllocator } from "../../generation-allocator" -import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" -import { ParserFatalError } from "../../parser-errors" -import { codeIndexLog } from "../../code-index-log" -import { listFiles } from "../../../glob/list-files" -import { generateRelativeFilePath } from "../../shared/get-relative-path" -import { PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, MAX_FILE_SIZE_BYTES } from "../../constants" - -import { clearAllMocks, restoreGlobals } from "../../../../test-utils/reset" -import { ProviderPacerRegistry } from "../../provider-pacer" - -vi.mock("fs/promises", () => ({ stat: vi.fn() })) +import { stat } from "fs/promises" +import * as vscode from "vscode" -// Stats has a private constructor in @types/node; the scanner reads only `size`. -const statsOfSize = (size: number): Stats => Object.assign(Object.create(Stats.prototype) as Stats, { size }) -vi.mock("vscode", () => ({ - workspace: { fs: { readFile: vi.fn() } }, - Uri: { file: (filePath: string) => ({ fsPath: filePath }) }, -})) -vi.mock("../../../../utils/path", () => ({ getWorkspacePathForContext: (directory: string) => directory })) -vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: class { - async initialize() {} - filterPaths(paths: string[]) { - return paths - } +// Mock TelemetryService +vi.mock("../../../../../packages/telemetry/src/TelemetryService", () => ({ + TelemetryService: { + instance: { + captureEvent: vi.fn(), + }, }, })) -vi.mock("../../../glob/list-files", () => ({ listFiles: vi.fn() })) -vi.mock("../../code-index-log", () => ({ - codeIndexLog: { - redact: (message: string) => message, - createFailureBatch: vi.fn(() => ({ record: vi.fn(), flush: vi.fn() })), - info: vi.fn(), - debug: vi.fn(), - warn: vi.fn(), + +vi.mock("fs/promises", () => ({ + default: { + readFile: vi.fn(), + writeFile: vi.fn(), + mkdir: vi.fn(), + access: vi.fn(), + rename: vi.fn(), + constants: {}, }, + stat: vi.fn(), })) -const workspacePath = path.resolve("scanner-workspace") -const fileA = path.join(workspacePath, "a.ts") -const fileB = path.join(workspacePath, "b.ts") -const hash = (content: string) => createHash("sha256").update(content).digest("hex") - -function deferred() { - let resolve!: (value: Value) => void - let reject!: (error: Error) => void - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - return { promise, resolve, reject } -} - -function blocks(filePath: string, count = 1): CodeBlock[] { - return Array.from({ length: count }, (_, blockNumber) => ({ - file_path: filePath, - content: `function block${blockNumber}() { return ${blockNumber} }`, - start_line: blockNumber + 1, - end_line: blockNumber + 1, - identifier: null, - type: "function", - fileHash: hash("current content"), - segmentHash: `${filePath}:${blockNumber}`, - })) -} - -function setup() { - const hashes = new Map() - const openFiles = new Set() - const points = new Map() - const cache = { - generationWatermark: 1024, - reserveGenerations: vi.fn().mockResolvedValue(undefined), - getHash: vi.fn((filePath: string) => hashes.get(filePath)), - getTrackedPaths: vi.fn(() => [...new Set([...hashes.keys(), ...openFiles])]), - invalidateFile: vi.fn(async (filePath: string) => { - hashes.delete(filePath) - openFiles.add(filePath) - }), - completeFile: vi.fn((filePath: string, revision: string) => { - hashes.set(filePath, revision) - openFiles.delete(filePath) - }), - completeRemoval: vi.fn((filePath: string) => { - hashes.delete(filePath) - openFiles.delete(filePath) +// Create a simple mock for vscode since we can't access the real one +vi.mock("vscode", () => ({ + workspace: { + workspaceFolders: [ + { + uri: { + fsPath: "/mock/workspace", + }, + }, + ], + getWorkspaceFolder: vi.fn().mockReturnValue({ + uri: { + fsPath: "/mock/workspace", + }, }), - } - const store = { - rateLimitScope: "qdrant:scanner-test", - initialize: vi.fn().mockResolvedValue(false), - upsertPoints: vi.fn().mockImplementation(async (batch) => { - for (const point of batch) points.set(point.id, point) + fs: { + readFile: vi.fn().mockResolvedValue(Buffer.from("test content")), + }, + }, + Uri: { + file: vi.fn().mockImplementation((path) => { + return path }), - deletePointsByFilePath: vi - .fn() - .mockImplementation(async (filePath, cutoff) => { - for (const point of points.values()) { - const payload: Record = point.payload - if ( - payload.filePath === generateRelativeFilePath(filePath, workspacePath) && - (typeof payload.generation !== "number" || payload.generation < cutoff) - ) - points.delete(point.id) - } - }), - deletePointsByMultipleFilePaths: vi - .fn() - .mockResolvedValue(undefined), - countPointsForFile: vi - .fn() - .mockImplementation(async (filePath, generation) => { - const selected = [...points.values()] - .map((point): Record => point.payload) - .filter((payload) => payload.filePath === generateRelativeFilePath(filePath, workspacePath)) - return { - current: selected.filter((payload) => payload.generation === generation).length, - other: selected.filter((payload) => payload.generation !== generation).length, - } - }), - search: vi.fn().mockResolvedValue([]), - clearCollection: vi.fn().mockResolvedValue(undefined), - deleteCollection: vi.fn().mockResolvedValue(undefined), - collectionExists: vi.fn().mockResolvedValue(true), - hasIndexedData: vi.fn().mockResolvedValue(true), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - } satisfies IVectorStore - const embedder = { - rateLimitScope: "openai:scanner-test", - createEmbeddings: vi - .fn() - .mockImplementation(async (texts) => ({ embeddings: texts.map(() => [1]) })), - validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), - embedderInfo: { name: "openai" }, - } satisfies IEmbedder - const parser = { - parseFile: vi.fn().mockImplementation(async (filePath) => blocks(filePath)), - } - const failures = new FileFailureRegistry() - const gate = new IndexWriteGate() - // The double supplies only the cache lookup, generation reservation, and completion methods used by scanner/writer. - const cacheManager = cache as unknown as CacheManager - const generations = new GenerationAllocator(cacheManager) - const ignored = ignore() - const pacers = new ProviderPacerRegistry() - let currentAttempt = true - const createScanner = () => - new DirectoryScanner( - embedder, - store, - parser, - cacheManager, - ignored, - 1, - gate, - generations, - failures, - workspacePath, - () => currentAttempt, - pacers, - ) - return { - cache, - hashes, - openFiles, - points, - store, - embedder, - parser, - failures, - gate, - ignored, - pacers, - createScanner, - scanner: createScanner(), - supersede: () => { - currentAttempt = false + }, + window: { + activeTextEditor: { + document: { + uri: { + fsPath: "/mock/workspace", + }, + }, }, - } -} - -beforeEach(() => { - vi.useFakeTimers() - clearAllMocks() - vi.mocked(stat).mockReset().mockResolvedValue(statsOfSize(100)) - vi.mocked(listFiles) - .mockReset() - .mockResolvedValue([[fileA], false]) - vi.mocked(vscode.workspace.fs.readFile).mockReset().mockResolvedValue(Buffer.from("current content")) -}) -afterEach(() => { - restoreGlobals() - vi.useRealTimers() -}) + }, +})) + +vi.mock("../../../../core/ignore/RooIgnoreController") +vi.mock("ignore") + +// Override the Jest-based mock with a vitest-compatible version +vi.mock("../../../glob/list-files", () => ({ + listFiles: vi.fn(), +})) describe("DirectoryScanner", () => { - it("preserves callback positions and completes only after all batches and verification", async () => { - const { scanner, parser, cache, store, points } = setup() - parser.parseFile.mockResolvedValue(blocks(fileA, 3)) - const held = deferred() - store.upsertPoints.mockImplementationOnce(async (batch) => { - await held.promise - for (const point of batch) points.set(point.id, point) - }) - const onError = vi.fn() - const onIndexed = vi.fn() - const onParsed = vi.fn() - const pending = scanner.scanDirectory(workspacePath, onError, onIndexed, onParsed) - await vi.advanceTimersByTimeAsync(0) - expect(onParsed).toHaveBeenCalledExactlyOnceWith(3) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(onIndexed).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - held.resolve(undefined) - expect(await pending).toEqual({ stats: { processed: 1, skipped: 0 }, totalBlockCount: 3, incompleteFiles: [] }) - expect(onIndexed).toHaveBeenCalledExactlyOnceWith(3) - expect(onError).not.toHaveBeenCalled() - expect(store.upsertPoints).toHaveBeenCalledTimes(3) - expect(new Set([...points.keys()]).size).toBe(3) - expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileA, hash("current content")) - expect(cache.invalidateFile.mock.invocationCallOrder[0]).toBeLessThan( - store.upsertPoints.mock.invocationCallOrder[0], - ) - expect(store.countPointsForFile.mock.invocationCallOrder[0]).toBeLessThan( - cache.completeFile.mock.invocationCallOrder[0], + let scanner: DirectoryScanner + let mockEmbedder: any + let mockVectorStore: any + let mockCodeParser: any + let mockCacheManager: any + let mockIgnoreInstance: any + let mockStats: any + + beforeEach(async () => { + mockEmbedder = { + createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), + embedderInfo: { name: "mock-embedder", dimensions: 384 }, + } + mockVectorStore = { + upsertPoints: vi.fn().mockResolvedValue(undefined), + deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), + deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(true), + search: vi.fn().mockResolvedValue([]), + clearCollection: vi.fn().mockResolvedValue(undefined), + deleteCollection: vi.fn().mockResolvedValue(undefined), + collectionExists: vi.fn().mockResolvedValue(true), + } + mockCodeParser = { + parseFile: vi.fn().mockResolvedValue([]), + } + mockCacheManager = { + getHash: vi.fn().mockReturnValue(undefined), + getAllHashes: vi.fn().mockReturnValue({}), + updateHash: vi.fn().mockResolvedValue(undefined), + deleteHash: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + clearCacheFile: vi.fn().mockResolvedValue(undefined), + } + mockIgnoreInstance = { + ignores: vi.fn().mockReturnValue(false), + } + + scanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, ) - expect(scanner.hasIncompleteFiles).toBe(false) + + // Mock default implementations - create proper Stats object + mockStats = { + size: 1024, + isFile: () => true, + isDirectory: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isSymbolicLink: () => false, + isFIFO: () => false, + isSocket: () => false, + dev: 0, + ino: 0, + mode: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blksize: 0, + blocks: 0, + atimeMs: 0, + mtimeMs: 0, + ctimeMs: 0, + birthtimeMs: 0, + atime: new Date(), + mtime: new Date(), + ctime: new Date(), + birthtime: new Date(), + atimeNs: BigInt(0), + mtimeNs: BigInt(0), + ctimeNs: BigInt(0), + birthtimeNs: BigInt(0), + } + vi.mocked(stat).mockResolvedValue(mockStats) + + // Get and mock the listFiles function + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) }) - it.each(["read", "parse"] as const)( - "records a single %s failure while unrelated files finish", - async (activity) => { - const { scanner, parser, failures, cache } = setup() - vi.mocked(listFiles).mockResolvedValue([[fileA, fileB], false]) - if (activity === "read") vi.mocked(stat).mockRejectedValueOnce(new Error("read failed")) - else parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) - const onError = vi.fn() - const result = await scanner.scanDirectory(workspacePath, onError) - expect(result.incompleteFiles).toEqual([fileA]) - expect(onError).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([ - expect.objectContaining({ path: "a.ts", activity, attempt: 1, state: "exhausted" }), + describe("scanDirectory", () => { + it("should skip files larger than MAX_FILE_SIZE_BYTES", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) + + // Create large file mock stats + const largeFileStats = { + ...mockStats, + size: 2 * 1024 * 1024, // 2MB > 1MB limit + } + vi.mocked(stat).mockResolvedValueOnce(largeFileStats) + + const result = await scanner.scanDirectory("/test") + expect(result.stats.skipped).toBe(1) + expect(mockCodeParser.parseFile).not.toHaveBeenCalled() + }) + + it("should parse changed files and return empty codeBlocks array", async () => { + // Create scanner without embedder to test the non-embedding path + const scannerNoEmbeddings = new DirectoryScanner( + null as any, // No embedder + null as any, // No vector store + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + ) + + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) + const mockBlocks: any[] = [ + { + file_path: "test/file1.js", + content: "test content", + start_line: 1, + end_line: 5, + identifier: "test", + type: "function", + fileHash: "hash", + segmentHash: "segment-hash", + }, + ] + ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) + + const result = await scannerNoEmbeddings.scanDirectory("/test") + expect(result.stats.processed).toBe(1) + }) + + it("should process embeddings for new/changed files", async () => { + const mockBlocks: any[] = [ + { + file_path: "test/file1.js", + content: "test content", + start_line: 1, + end_line: 5, + identifier: "test", + type: "function", + fileHash: "hash", + segmentHash: "segment-hash", + }, + ] + ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) + + await scanner.scanDirectory("/test") + expect(mockEmbedder.createEmbeddings).toHaveBeenCalled() + expect(mockVectorStore.upsertPoints).toHaveBeenCalled() + }) + + it("should clean up a threshold-triggered batch", async () => { + const thresholdScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + 1, + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) + mockCodeParser.parseFile.mockResolvedValue([ + { + file_path: "test/file1.js", + content: "test content", + start_line: 1, + end_line: 5, + identifier: "test", + type: "function", + fileHash: "hash", + segmentHash: "threshold-segment", + }, ]) - expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileB, hash("current content")) - expect(parser.parseFile).toHaveBeenCalledTimes(activity === "read" ? 1 : 2) - }, - ) - - it("keeps unchanged parse failures exhausted across scanner recreation and renews changed content", async () => { - const { scanner, parser, failures, createScanner } = setup() - const recorded = vi.fn[0]>() - failures.onDidRecord(recorded) - const onError = vi.fn<(error: Error) => void>() - parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) - await scanner.scanDirectory(workspacePath, onError) - const repeated = createScanner() - const result = await repeated.scanDirectory(workspacePath, onError) - expect(result.incompleteFiles).toEqual([fileA]) - expect(repeated.hasIncompleteFiles).toBe(true) - expect(recorded).toHaveBeenCalledOnce() - expect(onError).toHaveBeenCalledTimes(2) - expect(parser.parseFile).toHaveBeenCalledOnce() - vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("changed content")) - await createScanner().scanDirectory(workspacePath) - expect(parser.parseFile).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - }) - it.each(["embed", "upsert"] as const)( - "shares the %s retry allowance across batches and service recreation", - async (activity) => { - const { scanner, parser, embedder, store, failures, createScanner, cache } = setup() - parser.parseFile.mockResolvedValue(blocks(fileA, 2)) - const failure = Object.assign(new Error("unavailable"), { status: 503 }) - if (activity === "embed") - embedder.createEmbeddings - .mockRejectedValueOnce(failure) - .mockResolvedValueOnce({ embeddings: [[1]] }) - .mockRejectedValue(failure) - else - store.upsertPoints - .mockRejectedValueOnce(failure) - .mockResolvedValueOnce(undefined) - .mockRejectedValue(failure) - const pending = scanner.scanDirectory(workspacePath) - await vi.advanceTimersByTimeAsync(1500) - expect((await pending).incompleteFiles).toEqual([fileA]) - const request = activity === "embed" ? embedder.createEmbeddings : store.upsertPoints - expect(request).toHaveBeenCalledTimes(4) - expect(failures.getRetryBudget("a.ts", hash("current content"), activity).retriesUsed).toBe(2) - expect(failures.get("a.ts")).toMatchObject({ activity, attempt: 3, state: "exhausted" }) - await createScanner().scanDirectory(workspacePath) - expect(request).toHaveBeenCalledTimes(4) - expect(cache.completeFile).not.toHaveBeenCalled() - }, - ) - - it.each([400, 401, 403, 408, 409])("does not retry permanent HTTP %s failures", async (status) => { - const { scanner, embedder, failures } = setup() - embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("permanent"), { status })) - await scanner.scanDirectory(workspacePath) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(failures.get("a.ts")).toMatchObject({ activity: "embed", attempt: 1, state: "exhausted" }) - }) + await thresholdScanner.scanDirectory("/test") - it("keeps parser cancellation scoped to the session when a revision is superseded", async () => { - const { scanner, parser, failures, cache } = setup() - const controller = new AbortController() - const held = deferred() - parser.parseFile.mockReturnValueOnce(held.promise) - const pending = scanner.scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) - await vi.advanceTimersByTimeAsync(0) - const parseSignal = parser.parseFile.mock.calls[0][1]?.signal - expect(parseSignal).toBe(controller.signal) - failures.renew("a.ts", "newer revision") - expect(parseSignal?.aborted).toBe(false) - held.resolve(blocks(fileA)) - await pending - expect(cache.completeFile).not.toHaveBeenCalled() - controller.abort() - expect(parseSignal?.aborted).toBe(true) - }) + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1) + }) - it("preserves a fatal parser failure after the active revision was superseded", async () => { - const { scanner, parser, failures } = setup() - const held = deferred() - const fatal = new ParserFatalError("parse-timeout", fileA) - parser.parseFile.mockReturnValueOnce(held.promise) - const pending = scanner.scanDirectory(workspacePath).catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - failures.renew("a.ts", "newer revision") - held.reject(fatal) - expect(await pending).toBe(fatal) - }) + it("should delete points for removed files", async () => { + ;(mockCacheManager.getAllHashes as any).mockReturnValue({ "old/file.js": "old-hash" }) - it.each([60_000, 61_000])("paces repeated %s ms Retry-After without consuming retries", async (retryAfterMs) => { - const { scanner, embedder, failures, cache } = setup() - const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs }) - embedder.createEmbeddings.mockRejectedValueOnce(limited).mockRejectedValueOnce(limited) - const pending = scanner.scanDirectory(workspacePath) - await vi.advanceTimersByTimeAsync(0) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync(retryAfterMs - 1) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(1) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync(retryAfterMs - 1) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(2) - await vi.advanceTimersByTimeAsync(1) - expect((await pending).incompleteFiles).toEqual([]) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(3) - expect(failures.getRetryBudget("a.ts", hash("current content"), "embed").retriesUsed).toBe(0) - expect(cache.completeFile).toHaveBeenCalledOnce() - }) + await scanner.scanDirectory("/test") + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("old/file.js") + expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("old/file.js") + }) - it("retries saving without reembedding and reports grouped diagnostics", async () => { - const { scanner, embedder, store, failures } = setup() - store.upsertPoints.mockRejectedValueOnce(Object.assign(new Error("unavailable"), { status: 503 })) - const pending = scanner.scanDirectory(workspacePath) - await vi.advanceTimersByTimeAsync(500 + LATE_WRITE_GRACE_MS) - expect((await pending).incompleteFiles).toEqual([]) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(store.upsertPoints).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)?.value - expect(batch?.record).toHaveBeenCalledOnce() - expect(batch?.flush).toHaveBeenCalledOnce() - expect(codeIndexLog.info).toHaveBeenCalledWith( - "Scan summary", - expect.objectContaining({ failed: 0, blocksIndexed: 1 }), - ) - }) + it("should filter out files in hidden directories", async () => { + const { listFiles } = await import("../../../glob/list-files") + // Mock listFiles to return files including some in hidden directories + vi.mocked(listFiles).mockResolvedValue([ + [ + "test/file1.js", + "test/.hidden/file2.js", + ".git/config", + "src/.next/static/file3.js", + "normal/file4.js", + ], + false, + ]) - it.each(["parse", "embed", "retry"] as const)( - "cancels during %s without recording cancellation or late completion", - async (phase) => { - const { scanner, parser, embedder, store, cache, failures } = setup() - const controller = new AbortController() - const heldParse = deferred() - const heldEmbed = deferred<{ embeddings: number[][] }>() - if (phase === "parse") parser.parseFile.mockReturnValue(heldParse.promise) - else if (phase === "embed") embedder.createEmbeddings.mockReturnValue(heldEmbed.promise) - else - embedder.createEmbeddings.mockRejectedValue( - Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 }), - ) - const onError = vi.fn() - const pending = scanner - .scanDirectory(workspacePath, onError, undefined, undefined, controller.signal) - .catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - controller.abort() - heldParse.resolve(blocks(fileA)) - heldEmbed.resolve({ embeddings: [[1]] }) - expect(await pending).toMatchObject({ name: "AbortError" }) - await vi.advanceTimersByTimeAsync(60_000) - expect(store.upsertPoints).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - expect(onError).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(phase === "parse" ? 0 : 1) - }, - ) - - it("rejects already cancelled scans before discovery or removal", async () => { - const { scanner, store } = setup() - const controller = new AbortController() - controller.abort() - await expect( - scanner.scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal), - ).rejects.toMatchObject({ name: "AbortError" }) - expect(listFiles).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - }) + // Mock parseFile to track which files are actually processed + const processedFiles: string[] = [] + ;(mockCodeParser.parseFile as any).mockImplementation((filePath: string) => { + processedFiles.push(filePath) + return [] + }) - it("propagates fatal parser identity immediately and stops queued dispatch", async () => { - const { scanner, parser, store } = setup() - const concurrency = Math.min(PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY) - const paths = Array.from({ length: concurrency + 2 }, (_, fileNumber) => - path.join(workspacePath, `${fileNumber}.ts`), - ) - vi.mocked(listFiles).mockResolvedValue([paths, false]) - const fatal = new ParserFatalError("parse-timeout", paths[0]) - const held = deferred() - parser.parseFile.mockImplementation(async (filePath) => { - if (filePath === paths[0]) throw fatal - return held.promise + await scanner.scanDirectory("/test") + + // Verify that only non-hidden files were processed + expect(processedFiles).toEqual(["test/file1.js", "normal/file4.js"]) + expect(processedFiles).not.toContain("test/.hidden/file2.js") + expect(processedFiles).not.toContain(".git/config") + expect(processedFiles).not.toContain("src/.next/static/file3.js") + + // Verify the stats + expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(2) }) - const onError = vi.fn() - await expect(scanner.scanDirectory(workspacePath, onError)).rejects.toBe(fatal) - held.resolve([]) - await vi.advanceTimersByTimeAsync(0) - expect(parser.parseFile.mock.calls.length).toBeLessThanOrEqual(concurrency) - expect(onError).not.toHaveBeenCalled() - expect(store.upsertPoints).not.toHaveBeenCalled() - }) - it.each(["success", "failure", "fatal"] as const)( - "fences late %s parsing from a superseded attempt", - async (outcome) => { - const { scanner, parser, supersede, cache, failures } = setup() - const held = deferred() - parser.parseFile.mockReturnValue(held.promise) - const onError = vi.fn() - const onParsed = vi.fn() - const pending = scanner - .scanDirectory(workspacePath, onError, undefined, onParsed) - .catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - supersede() - if (outcome === "success") held.resolve(blocks(fileA)) - else - held.reject( - outcome === "fatal" ? new ParserFatalError("parser-abort", fileA) : new Error("late failure"), - ) - expect(await pending).toMatchObject({ name: "AbortError" }) - expect(failures.list()).toEqual([]) - expect(onError).not.toHaveBeenCalled() - expect(onParsed).not.toHaveBeenCalled() - expect(cache.completeFile).not.toHaveBeenCalled() - }, - ) - - it.each(["deleted", "excluded", "oversized"] as const)("fences removal of tracked %s files", async (selection) => { - const { scanner, hashes, cache, store, ignored } = setup() - hashes.set(fileA, "old hash") - if (selection === "deleted") vi.mocked(listFiles).mockResolvedValue([[], false]) - if (selection === "excluded") ignored.add("a.ts") - if (selection === "oversized") { - vi.mocked(stat).mockResolvedValue(statsOfSize(MAX_FILE_SIZE_BYTES + 1)) - } - expect((await scanner.scanDirectory(workspacePath)).incompleteFiles).toEqual([]) - expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(fileA, expect.any(Number)) - expect(cache.invalidateFile).toHaveBeenCalledWith(fileA) - expect(cache.completeRemoval).toHaveBeenCalledWith(fileA) - }) + it("should process markdown files alongside code files", async () => { + // Create scanner without embedder to test the non-embedding path + const scannerNoEmbeddings = new DirectoryScanner( + null as any, // No embedder + null as any, // No vector store + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + ) - it("cleans registry-only deleted paths and requires explicit retry after failed removal", async () => { - const { scanner, failures, store } = setup() - failures.renew("a.ts", "unread") - failures.record("a.ts", "unread", "read", "read failed", 1, "exhausted") - vi.mocked(listFiles).mockResolvedValue([[], false]) - store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) - await scanner.scanDirectory(workspacePath) - await scanner.scanDirectory(workspacePath) - expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() - failures.resetAllBudgets() - const pending = scanner.scanDirectory(workspacePath) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - await pending - expect(failures.list()).toEqual([]) - }) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/README.md", "test/app.js", "docs/guide.markdown"], false]) + + const mockMarkdownBlocks: any[] = [ + { + file_path: "test/README.md", + content: "# Introduction\nThis is a comprehensive guide...", + start_line: 1, + end_line: 10, + identifier: "Introduction", + type: "markdown_header_h1", + fileHash: "md-hash", + segmentHash: "md-segment-hash", + }, + ] + + const mockJsBlocks: any[] = [ + { + file_path: "test/app.js", + content: "function main() { return 'hello'; }", + start_line: 1, + end_line: 3, + identifier: "main", + type: "function", + fileHash: "js-hash", + segmentHash: "js-segment-hash", + }, + ] + + const mockMarkdownBlocks2: any[] = [ + { + file_path: "docs/guide.markdown", + content: "## Getting Started\nFollow these steps...", + start_line: 1, + end_line: 8, + identifier: "Getting Started", + type: "markdown_header_h2", + fileHash: "markdown-hash", + segmentHash: "markdown-segment-hash", + }, + ] + + // Mock parseFile to return different blocks based on file extension + ;(mockCodeParser.parseFile as any).mockImplementation((filePath: string) => { + if (filePath.endsWith(".md")) { + return mockMarkdownBlocks + } else if (filePath.endsWith(".markdown")) { + return mockMarkdownBlocks2 + } else if (filePath.endsWith(".js")) { + return mockJsBlocks + } + return [] + }) - it.each(["empty", "whitespace"] as const)( - "completes %s files only after cleanup and verification", - async (kind) => { - const { scanner, parser, hashes, store, cache, embedder } = setup() - hashes.set(fileA, "old hash") - parser.parseFile.mockResolvedValue(kind === "empty" ? [] : [{ ...blocks(fileA)[0], content: " \n\t" }]) - const onIndexed = vi.fn() - expect((await scanner.scanDirectory(workspacePath, undefined, onIndexed)).incompleteFiles).toEqual([]) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(fileA, expect.any(Number)) - expect(store.countPointsForFile).toHaveBeenCalledOnce() - expect(cache.completeFile).toHaveBeenCalledExactlyOnceWith(fileA, hash("current content")) - expect(onIndexed).toHaveBeenCalledExactlyOnceWith(0) - }, - ) - - it("checks cancellation after the parsed callback before invalidating or embedding", async () => { - const { scanner, store, cache, embedder } = setup() - const controller = new AbortController() - await expect( - scanner.scanDirectory(workspacePath, undefined, undefined, () => controller.abort(), controller.signal), - ).rejects.toMatchObject({ name: "AbortError" }) - expect(cache.invalidateFile).not.toHaveBeenCalled() - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.upsertPoints).not.toHaveBeenCalled() - }) + const result = await scannerNoEmbeddings.scanDirectory("/test") - it("does not mark verification mismatches complete", async () => { - const { scanner, store, cache, failures } = setup() - store.countPointsForFile.mockResolvedValue({ current: 0, other: 1 }) - expect((await scanner.scanDirectory(workspacePath)).incompleteFiles).toEqual([fileA]) - expect(cache.completeFile).not.toHaveBeenCalled() - expect(failures.get("a.ts")).toMatchObject({ activity: "verify", state: "exhausted" }) - }) + // Verify all files were processed + expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(3) + expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/README.md", expect.any(Object)) + expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/app.js", expect.any(Object)) + expect(mockCodeParser.parseFile).toHaveBeenCalledWith("docs/guide.markdown", expect.any(Object)) - it("skips verified unchanged content but rechecks uncertain content", async () => { - const { scanner, hashes, parser, gate } = setup() - hashes.set(fileA, hash("current content")) - expect((await scanner.scanDirectory(workspacePath)).stats.skipped).toBe(1) - expect(parser.parseFile).not.toHaveBeenCalled() - gate.recover([fileA]) - const pending = scanner.scanDirectory(workspacePath) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - expect((await pending).incompleteFiles).toEqual([]) - expect(parser.parseFile).toHaveBeenCalledOnce() - }) + // Verify processing still works without codeBlocks accumulation + expect(result.stats.processed).toBe(3) + }) - it("retains Zoo extension and hidden-directory filtering", async () => { - const { scanner, parser } = setup() - const selected = ["README.md", "guide.markdown", "app.js"].map((name) => path.join(workspacePath, name)) - vi.mocked(listFiles).mockResolvedValue([ - [ - ...selected, - path.join(workspacePath, ".hidden", "a.ts"), - path.join(workspacePath, "image.png"), - `${workspacePath}/directory/`, - ], - false, - ]) - await scanner.scanDirectory(workspacePath) - expect(parser.parseFile.mock.calls.map(([filePath]) => filePath)).toEqual(selected) - }) -}) + it("should generate unique point IDs for each block from the same file", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/large-doc.md"], false]) + + // Mock multiple blocks from the same file with different segmentHash values + const mockBlocks: any[] = [ + { + file_path: "test/large-doc.md", + content: "# Introduction\nThis is the intro section...", + start_line: 1, + end_line: 10, + identifier: "Introduction", + type: "markdown_header_h1", + fileHash: "same-file-hash", + segmentHash: "unique-segment-hash-1", + }, + { + file_path: "test/large-doc.md", + content: "## Getting Started\nHere's how to begin...", + start_line: 11, + end_line: 20, + identifier: "Getting Started", + type: "markdown_header_h2", + fileHash: "same-file-hash", + segmentHash: "unique-segment-hash-2", + }, + { + file_path: "test/large-doc.md", + content: "## Advanced Topics\nFor advanced users...", + start_line: 21, + end_line: 30, + identifier: "Advanced Topics", + type: "markdown_header_h2", + fileHash: "same-file-hash", + segmentHash: "unique-segment-hash-3", + }, + ] + + ;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks) + + await scanner.scanDirectory("/test") + + // Verify that upsertPoints was called with unique IDs for each block + expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(1) + const upsertCall = mockVectorStore.upsertPoints.mock.calls[0] + const points = upsertCall[0] + + // Extract the IDs from the points + const pointIds = points.map((point: any) => point.id) + + // Verify all IDs are unique + expect(pointIds).toHaveLength(3) + expect(new Set(pointIds).size).toBe(3) // All IDs should be unique + + // Verify that each point has the correct payload + expect(points[0].payload.segmentHash).toBe("unique-segment-hash-1") + expect(points[1].payload.segmentHash).toBe("unique-segment-hash-2") + expect(points[2].payload.segmentHash).toBe("unique-segment-hash-3") + }) + + it("should stop processing files when signal is aborted", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js", "test/file3.js"], false]) + + // Create an already-aborted signal + const controller = new AbortController() + controller.abort() -describe("T42 source missing at read is a removal", () => { - const contentHash = hash("current content") - const errorWithCode = (code: string) => Object.assign(new Error("gone"), { code }) - const failAt = (stage: "stat" | "read", error: Error) => { - if (stage === "stat") vi.mocked(stat).mockRejectedValueOnce(error) - else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(error) - } - const listPaths = (paths: string[]) => vi.mocked(listFiles).mockResolvedValue([paths, false]) - const filePoints = (points: Map, filePath: string) => - [...points.values()].filter((point) => { - const payload: Record = point.payload - return payload.filePath === generateRelativeFilePath(filePath, workspacePath) + const result = await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) + + // No files should have been processed since signal was already aborted + expect(mockCodeParser.parseFile).not.toHaveBeenCalled() + expect(result.stats.processed).toBe(0) }) - const seed = (points: Map, filePath: string) => - points.set(`seed:${filePath}`, { - id: `seed:${filePath}`, - vector: [0], - payload: { filePath: generateRelativeFilePath(filePath, workspacePath), generation: 0 }, + + it("should not write once the signal aborts mid-scan", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) + + const controller = new AbortController() + + const mockBlocks: any[] = [ + { + file_path: "test/file1.js", + content: "function hello() {}", + start_line: 1, + end_line: 3, + identifier: "hello", + type: "function", + fileHash: "hash1", + segmentHash: "seg-hash-1", + }, + ] + + // Abort after first file is parsed + ;(mockCodeParser.parseFile as any).mockImplementation(async () => { + controller.abort() + return mockBlocks + }) + + await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() }) - it.each( - (["stat", "read", "oversized"] as const).flatMap((source) => - (["new", "indexed", "failed"] as const).map((history) => ({ source, history })), - ), - )("preserves bookkeeping and logs for $history $source source", async ({ source, history }) => { - const { scanner, parser, cache, points, failures, store } = setup() - listPaths([fileA, fileB]) - if (history === "indexed") cache.completeFile(fileA, contentHash) - if (history !== "new") seed(points, fileA) - if (history === "failed") { - failures.renew("a.ts", "old") - failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") - } - if (source === "oversized") - vi.mocked(stat).mockImplementation(async (filePath) => - statsOfSize(filePath === fileA ? MAX_FILE_SIZE_BYTES + 1 : 100), - ) - else failAt(source, errorWithCode(source === "stat" ? "ENOENT" : "FileNotFound")) - const held = deferred() - parser.parseFile.mockReturnValue(held.promise) - const onError = vi.fn() - const pending = scanner.scanDirectory(workspacePath, onError) - await vi.advanceTimersByTimeAsync(0) - try { - expect(parser.parseFile).toHaveBeenCalledExactlyOnceWith(fileB, expect.anything()) - expect(scanner["incompleteFiles"]).toEqual(new Set(history === "new" ? [fileB] : [fileA, fileB])) - expect(store.deletePointsByFilePath).not.toHaveBeenCalled() - if (source === "oversized") expect(codeIndexLog.debug).not.toHaveBeenCalled() - else - expect(codeIndexLog.debug).toHaveBeenCalledExactlyOnceWith( - "Source missing at read; treating as removed", - { path: "a.ts" }, + it.each(["stat", "read"] as const)( + "CI-SC-01: a file missing at %s is dropped and its old entries deleted without a failure", + async (stage) => { + const failures = new FileFailureRegistry() + const missingScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, ) - expect(codeIndexLog.warn).not.toHaveBeenCalled() - const batch = vi.mocked(codeIndexLog.createFailureBatch).mock.results.at(-1)!.value - expect(batch.record).not.toHaveBeenCalled() - } finally { - held.resolve(blocks(fileB)) - await pending - } - const result = await pending - expect(result.stats).toEqual({ processed: 1, skipped: source === "oversized" ? 1 : 0 }) - expect(result.incompleteFiles).toEqual([]) - expect(scanner.hasIncompleteFiles).toBe(false) - expect(filePoints(points, fileA)).toEqual([]) - expect(cache.getHash(fileA)).toBeUndefined() - expect(cache.getHash(fileB)).toBe(contentHash) - expect(onError).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - if (history === "new") expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) - else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) - }) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/gone.js"], false]) + mockCacheManager.getAllHashes.mockReturnValue({ "test/gone.js": "old-hash" }) + const missing = Object.assign(new Error("gone"), { code: "ENOENT" }) + if (stage === "stat") vi.mocked(stat).mockRejectedValueOnce(missing) + else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(missing) + + await missingScanner.scanDirectory("/test") + + expect(mockCodeParser.parseFile).not.toHaveBeenCalled() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("test/gone.js") + expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("test/gone.js") + expect(failures.list()).toEqual([]) + }, + ) - it.each([ - ["stat", "ENOENT"], - ["read", "FileNotFound"], - ] as const)("never-indexed file missing at %s needs no recovery", async (stage, code) => { - const { scanner, cache, store, failures } = setup() - listPaths([fileA, fileB]) - failAt(stage, errorWithCode(code)) - const onError = vi.fn() - const result = await scanner.scanDirectory(workspacePath, onError) - expect(result.incompleteFiles).toEqual([]) - expect(result.stats).toEqual({ processed: 1, skipped: 0 }) - expect(cache.getHash(fileB)).toBe(contentHash) - expect(onError).not.toHaveBeenCalled() - expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) - expect(failures.list()).toEqual([]) - }) + it("records a non-missing read error as a read failure", async () => { + const failures = new FileFailureRegistry() + const readScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/denied.js"], false]) + vi.mocked(stat).mockRejectedValueOnce(Object.assign(new Error("denied"), { code: "EACCES" })) - it.each(["stat", "read"] as const)( - "previously indexed file missing at %s is removed in the same scan", - async (stage) => { - const { scanner, points, cache, failures } = setup() - await scanner.scanDirectory(workspacePath) - expect(filePoints(points, fileA)).toHaveLength(1) - failAt(stage, errorWithCode(stage === "stat" ? "ENOENT" : "FileNotFound")) - const onError = vi.fn() - const result = await scanner.scanDirectory(workspacePath, onError) - expect(result.incompleteFiles).toEqual([]) - expect(filePoints(points, fileA)).toEqual([]) - expect(cache.getHash(fileA)).toBeUndefined() - expect(onError).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - }, - ) - - it.each( - (["stat", "read"] as const).flatMap((stage) => - (["new", "indexed", "failed"] as const).flatMap((history) => - (["during", "after"] as const).map((timing) => ({ stage, history, timing })), - ), - ), - )( - "removes $history source missing at $stage $timing a concurrent provider wait", - async ({ stage, history, timing }) => { - const { scanner, points, cache, store, embedder, failures, pacers } = setup() - if (history === "indexed") await scanner.scanDirectory(workspacePath) - if (history === "failed") { - seed(points, fileA) - failures.renew("a.ts", "old") - failures.record("a.ts", "old", "read", "read failed", 1, "exhausted") - } - listPaths([fileA, fileB]) - const metadata = deferred() - vi.mocked(stat).mockImplementation(async (filePath) => - filePath === fileA ? metadata.promise : statsOfSize(100), + await readScanner.scanDirectory("/test") + + expect(failures.list()).toEqual([expect.objectContaining({ activity: "read", reason: "denied" })]) + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + }) + + it.each([ + ["parser load", new ParserLoadError(".js", new Error("wasm"))], + ["parse", new Error("syntax")], + ])("CI-SC-02: a %s failure is recorded, the hash is not stored and other files continue", async (_label, error) => { + const failures = new FileFailureRegistry() + const parseScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, ) - const readFile = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! - const missingRead = vi.fn() - vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => { - if (uri.fsPath === fileA) { - missingRead() - throw errorWithCode("FileNotFound") - } - return readFile(uri) - }) - embedder.createEmbeddings - .mockClear() - .mockRejectedValueOnce(Object.assign(new Error("limited"), { status: 429, retryAfterMs: 30_000 })) - const onError = vi.fn() - const pending = scanner.scanDirectory(workspacePath, onError) - await vi.advanceTimersByTimeAsync(0) - expect(pacers.waitingPaths()).toContain("b.ts") - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(missingRead).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(timing === "during" ? 29_999 : 30_000) - if (stage === "stat") metadata.reject(errorWithCode("ENOENT")) - else metadata.resolve(statsOfSize(100)) - await vi.advanceTimersByTimeAsync(0) - expect(missingRead).toHaveBeenCalledTimes(stage === "read" ? 1 : 0) - if (timing === "during") { - expect(pacers.waitingPaths()).toContain("b.ts") - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(1) + const block = { + file_path: "test/file2.js", + content: "ok", + start_line: 1, + end_line: 1, + identifier: null, + type: "function", + fileHash: "h", + segmentHash: "s", } - const result = await pending - expect(result.stats).toEqual({ processed: 1, skipped: 0 }) - expect(result.incompleteFiles).toEqual([]) - expect(scanner.hasIncompleteFiles).toBe(false) - expect(filePoints(points, fileA)).toEqual([]) - expect(cache.getHash(fileA)).toBeUndefined() - expect(filePoints(points, fileB)).toHaveLength(1) - expect(cache.getHash(fileB)).toBe(contentHash) - expect(onError).not.toHaveBeenCalled() - expect(failures.list()).toEqual([]) - expect(pacers.waitingPaths()).toEqual([]) - if (history === "new") - expect(store.deletePointsByFilePath).not.toHaveBeenCalledWith(fileA, expect.anything()) - else expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) - }, - ) - - it("clears a prior failure for a path found missing", async () => { - const { scanner, parser, failures } = setup() - parser.parseFile.mockRejectedValueOnce(new Error("parse failed")) - await scanner.scanDirectory(workspacePath, vi.fn()) - expect(failures.list()).toHaveLength(1) - failAt("read", errorWithCode("FileNotFound")) - const result = await scanner.scanDirectory(workspacePath, vi.fn()) - expect(result.incompleteFiles).toEqual([]) - expect(failures.list()).toEqual([]) - }) + mockCodeParser.parseFile.mockImplementation(async (filePath: string) => { + if (filePath === "test/file1.js") throw error + return [block] + }) - it("keeps a cleanup failure as a delete failure, not a read failure", async () => { - const { scanner, store, failures } = setup() - await scanner.scanDirectory(workspacePath) - failAt("stat", errorWithCode("ENOENT")) - store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) - const result = await scanner.scanDirectory(workspacePath, vi.fn()) - expect(result.incompleteFiles).toEqual([fileA]) - expect(failures.list()).toEqual([expect.objectContaining({ path: "a.ts", activity: "delete" })]) - }) + await parseScanner.scanDirectory("/test") - it.each([ - ["stat", "EACCES"], - ["read", "NoPermissions"], - ["read", "EISDIR"], - ] as const)("keeps denied access at %s (%s) as a read failure", async (stage, code) => { - const { scanner, failures } = setup() - failAt(stage, errorWithCode(code)) - const onError = vi.fn() - const result = await scanner.scanDirectory(workspacePath, onError) - expect(result.incompleteFiles).toEqual([fileA]) - expect(onError).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([ - expect.objectContaining({ path: "a.ts", activity: "read", state: "exhausted" }), - ]) - }) + expect(failures.list()).toEqual([expect.objectContaining({ path: "test/file1.js", activity: "parse" })]) + expect(mockCacheManager.updateHash).not.toHaveBeenCalledWith("test/file1.js", expect.anything()) + expect(mockCacheManager.updateHash).toHaveBeenCalledWith("test/file2.js", expect.any(String)) + }) + + it("should not process deleted files when signal is aborted", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([[], false]) + + // Set up cached files that would normally be detected as deleted + ;(mockCacheManager.getAllHashes as any).mockReturnValue({ "old/file.js": "old-hash" }) + + // Create an already-aborted signal + const controller = new AbortController() + controller.abort() - it("indexes a recreated file normally on the next scan", async () => { - const { scanner, points, cache } = setup() - await scanner.scanDirectory(workspacePath) - failAt("read", errorWithCode("FileNotFound")) - await scanner.scanDirectory(workspacePath) - expect(filePoints(points, fileA)).toEqual([]) - await scanner.scanDirectory(workspacePath) - expect(filePoints(points, fileA)).toHaveLength(1) - expect(cache.getHash(fileA)).toBe(contentHash) + await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) + + // Deleted file cleanup should not have run + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + }) }) }) diff --git a/src/services/code-index/processors/directory-scanner-factory.ts b/src/services/code-index/processors/directory-scanner-factory.ts index f3f3f3e20b..9efce99806 100644 --- a/src/services/code-index/processors/directory-scanner-factory.ts +++ b/src/services/code-index/processors/directory-scanner-factory.ts @@ -9,12 +9,8 @@ export class DirectoryScannerFactory implements IDirectoryScannerFactory { parser, cacheManager, ignoreInstance, - writeGate, - generations, - failures, - workspacePath, - isCurrentAttempt, - providerPacers, + failureRegistry, + pacers, }: DirectoryScannerFactoryOptions): DirectoryScanner { return new DirectoryScanner( embedder, @@ -23,12 +19,8 @@ export class DirectoryScannerFactory implements IDirectoryScannerFactory { cacheManager, ignoreInstance, getEmbeddingBatchSize(), - writeGate, - generations, - failures, - workspacePath, - isCurrentAttempt, - providerPacers, + failureRegistry, + pacers, ) } } diff --git a/src/services/code-index/processors/file-preparation-dependencies.ts b/src/services/code-index/processors/file-preparation-dependencies.ts index 001da428d7..9e11635bec 100644 --- a/src/services/code-index/processors/file-preparation-dependencies.ts +++ b/src/services/code-index/processors/file-preparation-dependencies.ts @@ -2,7 +2,7 @@ import type { Ignore } from "ignore" import type { FileSystem } from "vscode" import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import type { CacheManager } from "../cache-manager" -import type { ICodeParser, IEmbedder } from "../interfaces" +import type { ICodeParser } from "../interfaces" export interface FilePreparationDependencies { workspacePath: string @@ -11,5 +11,4 @@ export interface FilePreparationDependencies { fileSystem: Pick cacheManager: Pick parser: ICodeParser - embedder?: Pick } diff --git a/src/services/code-index/processors/file-preparation.ts b/src/services/code-index/processors/file-preparation.ts index a99ff6b56a..1f67e58b7a 100644 --- a/src/services/code-index/processors/file-preparation.ts +++ b/src/services/code-index/processors/file-preparation.ts @@ -1,51 +1,44 @@ import { createHash } from "crypto" import { Uri } from "vscode" -import { v5 as uuidv5 } from "uuid" -import type { CodeBlock, FileProcessingResult, PointStruct } from "../interfaces" +import type { FileProcessingResult } from "../interfaces" import type { FilePreparationDependencies } from "./file-preparation-dependencies" -import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { MAX_FILE_SIZE_BYTES } from "../constants" +import { generateRelativeFilePath } from "../shared/get-relative-path" +import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" -/** Prepares one file for batching without writing points or mutating the hash cache. */ +/** Prepares one file for batching without embedding, writing points or mutating the hash cache. */ export class FilePreparation { constructor(private readonly dependencies: FilePreparationDependencies) {} public async prepareFile(filePath: string): Promise { const dependencies = this.dependencies + let content: string try { const skipReason = await this.getSkipReason(filePath) - if (skipReason !== undefined) { - return this.skippedResult(filePath, skipReason) - } + if (skipReason === SOURCE_MISSING) return { path: filePath, status: "missing" } + if (skipReason !== undefined) return this.skippedResult(filePath, skipReason) - const fileContent = await dependencies.fileSystem.readFile(Uri.file(filePath)) - const content = Buffer.from(fileContent).toString("utf-8") - const newHash = createHash("sha256").update(content).digest("hex") + const fileContent = await readSource(dependencies.fileSystem.readFile(Uri.file(filePath))) + if (fileContent === SOURCE_MISSING) return { path: filePath, status: "missing" } + content = Buffer.from(fileContent).toString("utf-8") + } catch (error) { + return { path: filePath, status: "local_error", activity: "read", error: error as Error } + } + try { + const newHash = createHash("sha256").update(content).digest("hex") if (dependencies.cacheManager.getHash(filePath) === newHash) { return this.skippedResult(filePath, "File has not changed") } - const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash }) - const pointsToUpsert = await this.preparePoints(blocks) - - return { - path: filePath, - status: "processed_for_batching", - newHash, - pointsToUpsert, - } + return { path: filePath, status: "processed_for_batching", newHash, blocks } } catch (error) { - return { - path: filePath, - status: "local_error", - error: error as Error, - } + return { path: filePath, status: "local_error", activity: "parse", error: error as Error } } } - private async getSkipReason(filePath: string): Promise { + private async getSkipReason(filePath: string): Promise { const dependencies = this.dependencies // Use relative paths so ignored directories outside the workspace do not exclude the file. const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath) @@ -60,7 +53,8 @@ export class FilePreparation { return "File is ignored by .rooignore or .gitignore" } - const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath)) + const fileStat = await readSource(dependencies.fileSystem.stat(Uri.file(filePath))) + if (fileStat === SOURCE_MISSING) return SOURCE_MISSING if (fileStat.size > MAX_FILE_SIZE_BYTES) { return "File is too large" } @@ -70,31 +64,4 @@ export class FilePreparation { private skippedResult(filePath: string, reason: string): FileProcessingResult { return { path: filePath, status: "skipped", reason } } - - private async preparePoints(blocks: CodeBlock[]): Promise { - const dependencies = this.dependencies - if (!dependencies.embedder || blocks.length === 0) { - return [] - } - - const texts = blocks.map((block) => block.content) - const { embeddings } = await dependencies.embedder.createEmbeddings(texts) - - return blocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, dependencies.workspacePath) - const stableName = `${normalizedAbsolutePath}:${block.start_line}` - const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - }, - } - }) - } } diff --git a/src/services/code-index/processors/file-watcher-factory.ts b/src/services/code-index/processors/file-watcher-factory.ts index bedce2ff7e..d76e66d6c4 100644 --- a/src/services/code-index/processors/file-watcher-factory.ts +++ b/src/services/code-index/processors/file-watcher-factory.ts @@ -12,12 +12,8 @@ export class FileWatcherFactory implements IFileWatcherFactory { vectorStore, ignoreInstance, rooIgnoreController, - parser, - writeGate, - generations, - failures, - isCurrentAttempt, - providerPacers, + failureRegistry, + pacers, }: FileWatcherFactoryOptions): IFileWatcher { return new FileWatcher( workspacePath, @@ -28,12 +24,8 @@ export class FileWatcherFactory implements IFileWatcherFactory { ignoreInstance, rooIgnoreController, getEmbeddingBatchSize(), - parser, - writeGate, - generations, - failures, - isCurrentAttempt, - providerPacers, + failureRegistry, + pacers, ) } } diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 83a6eb58b6..ad96dd824f 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -1,53 +1,34 @@ import * as vscode from "vscode" -import * as path from "path" -import { createHash } from "crypto" -import type { Ignore } from "ignore" -import { MAX_FILE_SIZE_BYTES, BATCH_SEGMENT_THRESHOLD } from "../constants" +import { BATCH_SEGMENT_THRESHOLD } from "../constants" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" +import { Ignore } from "ignore" import { scannerExtensions } from "../shared/supported-extensions" -import type { - IFileWatcher, - FileProcessingResult, - IEmbedder, - IVectorStore, - ICodeParser, - BatchProcessingSummary, -} from "../interfaces" -import { ParseWorkerClient } from "../parse-worker-client" -import { IndexWriteGate } from "../index-write-gate" -import { GenerationAllocator } from "../generation-allocator" -import { FileIndexWriter } from "../file-completion" -import { abortError, ParserFatalError, throwIfAborted } from "../parser-errors" -import type { CacheManager } from "../cache-manager" -import { generateRelativeFilePath } from "../shared/get-relative-path" -import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" +import { IFileWatcher, FileProcessingResult, IEmbedder, IVectorStore, BatchProcessingSummary } from "../interfaces" +import { codeParser } from "./parser" +import { FilePreparation } from "./file-preparation" +import { CacheManager } from "../cache-manager" import { Package } from "../../../shared/package" -import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" -import { codeIndexLog } from "../code-index-log" +import { FileFailureRegistry, UNKNOWN_REVISION } from "../file-failure-registry" import { ProviderPacerRegistry } from "../provider-pacer" -import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" - -type FileEvent = { uri: vscode.Uri; type: "create" | "change" | "delete" } +import { FileIndexWriter } from "../file-index-writer" +import { generateRelativeFilePath } from "../shared/get-relative-path" +/** + * Implementation of the file watcher interface + */ export class FileWatcher implements IFileWatcher { + private readonly filePreparation: FilePreparation + private writer?: FileIndexWriter + private signal?: AbortSignal + private ignoreInstance?: Ignore private fileWatcher?: vscode.FileSystemWatcher - private readonly ignoreController: RooIgnoreController - private readonly accumulatedEvents = new Map() + private ignoreController: RooIgnoreController + private accumulatedEvents: Map = new Map() private batchProcessDebounceTimer?: NodeJS.Timeout private readonly BATCH_DEBOUNCE_DELAY_MS = 500 private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10 private readonly batchSegmentThreshold: number - private readonly writer?: FileIndexWriter - private readonly pathControllers = new Map() - private readonly localAbortController = new AbortController() - private signal = this.localAbortController.signal - private attemptOwner = this.localAbortController.signal - private processingBatch = false - private activeFiles = 0 - private disposed = false - private fatalReported = false - private readonly parserFatal = new vscode.EventEmitter() - public readonly onParserFatal = this.parserFatal.event + private readonly _onDidStartBatchProcessing = new vscode.EventEmitter() private readonly _onBatchProgressUpdate = new vscode.EventEmitter<{ processedInBatch: number @@ -55,38 +36,56 @@ export class FileWatcher implements IFileWatcher { currentFile?: string }>() private readonly _onDidFinishBatchProcessing = new vscode.EventEmitter() + + /** + * Event emitted when a batch of files begins processing + */ public readonly onDidStartBatchProcessing = this._onDidStartBatchProcessing.event + + /** + * Event emitted to report progress during batch processing + */ public readonly onBatchProgressUpdate = this._onBatchProgressUpdate.event + + /** + * Event emitted when a batch of files has finished processing + */ public readonly onDidFinishBatchProcessing = this._onDidFinishBatchProcessing.event + /** + * Creates a new file watcher + * @param workspacePath Path to the workspace + * @param context VS Code extension context + * @param embedder Optional embedder + * @param vectorStore Optional vector store + * @param cacheManager Cache manager + */ constructor( - private readonly workspacePath: string, + private workspacePath: string, _context: vscode.ExtensionContext, private readonly cacheManager: CacheManager, - private readonly embedder?: IEmbedder, - vectorStore?: IVectorStore, - private readonly ignoreInstance?: Ignore, + private embedder?: IEmbedder, + private vectorStore?: IVectorStore, + ignoreInstance?: Ignore, ignoreController?: RooIgnoreController, batchSegmentThreshold?: number, - private readonly parser: ICodeParser = new ParseWorkerClient(), - private readonly writeGate = new IndexWriteGate(), - generations = new GenerationAllocator(cacheManager), - private readonly failures = new FileFailureRegistry(), - private readonly isCurrentAttempt: () => boolean = () => true, - private readonly providerPacers = new ProviderPacerRegistry(), + private readonly failures: FileFailureRegistry = new FileFailureRegistry(), + private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), ) { - this.ignoreController = ignoreController ?? new RooIgnoreController(workspacePath) - if (vectorStore) { - this.writer = new FileIndexWriter( - cacheManager, - vectorStore, - writeGate, - generations, - workspacePath, - failures, - this.providerPacers, - ) + this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) + if (ignoreInstance) { + this.ignoreInstance = ignoreInstance } + this.filePreparation = new FilePreparation({ + workspacePath: this.workspacePath, + ignoreController: this.ignoreController, + ignoreInstance: this.ignoreInstance, + fileSystem: vscode.workspace.fs, + cacheManager: this.cacheManager, + parser: codeParser, + }) + // Get the configurable batch size from VSCode settings, fallback to default + // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -95,266 +94,183 @@ export class FileWatcher implements IFileWatcher { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { + // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } + if (this.embedder && this.vectorStore) { + this.writer = new FileIndexWriter( + this.embedder, + this.vectorStore, + this.cacheManager, + this.failures, + this.pacers, + this.workspacePath, + this.batchSegmentThreshold, + ) + } } - public get isProcessing(): boolean { - return this.processingBatch || this.activeFiles > 0 - } - - private isLive(): boolean { - return !this.disposed && !this.signal.aborted && this.isCurrentAttempt() - } - - private checkActive(signal = this.signal): void { - throwIfAborted(signal) - if (!this.isLive()) throw abortError() - } - + /** + * Initializes the file watcher; `signal` is the indexing session's abort signal. + */ async initialize(signal?: AbortSignal): Promise { - this.checkActive() - if (this.fileWatcher) return - this.attemptOwner = signal ?? this.localAbortController.signal this.signal = signal - ? AbortSignal.any([signal, this.localAbortController.signal]) - : this.localAbortController.signal - this.checkActive() - this.writeGate.own(this.signal, this.attemptOwner) + // Create file watcher const filePattern = new vscode.RelativePattern( this.workspacePath, - `**/*{${scannerExtensions.map((extension) => extension.substring(1)).join(",")}}`, + `**/*{${scannerExtensions.map((e) => e.substring(1)).join(",")}}`, ) this.fileWatcher = vscode.workspace.createFileSystemWatcher(filePattern) - this.fileWatcher.onDidCreate((uri) => this.handleFileEvent(uri, "create")) - this.fileWatcher.onDidChange((uri) => this.handleFileEvent(uri, "change")) - this.fileWatcher.onDidDelete((uri) => this.handleFileEvent(uri, "delete")) + + // Register event handlers + this.fileWatcher.onDidCreate(this.handleFileCreated.bind(this)) + this.fileWatcher.onDidChange(this.handleFileChanged.bind(this)) + this.fileWatcher.onDidDelete(this.handleFileDeleted.bind(this)) } + /** + * Disposes the file watcher + */ dispose(): void { - if (this.disposed) return - this.disposed = true - this.localAbortController.abort() - this.parserFatal.dispose() this.fileWatcher?.dispose() - clearTimeout(this.batchProcessDebounceTimer) + if (this.batchProcessDebounceTimer) { + clearTimeout(this.batchProcessDebounceTimer) + } this._onDidStartBatchProcessing.dispose() this._onBatchProgressUpdate.dispose() this._onDidFinishBatchProcessing.dispose() this.accumulatedEvents.clear() } - private handleFileEvent(uri: vscode.Uri, type: FileEvent["type"]): void { - if (!this.isLive()) return - this.pathControllers.get(uri.fsPath)?.abort() - this.accumulatedEvents.set(uri.fsPath, { uri, type }) + /** + * Handles file creation events + * @param uri URI of the created file + */ + private async handleFileCreated(uri: vscode.Uri): Promise { + this.accumulatedEvents.set(uri.fsPath, { uri, type: "create" }) this.scheduleBatchProcessing() } + /** + * Handles file change events + * @param uri URI of the changed file + */ + private async handleFileChanged(uri: vscode.Uri): Promise { + this.accumulatedEvents.set(uri.fsPath, { uri, type: "change" }) + this.scheduleBatchProcessing() + } + + /** + * Handles file deletion events + * @param uri URI of the deleted file + */ + private async handleFileDeleted(uri: vscode.Uri): Promise { + this.accumulatedEvents.set(uri.fsPath, { uri, type: "delete" }) + this.scheduleBatchProcessing() + } + + /** + * Schedules batch processing with debounce + */ private scheduleBatchProcessing(): void { - if (!this.isLive()) return - clearTimeout(this.batchProcessDebounceTimer) - this.batchProcessDebounceTimer = setTimeout( - () => void this.triggerBatchProcessing(), - this.BATCH_DEBOUNCE_DELAY_MS, - ) + if (this.batchProcessDebounceTimer) { + clearTimeout(this.batchProcessDebounceTimer) + } + this.batchProcessDebounceTimer = setTimeout(() => this.triggerBatchProcessing(), this.BATCH_DEBOUNCE_DELAY_MS) } + /** + * Triggers processing of accumulated events + */ private async triggerBatchProcessing(): Promise { - if (!this.isLive() || this.processingBatch || this.accumulatedEvents.size === 0) return + if (this.accumulatedEvents.size === 0) { + return + } + const eventsToProcess = new Map(this.accumulatedEvents) this.accumulatedEvents.clear() - this.processingBatch = true - try { - this._onDidStartBatchProcessing.fire([...eventsToProcess.keys()]) - await this.processBatch(eventsToProcess) - } catch (error) { - if (!this.isLive()) return - if (error instanceof ParserFatalError) this.reportFatal(error) - else { - this._onDidFinishBatchProcessing.fire({ processedFiles: [], batchError: this.toError(error) }) - } - } finally { - this.processingBatch = false - if (this.accumulatedEvents.size > 0) this.scheduleBatchProcessing() - } - } - private async processBatch(eventsToProcess: Map): Promise { - this.checkActive() - const events = [...eventsToProcess.values()] - const failureBatch = codeIndexLog.createFailureBatch({ source: "watcher" }) - const subscription = this.failures.onDidRecord((failure) => { - if (this.isLive()) failureBatch.record({ ...failure, willRetry: failure.state === "retrying" }) - }) - try { - const results: FileProcessingResult[] = [] - let nextEvent = 0 - let completed = 0 - this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: events.length }) - const processNext = async () => { - while (nextEvent < events.length) { - this.checkActive() - const event = events[nextEvent++] - const result = - event.type === "delete" - ? await this.removeFile(event.uri.fsPath) - : await this.processFile(event.uri.fsPath) - this.checkActive() - results.push(result) - if (result.status === "success" || result.status === "skipped") { - completed++ - this._onBatchProgressUpdate.fire({ - processedInBatch: completed, - totalInBatch: events.length, - currentFile: result.path, - }) - } - } - } - // Complete each file independently rather than waiting for a cross-file flush. - await Promise.all( - Array.from({ length: Math.min(events.length, this.FILE_PROCESSING_CONCURRENCY_LIMIT) }, processNext), - ) - this.checkActive() - const errors = results.filter((result) => result.status === "local_error") - this._onDidFinishBatchProcessing.fire({ - processedFiles: results, - batchError: errors.length ? new Error(`${errors.length} file changes could not be indexed`) : undefined, - }) - } finally { - subscription.dispose() - failureBatch.flush() - } - } + const filePathsInBatch = Array.from(eventsToProcess.keys()) + this._onDidStartBatchProcessing.fire(filePathsInBatch) - private reportFatal(error: ParserFatalError): void { - if (this.fatalReported || !this.isLive()) return - this.fatalReported = true - this.parserFatal.fire(error) - this.dispose() + await this.processBatch(eventsToProcess) } - private toError(error: unknown): Error { - return new Error(codeIndexLog.redact(error instanceof Error ? error.message : String(error)), { cause: error }) + private async removeFile(filePath: string): Promise { + if (!this.writer) return { path: filePath, status: "success" } + const removed = await this.writer.remove(filePath, this.signal) + return removed + ? { path: filePath, status: "success" } + : { path: filePath, status: "error", error: new Error(`Failed to remove ${filePath} from the index`) } } - private async removeFile(filePath: string, signal = this.signal): Promise { - this.checkActive(signal) - const controller = signal === this.signal ? new AbortController() : undefined - if (controller) { - this.pathControllers.get(filePath)?.abort() - this.pathControllers.set(filePath, controller) - signal = AbortSignal.any([signal, controller.signal]) - } - try { - this.checkActive(signal) - if ( - !this.cacheManager.isTracked(filePath) && - !this.failures.has(generateRelativeFilePath(filePath, this.workspacePath)) - ) { - return { path: filePath, status: "skipped", reason: "File is not tracked" } + private async indexFile(filePath: string): Promise { + const result = await this.processFile(filePath) + if (result.status === "missing") return this.removeFile(filePath) + if (result.status === "local_error") { + if (!this.signal?.aborted) { + const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + const revision = result.newHash ?? UNKNOWN_REVISION + const reason = result.error instanceof Error ? result.error.message : String(result.error) + this.failures.renew(relativePath, revision) + this.failures.record(relativePath, revision, result.activity ?? "read", reason, 1, "exhausted") } - if (!this.writer) throw new Error("Vector store is not configured") - await this.writer.remove(filePath, signal, this.attemptOwner) - this.checkActive(signal) - return { path: filePath, status: "success" } - } catch (error) { - this.checkActive() - if (signal.aborted) return { path: filePath, status: "skipped", reason: "File change was superseded" } - if (error instanceof Error && error.name === "AbortError") throw error - return { path: filePath, status: "local_error", error: this.toError(error) } - } finally { - if (controller && this.pathControllers.get(filePath) === controller) this.pathControllers.delete(filePath) + return result } + if (result.status !== "processed_for_batching" || !result.newHash || !this.writer) return result + const indexed = await this.writer.replace(filePath, result.newHash, result.blocks ?? [], this.signal) + return indexed + ? { path: filePath, status: "success" } + : { path: filePath, status: "error", error: new Error(`Failed to index ${filePath}`) } } - private removeMissingFile(filePath: string, signal: AbortSignal): Promise { - codeIndexLog.debug("Source missing at read; treating as removed", { - path: generateRelativeFilePath(filePath, this.workspacePath), - }) - return this.removeFile(filePath, signal) - } + private async processBatch( + eventsToProcess: Map, + ): Promise { + const batchResults: FileProcessingResult[] = [] + const totalFilesInBatch = eventsToProcess.size + let processedCountInBatch = 0 + this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: totalFilesInBatch, currentFile: undefined }) - async processFile(filePath: string): Promise { - this.checkActive() - this.activeFiles++ - this.pathControllers.get(filePath)?.abort() - const controller = new AbortController() - this.pathControllers.set(filePath, controller) - let signal = AbortSignal.any([this.signal, controller.signal]) - const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) - let revision = this.failures.getRevision(relativeFilePath) ?? UNKNOWN_REVISION - let activity: FileFailureActivity = "read" - try { - this.failures.renew(relativeFilePath, revision) - if ( - !scannerExtensions.includes(path.extname(filePath).toLowerCase()) || - isPathInIgnoredDirectory(relativeFilePath) || - !this.ignoreController.validateAccess(filePath) || - this.ignoreInstance?.ignores(relativeFilePath) - ) { - return await this.removeFile(filePath, signal) - } - const fileStat = await readSource(vscode.workspace.fs.stat(vscode.Uri.file(filePath))) - this.checkActive(signal) - if (fileStat === SOURCE_MISSING) return await this.removeMissingFile(filePath, signal) - if (fileStat.size > MAX_FILE_SIZE_BYTES) return await this.removeFile(filePath, signal) - const fileContent = await readSource(vscode.workspace.fs.readFile(vscode.Uri.file(filePath))) - this.checkActive(signal) - if (fileContent === SOURCE_MISSING) return await this.removeMissingFile(filePath, signal) - const content = Buffer.from(fileContent).toString("utf8") - const newHash = createHash("sha256").update(content).digest("hex") - revision = newHash - signal = AbortSignal.any([signal, this.failures.renew(relativeFilePath, newHash)]) - this.writeGate.own(signal, this.attemptOwner) - if ( - this.cacheManager.getHash(filePath) === newHash && - !this.writeGate.needsReconciliation(filePath) && - !this.failures.has(relativeFilePath) - ) { - return { path: filePath, status: "skipped", reason: "File has not changed" } - } - activity = "embed" - if (!this.writer || !this.embedder) throw new Error("Indexing services are not configured") - this.failures.assertRetryable(relativeFilePath, revision) - activity = "parse" - // Cancelling an active parse terminates the shared worker, not just this file's request. - const blocks = await this.parser.parseFile(filePath, { content, fileHash: newHash, signal: this.signal }) - this.checkActive(signal) - activity = "embed" - await this.writer.replace( - filePath, - newHash, - blocks, - this.embedder, - signal, - this.batchSegmentThreshold, - undefined, - this.attemptOwner, + const events = [...eventsToProcess.values()] + for (let offset = 0; offset < events.length; offset += this.FILE_PROCESSING_CONCURRENCY_LIMIT) { + if (this.signal?.aborted) break + const chunk = events.slice(offset, offset + this.FILE_PROCESSING_CONCURRENCY_LIMIT) + const results = await Promise.all( + chunk.map(async (event) => { + const filePath = event.uri.fsPath + try { + return event.type === "delete" ? await this.removeFile(filePath) : await this.indexFile(filePath) + } catch (error) { + return { path: filePath, status: "error", error: error as Error } satisfies FileProcessingResult + } + }), ) - this.checkActive(signal) - return { path: filePath, status: "success" } - } catch (error) { - this.checkActive() - if ( - (activity === "read" || activity === "parse") && - !signal.aborted && - !(error instanceof Error && error.name === "AbortError") - ) { - this.failures.record(relativeFilePath, revision, activity, this.toError(error).message, 1, "exhausted") - } - if (error instanceof ParserFatalError) { - this.reportFatal(error) - throw error + for (const result of results) { + batchResults.push(result) + processedCountInBatch++ + this._onBatchProgressUpdate.fire({ + processedInBatch: processedCountInBatch, + totalInBatch: totalFilesInBatch, + currentFile: result.path, + }) } - if (signal.aborted) return { path: filePath, status: "skipped", reason: "File change was superseded" } - if (error instanceof Error && error.name === "AbortError") throw error - return { path: filePath, status: "local_error", error: this.toError(error) } - } finally { - if (this.pathControllers.get(filePath) === controller) this.pathControllers.delete(filePath) - this.activeFiles-- } + + this._onDidFinishBatchProcessing.fire({ processedFiles: batchResults }) + this._onBatchProgressUpdate.fire({ processedInBatch: totalFilesInBatch, totalInBatch: totalFilesInBatch }) + if (this.accumulatedEvents.size === 0) { + this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: 0, currentFile: undefined }) + } + } + + /** + * Processes a file + * @param filePath Path to the file to process + * @returns Promise resolving to processing result + */ + async processFile(filePath: string): Promise { + return this.filePreparation.prepareFile(filePath) } } diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 1efd997f78..95e29105da 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -1,68 +1,26 @@ -import { codeIndexLog } from "../code-index-log" import { readFile } from "fs/promises" import { createHash } from "crypto" import * as path from "path" -import type { Node } from "web-tree-sitter" -import { type LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser" +import { Node } from "web-tree-sitter" +import { LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser" import { parseMarkdown } from "../../tree-sitter/markdownParser" -import type { ICodeParser, CodeBlock } from "../parser-types" -import { isFallbackExtension as shouldUseFallbackChunking } from "../../shared/fallback-extensions" -import { - MAX_BLOCK_CHARS, - MIN_BLOCK_CHARS, - MIN_CHUNK_REMAINDER_CHARS, - MAX_CHARS_TOLERANCE_FACTOR, -} from "./parser-constants" -import { isParserAbort, throwIfAborted } from "../parser-errors" - -// The scanner's extension barrel imports host-only code; keep workers independent of it. -const parserExtensions = new Set( - [ - "tla", - "js", - "jsx", - "ts", - "vue", - "tsx", - "py", - "rs", - "go", - "c", - "h", - "cpp", - "hpp", - "cs", - "rb", - "java", - "php", - "swift", - "sol", - "kt", - "kts", - "ex", - "exs", - "el", - "html", - "htm", - "md", - "markdown", - "txt", - "json", - "css", - "rdl", - "ml", - "mli", - "lua", - "scala", - "toml", - "zig", - "elm", - "ejs", - "erb", - "vb", - "dart", - ].map((extension) => `.${extension}`), -) +import { ICodeParser, CodeBlock } from "../interfaces" +import { scannerExtensions, shouldUseFallbackChunking } from "../shared/supported-extensions" +import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_TOLERANCE_FACTOR } from "../constants" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" +import { sanitizeErrorMessage } from "../shared/validation-helpers" + +/** An eligible file's language parser failed to load; the failure is not cached, so a later call retries. */ +export class ParserLoadError extends Error { + constructor( + readonly extension: string, + cause?: unknown, + ) { + super(`Failed to load the language parser for ${extension} files`, { cause }) + this.name = "ParserLoadError" + } +} /** * Implementation of the code parser interface @@ -70,17 +28,8 @@ const parserExtensions = new Set( export class CodeParser implements ICodeParser { private loadedParsers: LanguageParser = {} private pendingLoads: Map> = new Map() - public fallbackNeeded = false - - constructor( - private readonly parseInWorker?: ( - filePath: string, - content: string, - fileHash: string, - signal?: AbortSignal, - ) => Promise, - private readonly wasmDirectory?: string, - ) {} + // Markdown files are now supported using the custom markdown parser + // which extracts headers and sections for semantic indexing /** * Parses a code file into code blocks @@ -93,11 +42,8 @@ export class CodeParser implements ICodeParser { options?: { content?: string fileHash?: string - signal?: AbortSignal }, ): Promise { - this.fallbackNeeded = false - throwIfAborted(options?.signal) // Get file extension const ext = path.extname(filePath).toLowerCase() @@ -118,16 +64,17 @@ export class CodeParser implements ICodeParser { content = await readFile(filePath, "utf8") fileHash = this.createFileHash(content) } catch (error) { - throwIfAborted(options?.signal) - codeIndexLog.error(`Error reading file ${filePath}:`, error) + console.error(`Error reading file ${filePath}:`, error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, + location: "parseFile", + }) return [] } } - throwIfAborted(options?.signal) - if (this.parseInWorker && ext !== ".md" && ext !== ".markdown" && !shouldUseFallbackChunking(ext)) { - return this.parseInWorker(filePath, content, fileHash, options?.signal) - } + // Parse the file return this.parseContent(filePath, content, fileHash) } @@ -137,7 +84,7 @@ export class CodeParser implements ICodeParser { * @returns Boolean indicating if the language is supported */ private isSupportedLanguage(extension: string): boolean { - return parserExtensions.has(extension) + return scannerExtensions.includes(extension) } /** @@ -149,10 +96,13 @@ export class CodeParser implements ICodeParser { return createHash("sha256").update(content).digest("hex") } - public fallbackContent(filePath: string, content: string, fileHash: string): CodeBlock[] { - return this._performFallbackChunking(filePath, content, fileHash, new Set()) - } - + /** + * Parses file content into code blocks + * @param filePath Path to the file + * @param content File content + * @param fileHash File hash + * @returns Array of code blocks + */ private async parseContent(filePath: string, content: string, fileHash: string): Promise { const ext = path.extname(filePath).slice(1).toLowerCase() const seenSegmentHashes = new Set() @@ -174,12 +124,16 @@ export class CodeParser implements ICodeParser { try { await pendingLoad } catch (error) { - if (isParserAbort(error)) throw error - codeIndexLog.error(`Error in pending parser load for ${filePath}:`, error) - return [] + console.error(`Error in pending parser load for ${filePath}:`, error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, + location: "parseContent:loadParser", + }) + throw new ParserLoadError(ext, error) } } else { - const loadPromise = loadRequiredLanguageParsers([filePath], this.wasmDirectory) + const loadPromise = loadRequiredLanguageParsers([filePath]) this.pendingLoads.set(ext, loadPromise) try { const newParsers = await loadPromise @@ -187,9 +141,13 @@ export class CodeParser implements ICodeParser { this.loadedParsers = { ...this.loadedParsers, ...newParsers } } } catch (error) { - if (isParserAbort(error)) throw error - codeIndexLog.error(`Error loading language parser for ${filePath}:`, error) - return [] + console.error(`Error loading language parser for ${filePath}:`, error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, + location: "parseContent:loadParser", + }) + throw new ParserLoadError(ext, error) } finally { this.pendingLoads.delete(ext) } @@ -198,39 +156,59 @@ export class CodeParser implements ICodeParser { const language = this.loadedParsers[ext] if (!language) { - codeIndexLog.warn(`No parser available for file extension: ${ext}`) - return [] + console.warn(`No parser available for file extension: ${ext}`) + throw new ParserLoadError(ext) } const tree = language.parser.parse(content) - try { - const captures = tree ? language.query.captures(tree.rootNode) : [] - if (captures.length === 0) { - if (this.wasmDirectory) { - this.fallbackNeeded = true - return [] - } - return this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) + // We don't need to get the query string from languageQueries since it's already loaded + // in the language object + const captures = tree ? language.query.captures(tree.rootNode) : [] + + // Check if captures are empty + if (captures.length === 0) { + if (content.length >= MIN_BLOCK_CHARS) { + // Perform fallback chunking if content is large enough + const blocks = this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) + return blocks + } else { + // Return empty if content is too small for fallback + return [] } + } + + const results: CodeBlock[] = [] + + // Process captures if not empty + const queue: Node[] = Array.from(captures).map((capture) => capture.node) - const results: CodeBlock[] = [] - const queue: Node[] = Array.from(captures).map((capture) => capture.node) + while (queue.length > 0) { + const currentNode = queue.shift()! + // const lineSpan = currentNode.endPosition.row - currentNode.startPosition.row + 1 // Removed as per lint error - for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) { - const currentNode = queue[queueIndex] - if (currentNode.text.length < MIN_BLOCK_CHARS) continue + // Check if the node meets the minimum character requirement + if (currentNode.text.length >= MIN_BLOCK_CHARS) { + // If it also exceeds the maximum character limit, try to break it down if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) { - const children = currentNode.children.filter((child) => child !== null) - if (children.length > 0) { - queue.push(...children) + if (currentNode.children.filter((child) => child !== null).length > 0) { + // If it has children, process them instead + queue.push(...currentNode.children.filter((child) => child !== null)) } else { - results.push(...this._chunkLeafNodeByLines(currentNode, filePath, fileHash, seenSegmentHashes)) + // If it's a leaf node, chunk it + const chunkedBlocks = this._chunkLeafNodeByLines( + currentNode, + filePath, + fileHash, + seenSegmentHashes, + ) + results.push(...chunkedBlocks) } } else { + // Node meets min chars and is within max chars, create a block const identifier = currentNode.childForFieldName("name")?.text || - currentNode.children.find((child) => child?.type === "identifier")?.text || + currentNode.children.find((c) => c?.type === "identifier")?.text || null const type = currentNode.type const start_line = currentNode.startPosition.row + 1 @@ -256,11 +234,10 @@ export class CodeParser implements ICodeParser { } } } - - return results - } finally { - tree?.delete() + // Nodes smaller than minBlockChars are ignored } + + return results } /** @@ -279,8 +256,6 @@ export class CodeParser implements ICodeParser { let currentChunkLength = 0 let chunkStartLineIndex = 0 // 0-based index within the `lines` array const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR - const lineOffsets = [0] - for (const line of lines) lineOffsets.push(lineOffsets[lineOffsets.length - 1] + line.length + 1) const finalizeChunk = (endLineIndex: number) => { if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) { @@ -346,22 +321,14 @@ export class CodeParser implements ICodeParser { finalizeChunk(i - 1) } + // Split the oversized line into segments + let remainingLineContent = line let currentSegmentStartChar = 0 - while (currentSegmentStartChar < line.length) { - let segmentEnd = Math.min(currentSegmentStartChar + MAX_BLOCK_CHARS, line.length) - const precedingUnit = line.charCodeAt(segmentEnd - 1) - const followingUnit = line.charCodeAt(segmentEnd) - if ( - precedingUnit >= 0xd800 && - precedingUnit <= 0xdbff && - followingUnit >= 0xdc00 && - followingUnit <= 0xdfff - ) { - segmentEnd-- - } - const segment = line.slice(currentSegmentStartChar, segmentEnd) + while (remainingLineContent.length > 0) { + const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS) + remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS) createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar) - currentSegmentStartChar = segmentEnd + currentSegmentStartChar += MAX_BLOCK_CHARS } // Update chunkStartLineIndex to continue processing from the next line chunkStartLineIndex = i + 1 @@ -372,7 +339,10 @@ export class CodeParser implements ICodeParser { if (currentChunkLength > 0 && currentChunkLength + lineLength > effectiveMaxChars) { // Re-balancing Logic let splitIndex = i - 1 - const remainderLength = lineOffsets[lines.length] - lineOffsets[i] - 1 + let remainderLength = 0 + for (let j = i; j < lines.length; j++) { + remainderLength += lines[j].length + (j < lines.length - 1 ? 1 : 0) + } if ( currentChunkLength >= MIN_BLOCK_CHARS && @@ -380,8 +350,10 @@ export class CodeParser implements ICodeParser { currentChunkLines.length > 1 ) { for (let k = i - 2; k >= chunkStartLineIndex; k--) { - const potentialChunkLength = lineOffsets[k + 1] - lineOffsets[chunkStartLineIndex] - const potentialNextChunkLength = lineOffsets[lines.length] - lineOffsets[k + 1] + const potentialChunkLines = lines.slice(chunkStartLineIndex, k + 1) + const potentialChunkLength = potentialChunkLines.join("\n").length + 1 + const potentialNextChunkLines = lines.slice(k + 1) + const potentialNextChunkLength = potentialNextChunkLines.join("\n").length + 1 if ( potentialChunkLength >= MIN_BLOCK_CHARS && @@ -588,3 +560,6 @@ export class CodeParser implements ICodeParser { return results } } + +// Export a singleton instance for convenience +export const codeParser = new CodeParser() diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index d749f256d8..9e45a003f8 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -1,5 +1,5 @@ import { listFiles } from "../../glob/list-files" -import type { Ignore } from "ignore" +import { Ignore } from "ignore" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { stat } from "fs/promises" import * as path from "path" @@ -7,9 +7,11 @@ import { generateRelativeFilePath } from "../shared/get-relative-path" import { getWorkspacePathForContext } from "../../../utils/path" import { scannerExtensions } from "../shared/supported-extensions" import * as vscode from "vscode" -import type { ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" +import { ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" import { createHash } from "crypto" -import type { CacheManager } from "../cache-manager" +import pLimit from "p-limit" +import { CacheManager } from "../cache-manager" +import { t } from "../../../i18n" import { MAX_FILE_SIZE_BYTES, MAX_LIST_FILES_LIMIT_CODE_INDEX, @@ -18,19 +20,17 @@ import { BATCH_PROCESSING_CONCURRENCY, } from "../constants" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" +import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" -import { IndexWriteGate } from "../index-write-gate" -import { abortError, ParserFatalError, throwIfAborted } from "../parser-errors" -import { GenerationAllocator } from "../generation-allocator" -import { FileIndexWriter } from "../file-completion" import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" -import { codeIndexLog } from "../code-index-log" import { ProviderPacerRegistry } from "../provider-pacer" +import { FileIndexWriter } from "../file-index-writer" import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number - private incompleteFiles = new Set() constructor( private readonly embedder: IEmbedder, @@ -39,13 +39,11 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, - private readonly writeGate = new IndexWriteGate(), - private readonly generations = new GenerationAllocator(cacheManager), - private readonly failures = new FileFailureRegistry(), - private readonly workspacePath?: string, - private readonly isCurrentAttempt: () => boolean = () => true, - private readonly providerPacers = new ProviderPacerRegistry(), + private readonly failures: FileFailureRegistry = new FileFailureRegistry(), + private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), ) { + // Get the configurable batch size from VSCode settings, fallback to default + // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -54,242 +52,182 @@ export class DirectoryScanner implements IDirectoryScanner { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { + // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } } - public get hasIncompleteFiles(): boolean { - return this.incompleteFiles.size > 0 || this.failures.list().length > 0 - } - + /** + * Recursively scans a directory, indexes new or changed files and removes entries for files that are gone. + * File-level failures are recorded in the failure registry and do not stop the scan. + */ public async scanDirectory( directory: string, onError?: (error: Error) => void, onBlocksIndexed?: (indexedCount: number) => void, onFileParsed?: (fileBlockCount: number) => void, - signal = new AbortController().signal, - ): Promise<{ - stats: { processed: number; skipped: number } - totalBlockCount: number - incompleteFiles?: string[] - }> { - const startedAt = Date.now() + signal?: AbortSignal, + ): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> { + const directoryPath = directory + // Capture workspace context at scan start + const scanWorkspace = getWorkspacePathForContext(directoryPath) + const writer = new FileIndexWriter( + this.embedder, + this.qdrantClient, + this.cacheManager, + this.failures, + this.pacers, + scanWorkspace, + this.batchSegmentThreshold, + ) + + // Get all files recursively (handles .gitignore automatically) + const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) + + // Filter out directories (marked with trailing '/') + const filePaths = allPaths.filter((p) => !p.endsWith("/")) + + // Initialize RooIgnoreController if not provided + const ignoreController = new RooIgnoreController(directoryPath) + + await ignoreController.initialize() + + // Filter paths using .rooignore + const allowedPaths = ignoreController.filterPaths(filePaths) + + // Filter by supported extensions, ignore patterns, and excluded directories + const supportedPaths = allowedPaths.filter((filePath) => { + const ext = path.extname(filePath).toLowerCase() + const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace) + + // Check if file is in an ignored directory using the shared helper + // Use relative path to avoid matching parent directories outside the workspace + if (isPathInIgnoredDirectory(relativeFilePath)) { + return false + } + + return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath) + }) + + const processedFiles = new Set() let processedCount = 0 let skippedCount = 0 let totalBlockCount = 0 - let indexedBlockCount = 0 - let partialReason: string | undefined - let fatalError: ParserFatalError | undefined - const checkActive = () => { - throwIfAborted(signal) - if (!this.isCurrentAttempt()) throw abortError() - if (fatalError) throw fatalError + + const parseLimiter = pLimit(PARSING_CONCURRENCY) + const writeLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) + const writes: Promise[] = [] + + const recordLocalFailure = (filePath: string, revision: string, activity: FileFailureActivity, error: unknown) => { + if (signal?.aborted) return + const relativePath = generateRelativeFilePath(filePath, scanWorkspace) + const reason = error instanceof Error ? error.message : String(error) + this.failures.renew(relativePath, revision) + this.failures.record(relativePath, revision, activity, reason, 1, "exhausted") + this.reportError(error, `scanDirectory:${activity}`, filePath, scanWorkspace, onError) } - checkActive() - const failureBatch = codeIndexLog.createFailureBatch({ source: "scan" }) - const subscription = this.failures.onDidRecord((failure) => { - if (!signal.aborted && this.isCurrentAttempt()) { - failureBatch.record({ ...failure, willRetry: failure.state === "retrying" }) - } - }) - try { - const incompleteFiles = new Set() - this.incompleteFiles = incompleteFiles - const scanWorkspace = this.workspacePath ?? getWorkspacePathForContext(directory) - const [allPaths] = await listFiles(directory, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) - checkActive() - const ignoreController = new RooIgnoreController(directory) - await ignoreController.initialize() - checkActive() - const allowedPaths = ignoreController.filterPaths(allPaths.filter((filePath) => !filePath.endsWith("/"))) - const selectedPaths = new Set( - allowedPaths.filter((filePath) => { - const relativePath = generateRelativeFilePath(filePath, scanWorkspace) - return ( - !isPathInIgnoredDirectory(relativePath) && - scannerExtensions.includes(path.extname(filePath).toLowerCase()) && - !this.ignoreInstance.ignores(relativePath) - ) - }), - ) - // Discovery precedes dispatch so stopping during a read or parse still reports partial work. - for (const filePath of selectedPaths) incompleteFiles.add(filePath) - const trackedPaths = new Set([ - ...this.cacheManager.getTrackedPaths(), - ...this.failures.list().map((failure) => path.resolve(scanWorkspace, failure.path)), - ]) - for (const filePath of trackedPaths) { - if (!selectedPaths.has(filePath)) incompleteFiles.add(filePath) - } - const writer = new FileIndexWriter( - this.cacheManager, - this.qdrantClient, - this.writeGate, - this.generations, - scanWorkspace, - this.failures, - this.providerPacers, - ) - const reportError = (error: unknown, filePath: string) => { - checkActive() - if (error instanceof ParserFatalError) { - fatalError = error - throw error - } - if (error instanceof Error && error.name === "AbortError") throw error - const message = codeIndexLog.redact(error instanceof Error ? error.message : String(error)) - const relativePath = generateRelativeFilePath(filePath, scanWorkspace) - onError?.(new Error(`${message} (File: ${relativePath})`, { cause: error })) - } - const dropSelected = (filePath: string) => { - selectedPaths.delete(filePath) - if (!trackedPaths.has(filePath)) incompleteFiles.delete(filePath) - } - // A missing source falls into the removal phase; it is neither processed nor skipped. - const dropMissing = (filePath: string, relativePath: string) => { - codeIndexLog.debug("Source missing at read; treating as removed", { path: relativePath }) - dropSelected(filePath) - } - const processFile = async (filePath: string) => { - checkActive() - const relativePath = generateRelativeFilePath(filePath, scanWorkspace) - let revision = this.failures.getRevision(relativePath) ?? UNKNOWN_REVISION - this.failures.renew(relativePath, revision) - let activity: FileFailureActivity = "read" - let fileSignal = signal + const parsePromises = supportedPaths.map((filePath) => + parseLimiter(async () => { + if (signal?.aborted) return + + let content: string try { const stats = await readSource(stat(filePath)) - checkActive() - if (stats === SOURCE_MISSING) return dropMissing(filePath, relativePath) + // A file deleted after listing is a removal: leaving it out lets the removal phase delete it. + if (stats === SOURCE_MISSING) return if (stats.size > MAX_FILE_SIZE_BYTES) { - dropSelected(filePath) skippedCount++ return } const buffer = await readSource(vscode.workspace.fs.readFile(vscode.Uri.file(filePath))) - checkActive() - if (buffer === SOURCE_MISSING) return dropMissing(filePath, relativePath) - const content = Buffer.from(buffer).toString("utf-8") - const hash = createHash("sha256").update(content).digest("hex") - revision = hash - fileSignal = AbortSignal.any([signal, this.failures.renew(relativePath, hash)]) - this.writeGate.own(fileSignal, signal) - if ( - this.cacheManager.getHash(filePath) === hash && - !this.writeGate.needsReconciliation(filePath) && - !this.failures.has(relativePath) - ) { - incompleteFiles.delete(filePath) - skippedCount++ - return - } - activity = "embed" - this.failures.assertRetryable(relativePath, hash) - activity = "parse" - // Cancelling an active parse terminates the shared worker, so only the session may cancel it. - const blocks = await this.codeParser.parseFile(filePath, { - content, - fileHash: hash, - signal, - }) - throwIfAborted(fileSignal) - checkActive() - processedCount++ - const selectedBlockCount = blocks.filter((block) => block.content.trim()).length - onFileParsed?.(selectedBlockCount) - checkActive() - if (!this.embedder || !this.qdrantClient) return - totalBlockCount += selectedBlockCount - activity = "embed" - await writer.replace( - filePath, - hash, - blocks, - this.embedder, - fileSignal, - this.batchSegmentThreshold, - (count) => { - checkActive() - incompleteFiles.delete(filePath) - indexedBlockCount += count - onBlocksIndexed?.(count) - }, - signal, - ) + if (buffer === SOURCE_MISSING) return + content = Buffer.from(buffer).toString("utf-8") } catch (error) { - checkActive() - if ( - (activity === "read" || activity === "parse") && - !fileSignal.aborted && - !(error instanceof Error && error.name === "AbortError") - ) { - this.failures.record( - relativePath, - revision, - activity, - codeIndexLog.redact(error instanceof Error ? error.message : String(error)), - 1, - "exhausted", - ) - } - if (error instanceof ParserFatalError) reportError(error, filePath) - if (fileSignal.aborted) return - reportError(error, filePath) + processedFiles.add(filePath) + recordLocalFailure(filePath, UNKNOWN_REVISION, "read", error) + return } - } - const processPaths = async (paths: string[], process: (filePath: string) => Promise) => { - let nextPath = 0 - const worker = async () => { - while (nextPath < paths.length) { - checkActive() - const filePath = paths[nextPath++] - await process(filePath) - } + + const currentFileHash = createHash("sha256").update(content).digest("hex") + processedFiles.add(filePath) + if (this.cacheManager.getHash(filePath) === currentFileHash) { + skippedCount++ + return } - const concurrency = Math.min(PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, paths.length) - await Promise.all(Array.from({ length: concurrency }, worker)) - } - await processPaths([...selectedPaths], processFile) - checkActive() - const removedPaths = [...new Set([...trackedPaths, ...this.cacheManager.getTrackedPaths()])].filter( - (filePath) => !selectedPaths.has(filePath), - ) - for (const filePath of removedPaths) incompleteFiles.add(filePath) - await processPaths(removedPaths, async (filePath) => { + let blocks try { - checkActive() - if (!this.qdrantClient) return - await writer.remove(filePath, signal) - checkActive() - incompleteFiles.delete(filePath) + blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash }) } catch (error) { - reportError(error, filePath) + // Covers ParserLoadError: the hash stays unstored so Start Indexing retries the unchanged file. + recordLocalFailure(filePath, currentFileHash, "parse", error) + return + } + onFileParsed?.(blocks.length) + processedCount++ + if (!this.embedder || !this.qdrantClient) { + this.cacheManager.updateHash(filePath, currentFileHash) + return } - }) - checkActive() - return { - stats: { processed: processedCount, skipped: skippedCount }, - totalBlockCount, - incompleteFiles: [...incompleteFiles], + totalBlockCount += blocks.length + + writes.push( + writeLimiter(async () => { + if (signal?.aborted) return + const indexed = await writer.replace(filePath, currentFileHash, blocks, signal) + if (indexed) onBlocksIndexed?.(blocks.filter((block) => block.content.trim()).length) + }), + ) + }), + ) + + await Promise.all(parsePromises) + await Promise.all(writes) + + if (signal?.aborted) { + return { stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount } + } + + const oldHashes = this.cacheManager.getAllHashes() + for (const cachedFilePath of Object.keys(oldHashes)) { + if (processedFiles.has(cachedFilePath)) continue + if (signal?.aborted) break + const removed = await writer.remove(cachedFilePath, signal) + if (!removed && !signal?.aborted) { + this.reportError( + new Error(t("embeddings:scanner.unknownErrorDeletingPoints", { filePath: cachedFilePath })), + "scanDirectory:deleteRemovedFiles", + cachedFilePath, + scanWorkspace, + onError, + ) } - } catch (error) { - partialReason = error instanceof Error ? error.message : String(error) - throw error - } finally { - subscription.dispose() - failureBatch.flush() - codeIndexLog.info("Scan summary", { - processed: processedCount, - skipped: skippedCount, - failed: this.failures.list().length, - blocksFound: totalBlockCount, - blocksIndexed: indexedBlockCount, - elapsedMs: Date.now() - startedAt, - partialReason: partialReason ?? (this.hasIncompleteFiles ? "Unresolved files" : undefined), - }) } + + return { stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount } + } + + private reportError( + error: unknown, + location: string, + filePath: string, + scanWorkspace: string, + onError?: (error: Error) => void, + ): void { + console.error(`Error processing file ${filePath} in workspace ${scanWorkspace}:`, error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), + stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, + location, + }) + onError?.( + error instanceof Error + ? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`) + : new Error( + t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) + ` (Workspace: ${scanWorkspace})`, + ), + ) } } diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index 0ee62730b9..b54d47add6 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -8,17 +8,14 @@ import { RooIgnoreController } from "../../core/ignore/RooIgnoreController" import { EmbedderFactory } from "./embedders/embedder-factory" import { EmbedderValidationManager } from "./embedders/embedder-validation-manager" import { VectorStoreFactory } from "./vector-store/vector-store-factory" -import type { DirectoryScanner } from "./processors/scanner" -import { ParseWorkerClient } from "./parse-worker-client" -import { IndexWriteGate } from "./index-write-gate" -import { GenerationAllocator } from "./generation-allocator" -import { FileFailureRegistry } from "./file-failure-registry" -import { ProviderPacerRegistry } from "./provider-pacer" +import { codeParser, DirectoryScanner } from "./processors" import { DirectoryScannerFactory } from "./processors/directory-scanner-factory" import { FileWatcherFactory } from "./processors/file-watcher-factory" -import type { IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" +import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" +import { FileFailureRegistry } from "./file-failure-registry" +import { ProviderPacerRegistry } from "./provider-pacer" /** * Factory class responsible for creating and configuring code indexing service dependencies. @@ -37,12 +34,8 @@ export class CodeIndexServiceFactory { private readonly configManager: CodeIndexConfigManager, private readonly workspacePath: string, private readonly cacheManager: CacheManager, - private readonly writeGate = new IndexWriteGate(), - private readonly generations = new GenerationAllocator(cacheManager), - private readonly failures = new FileFailureRegistry(), - private readonly parser = new ParseWorkerClient(), - private readonly isCurrentAttempt: () => boolean = () => true, - private readonly providerPacers = new ProviderPacerRegistry(), + private readonly failureRegistry: FileFailureRegistry = new FileFailureRegistry(), + private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), ) {} /** @@ -68,7 +61,7 @@ export class CodeIndexServiceFactory { ): { embedder: IEmbedder vectorStore: IVectorStore - parser: ParseWorkerClient + parser: ICodeParser scanner: DirectoryScanner fileWatcher: IFileWatcher } { @@ -77,21 +70,17 @@ export class CodeIndexServiceFactory { } const config = this.configManager.getConfig() - const embedder = this.embedderFactory.create(config) - const vectorStore = this.vectorStoreFactory.create(config, this.workspacePath) - const parser = this.parser + const embedder = this.embedderFactory.create(config, this.pacers) + const vectorStore = this.vectorStoreFactory.create(config, this.workspacePath, this.pacers) + const parser = codeParser const scanner = this.directoryScannerFactory.create({ embedder, vectorStore, parser, cacheManager: this.cacheManager, ignoreInstance, - writeGate: this.writeGate, - generations: this.generations, - failures: this.failures, - workspacePath: this.workspacePath, - isCurrentAttempt: this.isCurrentAttempt, - providerPacers: this.providerPacers, + failureRegistry: this.failureRegistry, + pacers: this.pacers, }) const fileWatcher = this.fileWatcherFactory.create({ workspacePath: this.workspacePath, @@ -101,12 +90,8 @@ export class CodeIndexServiceFactory { cacheManager, ignoreInstance, rooIgnoreController, - parser, - writeGate: this.writeGate, - generations: this.generations, - failures: this.failures, - isCurrentAttempt: this.isCurrentAttempt, - providerPacers: this.providerPacers, + failureRegistry: this.failureRegistry, + pacers: this.pacers, }) return { From 850a2cc832fc692af27512fb6875c0ae40c94e91 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 13:43:39 -0800 Subject: [PATCH 22/42] fix(code-index): single-attempt providers with rate-limit scopes; Bedrock and Qdrant creation retries; search without task signal --- src/core/tools/CodebaseSearchTool.ts | 2 +- .../__tests__/CodebaseSearchTool.spec.ts | 46 +-- .../CodebaseSearchTool.workspace.spec.ts | 17 +- .../webview/__tests__/ClineProvider.spec.ts | 288 ++----------- src/core/webview/webviewMessageHandler.ts | 45 +- src/i18n/locales/ca/embeddings.json | 8 +- src/i18n/locales/de/embeddings.json | 8 +- src/i18n/locales/en/embeddings.json | 8 +- src/i18n/locales/es/embeddings.json | 8 +- src/i18n/locales/fr/embeddings.json | 8 +- src/i18n/locales/hi/embeddings.json | 8 +- src/i18n/locales/id/embeddings.json | 8 +- src/i18n/locales/it/embeddings.json | 8 +- src/i18n/locales/ja/embeddings.json | 8 +- src/i18n/locales/ko/embeddings.json | 8 +- src/i18n/locales/nl/embeddings.json | 8 +- src/i18n/locales/pl/embeddings.json | 8 +- src/i18n/locales/pt-BR/embeddings.json | 8 +- src/i18n/locales/ru/embeddings.json | 8 +- src/i18n/locales/tr/embeddings.json | 8 +- src/i18n/locales/vi/embeddings.json | 8 +- src/i18n/locales/zh-CN/embeddings.json | 8 +- src/i18n/locales/zh-TW/embeddings.json | 8 +- .../__tests__/search-service.spec.ts | 386 ++++-------------- ...kspace-indexing-enablement-manager.spec.ts | 53 +-- .../__tests__/bedrock-pacing.spec.ts | 70 ---- .../embedders/__tests__/bedrock.spec.ts | 77 ++-- .../embedders/__tests__/cancellation.spec.ts | 261 ------------ .../__tests__/embedder-factory.spec.ts | 26 ++ .../embedder-validation-manager.spec.ts | 9 - .../openai-compatible-rate-limit.spec.ts | 1 - .../__tests__/openai-compatible.spec.ts | 14 +- .../embedders/__tests__/openai.spec.ts | 14 +- .../embedders/__tests__/openrouter.spec.ts | 6 +- .../__tests__/paced-transport-writer.ts | 82 ---- .../__tests__/rate-limit-delay.spec.ts | 130 ------ .../__tests__/rate-limit-scopes.spec.ts | 71 ---- src/services/code-index/embedders/bedrock.ts | 93 ++--- .../code-index/embedders/embedder-factory.ts | 5 +- .../embedders/embedder-validation-manager.ts | 2 - .../factories/bedrock-embedder-factory.ts | 4 +- src/services/code-index/embedders/gemini.ts | 15 +- src/services/code-index/embedders/mistral.ts | 15 +- src/services/code-index/embedders/ollama.ts | 73 ++-- .../code-index/embedders/openai-compatible.ts | 94 ++--- src/services/code-index/embedders/openai.ts | 44 +- .../code-index/embedders/openrouter.ts | 78 ++-- .../code-index/embedders/vercel-ai-gateway.ts | 19 +- .../code-index/interfaces/embedder-factory.ts | 3 +- .../code-index/interfaces/embedder.ts | 7 +- .../interfaces/vector-store-factory.ts | 3 +- .../code-index/interfaces/vector-store.ts | 21 +- src/services/code-index/manager.ts | 3 + .../shared/__tests__/indexing-errors.spec.ts | 6 +- .../__tests__/validation-helpers.spec.ts | 101 +---- .../code-index/shared/validation-helpers.ts | 33 +- .../qdrant-client.http-pacing.spec.ts | 178 -------- .../qdrant-client.rate-limit.spec.ts | 153 ++----- .../qdrant-client.retry-contract.spec.ts | 205 ---------- .../__tests__/qdrant-client.spec.ts | 102 ++--- .../code-index/vector-store/qdrant-client.ts | 295 ++++++------- .../vector-store/vector-store-factory.ts | 5 +- .../workspace-indexing-enablement-manager.ts | 4 +- .../chat/__tests__/CodeIndexPopover.test.tsx | 4 +- webview-ui/src/i18n/locales/ca/settings.json | 3 +- webview-ui/src/i18n/locales/de/settings.json | 3 +- webview-ui/src/i18n/locales/en/settings.json | 3 +- webview-ui/src/i18n/locales/es/settings.json | 3 +- webview-ui/src/i18n/locales/fr/settings.json | 3 +- webview-ui/src/i18n/locales/hi/settings.json | 3 +- webview-ui/src/i18n/locales/id/settings.json | 3 +- webview-ui/src/i18n/locales/it/settings.json | 3 +- webview-ui/src/i18n/locales/ja/settings.json | 3 +- webview-ui/src/i18n/locales/ko/settings.json | 3 +- webview-ui/src/i18n/locales/nl/settings.json | 3 +- webview-ui/src/i18n/locales/pl/settings.json | 3 +- .../src/i18n/locales/pt-BR/settings.json | 3 +- webview-ui/src/i18n/locales/ru/settings.json | 3 +- webview-ui/src/i18n/locales/tr/settings.json | 3 +- webview-ui/src/i18n/locales/vi/settings.json | 3 +- .../src/i18n/locales/zh-CN/settings.json | 3 +- .../src/i18n/locales/zh-TW/settings.json | 3 +- 82 files changed, 836 insertions(+), 2525 deletions(-) delete mode 100644 src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts delete mode 100644 src/services/code-index/embedders/__tests__/cancellation.spec.ts create mode 100644 src/services/code-index/embedders/__tests__/embedder-factory.spec.ts delete mode 100644 src/services/code-index/embedders/__tests__/paced-transport-writer.ts delete mode 100644 src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts delete mode 100644 src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts delete mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.http-pacing.spec.ts delete mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts diff --git a/src/core/tools/CodebaseSearchTool.ts b/src/core/tools/CodebaseSearchTool.ts index 1a69b9153c..ee5f52efa8 100644 --- a/src/core/tools/CodebaseSearchTool.ts +++ b/src/core/tools/CodebaseSearchTool.ts @@ -82,7 +82,7 @@ export class CodebaseSearchTool extends BaseTool<"codebase_search"> { incomplete, failedPaths, waitingPaths, - } = await manager.searchIndex(query, directoryPrefix, task.lifecycleSignal) + } = await manager.searchIndex(query, directoryPrefix) const incompleteNotice = incomplete ? [ "Partial index: results may be incomplete.", diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts index 739b4068dc..45decee1c6 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -46,7 +46,6 @@ describe("CodebaseSearchTool", () => { Task, | "cwd" | "providerRef" - | "lifecycleSignal" | "consecutiveMistakeCount" | "didToolFailInCurrentTurn" | "sayAndCreateMissingParamError" @@ -55,7 +54,6 @@ describe("CodebaseSearchTool", () => { > = { cwd: "/task", providerRef: { deref, [Symbol.toStringTag]: "WeakRef" }, - lifecycleSignal: new AbortController().signal, consecutiveMistakeCount: 3, didToolFailInCurrentTurn: false, sayAndCreateMissingParamError: vi @@ -279,7 +277,7 @@ describe("CodebaseSearchTool", () => { }) await tool.execute({ query, path }, task, callbacks) expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, path, task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, path) expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( `No relevant code snippets found for the query: "${query}"`, ) @@ -305,7 +303,7 @@ describe("CodebaseSearchTool", () => { await tool.execute({ query, path: "src" }, task, callbacks) - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src") expect(task.say).toHaveBeenCalledExactlyOnceWith( "codebase_search_result", JSON.stringify({ @@ -356,7 +354,7 @@ describe("CodebaseSearchTool", () => { .mockReturnValueOnce("lib/other.ts") await tool.execute({ query }, task, callbacks) expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined, task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined) expect(vscode.workspace.asRelativePath).toHaveBeenCalledTimes(2) expect(vscode.workspace.asRelativePath).toHaveBeenNthCalledWith(1, "/task/src/result.ts", false) expect(vscode.workspace.asRelativePath).toHaveBeenNthCalledWith(2, "/task/lib/other.ts", false) @@ -469,7 +467,7 @@ describe("CodebaseSearchTool", () => { await tool.execute({ query }, task, callbacks) - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined, task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, undefined) expect(callbacks.handleError).toHaveBeenCalledExactlyOnceWith(toolNamesSchema.enum.codebase_search, error) expect(vi.mocked(callbacks.handleError).mock.calls[0][1]).toBe(error) expect(callbacks.pushToolResult).not.toHaveBeenCalled() @@ -477,40 +475,6 @@ describe("CodebaseSearchTool", () => { expect(task.say).not.toHaveBeenCalled() }) - it.each(["already aborted", "midflight"] as const)("preserves the exact %s cancellation reason", async (timing) => { - const controller = new AbortController() - const reason = Object.freeze({ cancelled: timing }) - Object.defineProperty(task, "lifecycleSignal", { value: controller.signal }) - if (timing === "already aborted") controller.abort(reason) - vi.mocked(manager.searchIndex).mockImplementation(async (_query, _path, signal) => { - expect(signal).toBe(controller.signal) - if (!signal) throw new Error("Missing lifecycle signal") - signal.throwIfAborted() - return new Promise>>((_resolve, reject) => { - const onAbort = () => { - clearTimeout(deadline) - reject(signal.reason) - } - const deadline = setTimeout(() => { - signal.removeEventListener("abort", onAbort) - reject(new Error("Search cancellation was not observed")) - }, 1_000) - signal.addEventListener("abort", onAbort, { once: true }) - queueMicrotask(() => controller.abort(reason)) - }) - }) - - await tool.execute({ query, path: "src" }, task, callbacks) - - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", controller.signal) - expect(controller.signal.reason).toBe(reason) - expect(callbacks.handleError).toHaveBeenCalledExactlyOnceWith(toolNamesSchema.enum.codebase_search, reason) - expect(vi.mocked(callbacks.handleError).mock.calls[0][1]).toBe(reason) - expect(callbacks.pushToolResult).not.toHaveBeenCalled() - expect(vscode.workspace.asRelativePath).not.toHaveBeenCalled() - expect(task.say).not.toHaveBeenCalled() - }) - it("forwards a result-message rejection without publishing a text result", async () => { const error = new Error("say failed") vi.mocked(manager.searchIndex).mockResolvedValue(searchResponse([result()])) @@ -550,7 +514,7 @@ describe("CodebaseSearchTool", () => { } expect(CodeIndexManagerRegistry.getOrCreate).toHaveBeenCalledExactlyOnceWith(context, "/task") - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src", task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith(query, "src") expect(task.consecutiveMistakeCount).toBe(0) expect(callbacks.handleError).not.toHaveBeenCalled() }) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts index b4ae44fd8f..808afa4916 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts @@ -81,9 +81,8 @@ describe("CodebaseSearchTool workspace selection", () => { vi.mocked(getWorkspacePath).mockReturnValue(first.uri.fsPath) // Only the provider context and task members consumed by this tool are needed. provider = { context: makeExtensionContext(), contextProxy: {} } as ClineProvider - const taskStub: Pick = { + const taskStub: Pick = { cwd: second.uri.fsPath, - lifecycleSignal: new AbortController().signal, providerRef: new WeakRef(provider), consecutiveMistakeCount: 0, say: vi.fn().mockResolvedValue(undefined), @@ -113,7 +112,7 @@ describe("CodebaseSearchTool workspace selection", () => { await new CodebaseSearchTool().execute({ query: "find match", path: "src" }, task, callbacks) expect(callbacks.handleError).not.toHaveBeenCalled() - expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("find match", "src", task.lifecycleSignal) + expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("find match", "src") expect(taskManager.initialize).not.toHaveBeenCalled() expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() @@ -147,7 +146,7 @@ describe("CodebaseSearchTool workspace selection", () => { await new CodebaseSearchTool().execute({ query: "fallback" }, task, callbacks) expect(getWorkspacePath).toHaveBeenCalledExactlyOnceWith() - expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("fallback", undefined, task.lifecycleSignal) + expect(taskManager.searchIndex).toHaveBeenCalledExactlyOnceWith("fallback", undefined) expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(callbacks.handleError).not.toHaveBeenCalled() expect(callbacks.pushToolResult).toHaveBeenCalledExactlyOnceWith( @@ -194,16 +193,14 @@ describe("CodebaseSearchTool workspace selection", () => { expect(taskManager.searchIndex).toHaveBeenNthCalledWith( 1, "before editor switch", - undefined, - task.lifecycleSignal, + undefined ) expect(taskManager.searchIndex).toHaveBeenNthCalledWith( 2, "after editor switch", - undefined, - task.lifecycleSignal, + undefined ) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith(3, "without an editor", undefined, task.lifecycleSignal) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith(3, "without an editor", undefined) expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() expect(callbacks.handleError).not.toHaveBeenCalled() @@ -245,7 +242,7 @@ describe("CodebaseSearchTool workspace selection", () => { expect(manager.state).toBe("Indexing") expect(manager.initialize).not.toHaveBeenCalled() - expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith("external match", undefined, task.lifecycleSignal) + expect(manager.searchIndex).toHaveBeenCalledExactlyOnceWith("external match", undefined) expect(callbacks.handleError).not.toHaveBeenCalled() }) }) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index d86bc1f4d7..464889418e 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -31,7 +31,6 @@ import { experimentDefault } from "../../../shared/experiments" import { setTtsEnabled } from "../../../utils/tts" import { ContextProxy } from "../../config/ContextProxy" import { WorkspaceIndexingEnablementManager } from "../../../services/code-index/workspace-indexing-enablement-manager" -import type { CodeIndexManager } from "../../../services/code-index/manager" import { Task, TaskOptions } from "../../task/Task" import { safeWriteJson } from "../../../utils/safeWriteJson" @@ -5814,23 +5813,10 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { initialize: vi.fn().mockResolvedValue(undefined), state: "Standby", isInitialized: true, - startIndexing: vi.fn().mockResolvedValue(undefined), - stopIndexing: vi.fn().mockResolvedValue(undefined), - recoverFromError: vi.fn().mockResolvedValue(undefined), + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn(), clearIndexData: vi.fn().mockResolvedValue(undefined), - getCurrentStatus: vi.fn().mockReturnValue({ - systemStatus: "Standby", - message: "Ready", - processedItems: 0, - totalItems: 0, - currentItemUnit: "blocks", - workspacePath: "/test/workspace", - workspaceEnabled: true, - autoEnableDefault: false, - failedFiles: [], - failedFileSummary: undefined, - providerWaits: [], - }), + getCurrentStatus: vi.fn().mockReturnValue({ systemStatus: "Standby" }), }, overrides, ) @@ -5839,51 +5825,27 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { vi.clearAllMocks() }) - const deferred = () => { - let resolve!: () => void - let reject!: (error: Error) => void - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise + it("logs a detached indexing rejection without rejecting the handler", async () => { + let rejectIndexing!: (error: Error) => void + const indexingPromise = new Promise((_, reject) => { + rejectIndexing = reject }) - return { promise, resolve, reject } - } - - it.each(["resolve", "reject"] as const)("awaits explicit indexing startup before %s", async (outcome) => { - const startup = deferred() - const entered = deferred() - const manager = createIndexManager() - manager.startIndexing.mockImplementation(() => { - entered.resolve() - return startup.promise + const manager = createIndexManager({ + startIndexing: vi.fn().mockReturnValue(indexingPromise), }) const provider = createProvider({ getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), }) - let completed = false - const pending = webviewMessageHandler(provider, { type: "startIndexing" }).then(() => { - completed = true - }) - await entered.promise - await Promise.resolve() - await Promise.resolve() - expect(completed).toBe(false) + await expect(webviewMessageHandler(provider, { type: "startIndexing" })).resolves.toBeUndefined() expect(provider.getCurrentWorkspaceCodeIndexScope).toHaveBeenCalledOnce() - expect(manager.setWorkspaceEnabled).toHaveBeenCalledExactlyOnceWith(true) - expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.startIndexing) - expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) - expect(manager.initialize).not.toHaveBeenCalled() - expect(manager.recoverFromError).not.toHaveBeenCalled() - expect(provider.log).not.toHaveBeenCalled() - if (outcome === "reject") startup.reject(new Error("boom")) - else startup.resolve() - await pending - - expect(completed).toBe(true) expect(manager.startIndexing).toHaveBeenCalledOnce() - if (outcome === "reject") expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error starting indexing: boom") - else expect(provider.log).not.toHaveBeenCalled() + + rejectIndexing(new Error("boom")) + await Promise.resolve() + await Promise.resolve() + + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: boom") }) it("covers the changed task-operation happy paths", async () => { @@ -6039,15 +6001,7 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(manager.getCurrentStatus).not.toHaveBeenCalled() expect(provider.postMessageToWebview).toHaveBeenCalledWith({ type: "indexingStatusUpdate", - values: { - systemStatus: "Error", - message: expect.any(String), - processedItems: 0, - totalItems: 0, - currentItemUnit: "items", - workspacePath: undefined, - failedFiles: [], - }, + values: expect.objectContaining({ systemStatus: "Error", message: expect.any(String) }), }) await webviewMessageHandler(provider, { type: "requestIndexingStatus" }) expect(getScope).toHaveBeenCalledTimes(2) @@ -6068,108 +6022,27 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.log).toHaveBeenCalledWith("Cannot start indexing: No workspace folder open") }) - it.each(["requestIndexingStatus", "stopIndexing"] as const)( - "forwards the whole computed status snapshot for %s", - async (type) => { - const status: ReturnType = { - systemStatus: "Standby", - message: "Partial index: 2 files failed", - processedItems: 7, - totalItems: 9, - currentItemUnit: "files", - workspacePath: "/snapshot/workspace", - workspaceEnabled: false, - autoEnableDefault: true, - failedFiles: [ - { path: "src/first.ts", activity: "embed", reason: "Provider unavailable", state: "exhausted" }, - { path: "src/second.ts", activity: "verify", reason: "Verification failed", state: "exhausted" }, - ], - failedFileSummary: { - path: "src/first.ts", - activity: "embed", - reason: "Provider unavailable", - state: "exhausted", - count: 2, - }, - providerWaits: [{ service: "embedding", resumeAt: 123_000, waitingFiles: ["src/waiting.ts"] }], - } - const manager = createIndexManager() - manager.getCurrentStatus.mockReturnValue(status) - const provider = createProvider({ - getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), - }) - - await webviewMessageHandler(provider, { type }) - - expect(manager.getCurrentStatus).toHaveBeenCalledOnce() - expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ - type: "indexingStatusUpdate", - values: status, - }) - const [message] = vi.mocked(provider.postMessageToWebview).mock.calls[0] - expect(message.values).toBe(status) - }, - ) - - it.each(["resolve", "reject"] as const)("awaits stop completion before %s", async (outcome) => { - const stopping = deferred() - const entered = deferred() - const manager = createIndexManager() - manager.stopIndexing.mockImplementation(() => { - entered.resolve() - return stopping.promise + it("catches both start-indexing calls during error recovery", async () => { + const manager = createIndexManager({ + isInitialized: false, + startIndexing: vi + .fn() + .mockRejectedValueOnce(new Error("first failure")) + .mockRejectedValueOnce(new Error("second failure")), }) const provider = createProvider({ getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), }) - let completed = false - const pending = webviewMessageHandler(provider, { type: "stopIndexing" }).then(() => { - completed = true - }) - await entered.promise - await Promise.resolve() - await Promise.resolve() - - expect(completed).toBe(false) - expect(manager.stopIndexing).toHaveBeenCalledExactlyOnceWith() - expect(manager.getCurrentStatus).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - expect(provider.log).not.toHaveBeenCalled() - if (outcome === "reject") stopping.reject(new Error("stop failure")) - else stopping.resolve() - await pending - - expect(completed).toBe(true) - if (outcome === "reject") { - expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error stopping indexing: stop failure") - expect(manager.getCurrentStatus).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - } else { - expect(manager.getCurrentStatus).toHaveBeenCalledOnce() - expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ - type: "indexingStatusUpdate", - values: manager.getCurrentStatus.mock.results[0].value, - }) - expect(provider.log).not.toHaveBeenCalled() - } - }) - - it("starts once from an uninitialized error without separate initialization or recovery", async () => { - const manager = createIndexManager({ isInitialized: false, state: "Error" }) - manager.startIndexing.mockRejectedValue(new Error("startup failure")) - const provider = createProvider({ - getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), - }) await webviewMessageHandler(provider, { type: "startIndexing" }) + await Promise.resolve() - expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) - expect(manager.initialize).not.toHaveBeenCalled() - expect(manager.recoverFromError).not.toHaveBeenCalled() - expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error starting indexing: startup failure") + expect(manager.startIndexing).toHaveBeenCalledTimes(2) + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: first failure") + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: second failure") }) - it("logs workspace-toggle startup failures without publishing a successful toggle status", async () => { + it("covers changed stop, toggle, and detached toggle rejection paths", async () => { const manager = createIndexManager({ startIndexing: vi.fn().mockRejectedValue(new Error("toggle failure")), }) @@ -6187,11 +6060,13 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { values: manager.getCurrentStatus(), }) await webviewMessageHandler(provider, { type: "toggleWorkspaceIndexing", bool: true }) + await Promise.resolve() + expect(manager.stopIndexing).toHaveBeenCalledOnce() - expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) - expect(manager.initialize).not.toHaveBeenCalled() - expect(provider.log).toHaveBeenCalledExactlyOnceWith("Error toggling workspace indexing: toggle failure") - expect(provider.postMessageToWebview).toHaveBeenCalledOnce() + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: toggle failure") + expect(provider.postMessageToWebview).toHaveBeenCalledWith( + expect.objectContaining({ type: "indexingStatusUpdate" }), + ) }) it("does not toggle indexing or publish status without a workspace scope", async () => { @@ -6273,28 +6148,16 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.postMessageToWebview).not.toHaveBeenCalled() }) - it.each([ - { enabled: true, outcome: "resolve" }, - { enabled: true, outcome: "reject" }, - { enabled: false, outcome: "resolve" }, - { enabled: false, outcome: "reject" }, - ] as const)("awaits auto-enable enabled=$enabled with $outcome", async ({ enabled, outcome }) => { + it("catches auto-enabled indexing failures and posts the resulting status", async () => { const { CodeIndexManagerRegistry } = await import("../../../services/code-index/code-index-manager-registry") - const operation = deferred() - const entered = deferred() - let workspaceEnabled = !enabled + let workspaceEnabled = false const manager = createIndexManager({ setAutoEnableDefault: vi.fn().mockImplementation(async () => { - workspaceEnabled = enabled + workspaceEnabled = true }), - }) - const indexingOperation = enabled ? manager.startIndexing : manager.stopIndexing - indexingOperation.mockImplementation(() => { - entered.resolve() - return operation.promise + startIndexing: vi.fn().mockRejectedValue(new Error("auto-enable failure")), }) Object.defineProperty(manager, "isWorkspaceEnabled", { get: () => workspaceEnabled }) - // The registry double provides the enablement, startup, stop, and status members used by this handler. const getAllInstances = vi .spyOn(CodeIndexManagerRegistry, "getAllInstances") .mockReturnValue([manager] as unknown as ReturnType) @@ -6303,77 +6166,14 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { }) try { - let completed = false - const pending = webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: enabled }).then( - () => { - completed = true - }, - ) - await entered.promise - await Promise.resolve() + await webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: true }) await Promise.resolve() - expect(completed).toBe(false) expect(provider.getCurrentWorkspaceCodeIndexScope).toHaveBeenCalledOnce() - expect(manager.setAutoEnableDefault).toHaveBeenCalledExactlyOnceWith(enabled) - expect(manager.initialize).not.toHaveBeenCalled() - expect(manager.recoverFromError).not.toHaveBeenCalled() - expect(manager.getCurrentStatus).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - if (enabled) { - expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) - expect(manager.stopIndexing).not.toHaveBeenCalled() - } else { - expect(manager.stopIndexing).toHaveBeenCalledExactlyOnceWith() - expect(manager.startIndexing).not.toHaveBeenCalled() - } - if (outcome === "reject") operation.reject(new Error("auto-enable failure")) - else operation.resolve() - await pending - - expect(completed).toBe(true) - expect(indexingOperation).toHaveBeenCalledOnce() - if (outcome === "reject") { - expect(provider.log).toHaveBeenCalledExactlyOnceWith( - enabled - ? "Error starting auto-enabled indexing: auto-enable failure" - : "Error stopping auto-disabled indexing: auto-enable failure", - ) - } else expect(provider.log).not.toHaveBeenCalled() - expect(manager.getCurrentStatus).toHaveBeenCalledOnce() - expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ - type: "indexingStatusUpdate", - values: manager.getCurrentStatus.mock.results[0].value, - }) - } finally { - getAllInstances.mockRestore() - } - }) - - it("keeps stopping other workspaces after one auto-disabled stop fails", async () => { - const { CodeIndexManagerRegistry } = await import("../../../services/code-index/code-index-manager-registry") - let workspaceEnabled = true - const managers = [0, 1].map(() => createIndexManager()) - managers[0].setAutoEnableDefault.mockImplementation(async () => { - workspaceEnabled = false - }) - managers[0].stopIndexing.mockRejectedValue(new Error("first stop failed")) - for (const manager of managers) { - Object.defineProperty(manager, "isWorkspaceEnabled", { get: () => workspaceEnabled }) - } - // The registry double provides the enablement, stop, and status members used by this handler. - const getAllInstances = vi - .spyOn(CodeIndexManagerRegistry, "getAllInstances") - .mockReturnValue(managers as unknown as ReturnType) - const provider = createProvider({ - getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: managers[0] }), - }) - try { - await webviewMessageHandler(provider, { type: "setAutoEnableDefault", bool: false }) - expect(managers[0].stopIndexing).toHaveBeenCalledOnce() - expect(managers[1].stopIndexing).toHaveBeenCalledOnce() - expect(provider.log).toHaveBeenCalledExactlyOnceWith( - "Error stopping auto-disabled indexing: first stop failed", + expect(manager.startIndexing).toHaveBeenCalledOnce() + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: auto-enable failure") + expect(provider.postMessageToWebview).toHaveBeenCalledWith( + expect.objectContaining({ type: "indexingStatusUpdate" }), ) } finally { getAllInstances.mockRestore() diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index b7e0e23a41..590d5a8acc 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3452,14 +3452,22 @@ export const webviewMessageHandler = async ( processedItems: 0, totalItems: 0, currentItemUnit: "items", - workspacePath: undefined, - failedFiles: [], + workerspacePath: undefined, }, }) return } - const status = manager.getCurrentStatus() + const status = manager + ? manager.getCurrentStatus() + : { + systemStatus: "Standby", + message: "No workspace folder open", + processedItems: 0, + totalItems: 0, + currentItemUnit: "items", + workspacePath: undefined, + } await provider.postMessageToWebview({ type: "indexingStatusUpdate", @@ -3518,7 +3526,19 @@ export const webviewMessageHandler = async ( await manager.setWorkspaceEnabled(true) if (manager.isFeatureEnabled && manager.isFeatureConfigured) { - await manager.startIndexing(provider.contextProxy) + await manager.initialize(provider.contextProxy) + + const currentState = manager.state + if (currentState === "Standby" || currentState === "Error") { + void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + + if (!manager.isInitialized) { + await manager.initialize(provider.contextProxy) + if (manager.state === "Standby" || manager.state === "Error") { + void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + } + } + } } } catch (error) { provider.log(`Error starting indexing: ${error instanceof Error ? error.message : String(error)}`) @@ -3574,21 +3594,10 @@ export const webviewMessageHandler = async ( const wasEnabled = priorStates.get(m)! const isNowEnabled = m.isWorkspaceEnabled if (wasEnabled && !isNowEnabled) { - try { - await m.stopIndexing() - } catch (error) { - provider.log( - `Error stopping auto-disabled indexing: ${error instanceof Error ? error.message : String(error)}`, - ) - } + await m.stopIndexing() } else if (!wasEnabled && isNowEnabled && m.isFeatureEnabled && m.isFeatureConfigured) { - try { - await m.startIndexing(provider.contextProxy) - } catch (error) { - provider.log( - `Error starting auto-enabled indexing: ${error instanceof Error ? error.message : String(error)}`, - ) - } + await m.initialize(provider.contextProxy) + void m.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) } } await provider.postMessageToWebview({ diff --git a/src/i18n/locales/ca/embeddings.json b/src/i18n/locales/ca/embeddings.json index 6e856bfd19..87045e30f7 100644 --- a/src/i18n/locales/ca/embeddings.json +++ b/src/i18n/locales/ca/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Esperant {{service}}; la indexació es reprèn automàticament", "providerWaitingUntil": "Esperant {{service}}; la indexació es reprèn automàticament a les {{time}}", - "parserFailed": "La indexació del codi s'ha aturat en analitzar {{filePath}} ({{category}}). Corregeix el fitxer i inicia explícitament la indexació per tornar-ho a provar, o desactiva-la. No és un error del model ni del proveïdor.", - "indexingFilesRetrying": "S'estan reintentant els fitxers amb errors. Hi ha un índex parcial disponible.", "indexingFilesFailed": "Encara hi ha errors en fitxers. No hi ha reintents automàtics pendents. Hi ha un índex parcial disponible.", "indexingStopping": "S'està aturant la indexació...", "indexingFailedNoBlocks": "Indexació fallida: No s'ha indexat cap bloc de codi amb èxit. Això normalment indica un problema de configuració de l'embedder.", @@ -71,7 +69,8 @@ "unknownError": "Error desconegut", "indexingRequiresWorkspace": "Indexació requereix una carpeta de workspace oberta", "indexingStopped": "Indexació aturada per l'usuari.", - "indexingStoppedPartial": "Indexació aturada. Dades d'índex parcials conservades." + "indexingStoppedPartial": "Indexació aturada. Dades d'índex parcials conservades.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Descarregant el binari de semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Error en descarregar semble: {{errorMessage}}", "checkFailed": "Comprovació de semble fallida: {{errorMessage}}", "providerReset": "Proveïdor de Semble restablert. La memòria cau al disc es conserva fins a la propera reconstrucció." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/de/embeddings.json b/src/i18n/locales/de/embeddings.json index 87b9462f2d..a9891c4952 100644 --- a/src/i18n/locales/de/embeddings.json +++ b/src/i18n/locales/de/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Warten auf {{service}}; die Indexierung wird automatisch fortgesetzt", "providerWaitingUntil": "Warten auf {{service}}; die Indexierung wird um {{time}} automatisch fortgesetzt", - "parserFailed": "Die Code-Indexierung wurde beim Parsen von {{filePath}} ({{category}}) gestoppt. Behebe den Fehler in der Datei und starte die Indexierung ausdrücklich erneut, oder deaktiviere sie. Dies ist kein Modell- oder Anbieterfehler.", - "indexingFilesRetrying": "Fehlgeschlagene Dateien werden erneut versucht. Ein Teilindex ist verfügbar.", "indexingFilesFailed": "Es bestehen weiterhin Dateifehler. Keine automatischen Wiederholungen ausstehend. Ein Teilindex ist verfügbar.", "indexingStopping": "Indexierung wird gestoppt...", "indexingFailedNoBlocks": "Indizierung fehlgeschlagen: Keine Code-Blöcke wurden erfolgreich indiziert. Dies deutet normalerweise auf ein Embedder-Konfigurationsproblem hin.", @@ -71,7 +69,8 @@ "unknownError": "Unbekannter Fehler", "indexingRequiresWorkspace": "Indexierung erfordert einen offenen Workspace-Ordner", "indexingStopped": "Indexierung vom Benutzer gestoppt.", - "indexingStoppedPartial": "Indexierung gestoppt. Teilweise Indexdaten beibehalten." + "indexingStoppedPartial": "Indexierung gestoppt. Teilweise Indexdaten beibehalten.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Semble-Binary wird heruntergeladen...", @@ -80,5 +79,6 @@ "downloadFailed": "Fehler beim Herunterladen von Semble: {{errorMessage}}", "checkFailed": "Semble-Überprüfung fehlgeschlagen: {{errorMessage}}", "providerReset": "Semble-Anbieter zurückgesetzt. Der Cache auf der Festplatte bleibt bis zum nächsten Neuaufbau erhalten." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/en/embeddings.json b/src/i18n/locales/en/embeddings.json index 215e8a1334..ea8e29f345 100644 --- a/src/i18n/locales/en/embeddings.json +++ b/src/i18n/locales/en/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Waiting for {{service}}; indexing resumes automatically", "providerWaitingUntil": "Waiting for {{service}}; indexing resumes automatically at {{time}}", - "parserFailed": "Code indexing stopped while parsing {{filePath}} ({{category}}). Fix the file and explicitly start indexing to retry, or disable indexing. This is not a model/provider error.", - "indexingFilesRetrying": "Retrying failed files. Partial index available.", "indexingFilesFailed": "File failures remain. No automatic retries pending. Partial index available.", "indexingStopping": "Stopping indexing...", "indexingFailedNoBlocks": "Indexing failed: No code blocks were successfully indexed. This usually indicates an embedder configuration issue.", @@ -71,7 +69,8 @@ "unknownError": "Unknown error", "indexingRequiresWorkspace": "Indexing requires an open workspace folder", "indexingStopped": "Indexing stopped by user.", - "indexingStoppedPartial": "Indexing stopped. Partial index data preserved." + "indexingStoppedPartial": "Indexing stopped. Partial index data preserved.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Downloading semble binary...", @@ -80,5 +79,6 @@ "downloadFailed": "Failed to download semble: {{errorMessage}}", "checkFailed": "Semble check failed: {{errorMessage}}", "providerReset": "Semble provider reset. On-disk cache remains until next rebuild." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/es/embeddings.json b/src/i18n/locales/es/embeddings.json index b4e49cc838..dbce931bed 100644 --- a/src/i18n/locales/es/embeddings.json +++ b/src/i18n/locales/es/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Esperando a {{service}}; la indexación se reanuda automáticamente", "providerWaitingUntil": "Esperando a {{service}}; la indexación se reanuda automáticamente a las {{time}}", - "parserFailed": "La indexación del código se detuvo al analizar {{filePath}} ({{category}}). Corrige el archivo e inicia explícitamente la indexación para reintentarlo, o desactívala. No es un error del modelo ni del proveedor.", - "indexingFilesRetrying": "Reintentando los archivos con errores. Hay un índice parcial disponible.", "indexingFilesFailed": "Siguen existiendo errores en archivos. No hay reintentos automáticos pendientes. Hay un índice parcial disponible.", "indexingStopping": "Deteniendo la indexación...", "indexingFailedNoBlocks": "Indexación fallida: No se indexaron exitosamente bloques de código. Esto usualmente indica un problema de configuración del incrustador.", @@ -71,7 +69,8 @@ "unknownError": "Error desconocido", "indexingRequiresWorkspace": "La indexación requiere una carpeta de workspace abierta", "indexingStopped": "Indexación detenida por el usuario.", - "indexingStoppedPartial": "Indexación detenida. Datos de índice parciales conservados." + "indexingStoppedPartial": "Indexación detenida. Datos de índice parciales conservados.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Descargando el binario de semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Error al descargar semble: {{errorMessage}}", "checkFailed": "Verificación de semble fallida: {{errorMessage}}", "providerReset": "Proveedor de Semble restablecido. La caché en disco se conserva hasta la próxima reconstrucción." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/fr/embeddings.json b/src/i18n/locales/fr/embeddings.json index 88e98e0411..5f81c08006 100644 --- a/src/i18n/locales/fr/embeddings.json +++ b/src/i18n/locales/fr/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "En attente de {{service}} ; l’indexation reprend automatiquement", "providerWaitingUntil": "En attente de {{service}} ; l’indexation reprend automatiquement à {{time}}", - "parserFailed": "L'indexation du code s'est arrêtée pendant l'analyse de {{filePath}} ({{category}}). Corrige le fichier et lance explicitement l'indexation pour réessayer, ou désactive-la. Ce n'est pas une erreur du modèle ni du fournisseur.", - "indexingFilesRetrying": "Nouvelle tentative pour les fichiers en échec. Un index partiel est disponible.", "indexingFilesFailed": "Des fichiers restent en échec. Aucune nouvelle tentative automatique en attente. Un index partiel est disponible.", "indexingStopping": "Arrêt de l'indexation en cours...", "indexingFailedNoBlocks": "Échec de l'indexation : Aucun bloc de code n'a été indexé avec succès. Cela indique généralement un problème de configuration de l'embedder.", @@ -71,7 +69,8 @@ "unknownError": "Erreur inconnue", "indexingRequiresWorkspace": "L'indexation nécessite l'ouverture d'un dossier workspace", "indexingStopped": "Indexation arrêtée par l'utilisateur.", - "indexingStoppedPartial": "Indexation arrêtée. Données d'index partielles conservées." + "indexingStoppedPartial": "Indexation arrêtée. Données d'index partielles conservées.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Téléchargement du binaire semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Échec du téléchargement de semble : {{errorMessage}}", "checkFailed": "Vérification de semble échouée : {{errorMessage}}", "providerReset": "Fournisseur Semble réinitialisé. Le cache sur disque reste jusqu'à la prochaine reconstruction." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/hi/embeddings.json b/src/i18n/locales/hi/embeddings.json index f8883d91aa..ccdad44e1f 100644 --- a/src/i18n/locales/hi/embeddings.json +++ b/src/i18n/locales/hi/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "{{service}} की प्रतीक्षा है; इंडेक्सिंग अपने आप फिर शुरू होगी", "providerWaitingUntil": "{{service}} की प्रतीक्षा है; इंडेक्सिंग {{time}} पर अपने आप फिर शुरू होगी", - "parserFailed": "{{filePath}} को पार्स करते समय कोड इंडेक्सिंग रुक गई ({{category}})। फ़ाइल ठीक करो और फिर से कोशिश करने के लिए खुद इंडेक्सिंग शुरू करो, या इंडेक्सिंग बंद कर दो। यह मॉडल या प्रदाता की त्रुटि नहीं है।", - "indexingFilesRetrying": "विफल फ़ाइलों के लिए फिर से कोशिश की जा रही है। आंशिक इंडेक्स उपलब्ध है।", "indexingFilesFailed": "फ़ाइलों में विफलताएँ अभी भी हैं। कोई स्वचालित पुनः प्रयास लंबित नहीं है। आंशिक इंडेक्स उपलब्ध है।", "indexingStopping": "इंडेक्सिंग रोकी जा रही है...", "indexingFailedNoBlocks": "इंडेक्सिंग असफल: कोई भी कोड ब्लॉक सफलतापूर्वक इंडेक्स नहीं हुआ। यह आमतौर पर एम्बेडर कॉन्फ़िगरेशन समस्या को दर्शाता है।", @@ -71,7 +69,8 @@ "unknownError": "अज्ञात त्रुटि", "indexingRequiresWorkspace": "इंडेक्सिंग के लिए एक खुला वर्कस्पेस फ़ोल्डर आवश्यक है", "indexingStopped": "उपयोगकर्ता द्वारा इंडेक्सिंग रोकी गई।", - "indexingStoppedPartial": "इंडेक्सिंग रोकी गई। आंशिक इंडेक्स डेटा संरक्षित।" + "indexingStoppedPartial": "इंडेक्सिंग रोकी गई। आंशिक इंडेक्स डेटा संरक्षित।", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "semble बाइनरी डाउनलोड हो रहा है...", @@ -80,5 +79,6 @@ "downloadFailed": "semble डाउनलोड करने में विफल: {{errorMessage}}", "checkFailed": "Semble जाँच विफल: {{errorMessage}}", "providerReset": "Semble प्रदाता रीसेट किया गया। डिस्क कैश अगले रीबिल्ड तक बना रहता है।" - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/id/embeddings.json b/src/i18n/locales/id/embeddings.json index ada4ea3f68..27e68b0ffd 100644 --- a/src/i18n/locales/id/embeddings.json +++ b/src/i18n/locales/id/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis", "providerWaitingUntil": "Menunggu {{service}}; pengindeksan dilanjutkan secara otomatis pada {{time}}", - "parserFailed": "Pengindeksan kode berhenti saat mengurai {{filePath}} ({{category}}). Perbaiki file dan mulai pengindeksan secara manual untuk mencoba lagi, atau nonaktifkan pengindeksan. Ini bukan kesalahan model atau provider.", - "indexingFilesRetrying": "Mencoba ulang file yang gagal. Indeks parsial tersedia.", "indexingFilesFailed": "Masih ada kegagalan file. Tidak ada percobaan ulang otomatis yang tertunda. Indeks parsial tersedia.", "indexingStopping": "Menghentikan pengindeksan...", "indexingFailedNoBlocks": "Pengindeksan gagal: Tidak ada blok kode yang berhasil diindeks. Ini biasanya menunjukkan masalah konfigurasi embedder.", @@ -71,7 +69,8 @@ "unknownError": "Kesalahan tidak diketahui", "indexingRequiresWorkspace": "Pengindeksan memerlukan folder workspace yang terbuka", "indexingStopped": "Pengindeksan dihentikan oleh pengguna.", - "indexingStoppedPartial": "Pengindeksan dihentikan. Data indeks parsial dipertahankan." + "indexingStoppedPartial": "Pengindeksan dihentikan. Data indeks parsial dipertahankan.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Mengunduh biner semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Gagal mengunduh semble: {{errorMessage}}", "checkFailed": "Pemeriksaan semble gagal: {{errorMessage}}", "providerReset": "Penyedia Semble direset. Cache di disk tetap ada hingga pembangunan ulang berikutnya." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/it/embeddings.json b/src/i18n/locales/it/embeddings.json index 7e6505e834..578edc93bc 100644 --- a/src/i18n/locales/it/embeddings.json +++ b/src/i18n/locales/it/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "In attesa di {{service}}; l’indicizzazione riprende automaticamente", "providerWaitingUntil": "In attesa di {{service}}; l’indicizzazione riprende automaticamente alle {{time}}", - "parserFailed": "L'indicizzazione del codice si è fermata durante l'analisi di {{filePath}} ({{category}}). Correggi il file e avvia esplicitamente l'indicizzazione per riprovare, oppure disattivala. Non è un errore del modello o del provider.", - "indexingFilesRetrying": "Nuovo tentativo per i file non riusciti. È disponibile un indice parziale.", "indexingFilesFailed": "Persistono errori nei file. Nessun nuovo tentativo automatico in attesa. È disponibile un indice parziale.", "indexingStopping": "Arresto dell'indicizzazione in corso...", "indexingFailedNoBlocks": "Indicizzazione fallita: Nessun blocco di codice è stato indicizzato con successo. Questo di solito indica un problema di configurazione dell'embedder.", @@ -71,7 +69,8 @@ "unknownError": "Errore sconosciuto", "indexingRequiresWorkspace": "L'indicizzazione richiede una cartella di workspace aperta", "indexingStopped": "Indicizzazione interrotta dall'utente.", - "indexingStoppedPartial": "Indicizzazione interrotta. Dati di indice parziali conservati." + "indexingStoppedPartial": "Indicizzazione interrotta. Dati di indice parziali conservati.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Download del binario semble in corso...", @@ -80,5 +79,6 @@ "downloadFailed": "Download di semble fallito: {{errorMessage}}", "checkFailed": "Verifica di semble fallita: {{errorMessage}}", "providerReset": "Provider Semble ripristinato. La cache su disco rimane fino alla prossima ricostruzione." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/ja/embeddings.json b/src/i18n/locales/ja/embeddings.json index d8bcf9f32b..3742e0e04b 100644 --- a/src/i18n/locales/ja/embeddings.json +++ b/src/i18n/locales/ja/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "{{service}} を待機中です。インデックス作成は自動的に再開されます", "providerWaitingUntil": "{{service}} を待機中です。インデックス作成は {{time}} に自動的に再開されます", - "parserFailed": "{{filePath}} の解析中にコードのインデックス作成が停止しました（{{category}}）。ファイルを修正し、インデックス作成を明示的に開始して再試行するか、インデックス作成を無効にしてください。モデルやプロバイダーのエラーではありません。", - "indexingFilesRetrying": "失敗したファイルを再試行しています。部分的なインデックスを利用できます。", "indexingFilesFailed": "ファイルのエラーが残っています。自動再試行の予定はありません。部分的なインデックスを利用できます。", "indexingStopping": "インデックス作成を停止中...", "indexingFailedNoBlocks": "インデックス作成に失敗しました：コードブロックが正常にインデックス化されませんでした。これは通常、エンベッダーの設定問題を示しています。", @@ -71,7 +69,8 @@ "unknownError": "不明なエラー", "indexingRequiresWorkspace": "インデックス作成には、開かれたワークスペースフォルダーが必要です", "indexingStopped": "ユーザーによりインデックス作成が停止されました。", - "indexingStoppedPartial": "インデックス作成が停止されました。部分的なインデックスデータは保持されています。" + "indexingStoppedPartial": "インデックス作成が停止されました。部分的なインデックスデータは保持されています。", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "sembleバイナリをダウンロード中...", @@ -80,5 +79,6 @@ "downloadFailed": "sembleのダウンロードに失敗しました: {{errorMessage}}", "checkFailed": "Sembleの確認に失敗しました: {{errorMessage}}", "providerReset": "Sembleプロバイダーをリセットしました。次回の再構築までディスクキャッシュは残ります。" - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/ko/embeddings.json b/src/i18n/locales/ko/embeddings.json index de01158384..ef18cda970 100644 --- a/src/i18n/locales/ko/embeddings.json +++ b/src/i18n/locales/ko/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "{{service}} 대기 중; 인덱싱이 자동으로 재개됩니다", "providerWaitingUntil": "{{service}} 대기 중; {{time}}에 인덱싱이 자동으로 재개됩니다", - "parserFailed": "{{filePath}} 파싱 중 코드 인덱싱이 중지되었습니다({{category}}). 파일을 수정하고 인덱싱을 직접 시작해 다시 시도하거나 인덱싱을 비활성화하세요. 모델 또는 제공자 오류가 아닙니다.", - "indexingFilesRetrying": "실패한 파일을 다시 시도하고 있습니다. 부분 인덱스를 사용할 수 있습니다.", "indexingFilesFailed": "파일 오류가 남아 있습니다. 예정된 자동 재시도가 없습니다. 부분 인덱스를 사용할 수 있습니다.", "indexingStopping": "인덱싱 중지 중...", "indexingFailedNoBlocks": "인덱싱 실패: 코드 블록이 성공적으로 인덱싱되지 않았습니다. 이는 일반적으로 임베더 구성 문제를 나타냅니다.", @@ -71,7 +69,8 @@ "unknownError": "알 수 없는 오류", "indexingRequiresWorkspace": "인덱싱에는 열린 워크스페이스 폴더가 필요합니다", "indexingStopped": "사용자에 의해 인덱싱이 중지되었습니다.", - "indexingStoppedPartial": "인덱싱이 중지되었습니다. 부분 인덱스 데이터가 보존되었습니다." + "indexingStoppedPartial": "인덱싱이 중지되었습니다. 부분 인덱스 데이터가 보존되었습니다.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "semble 바이너리 다운로드 중...", @@ -80,5 +79,6 @@ "downloadFailed": "semble 다운로드 실패: {{errorMessage}}", "checkFailed": "Semble 확인 실패: {{errorMessage}}", "providerReset": "Semble 공급자가 재설정되었습니다. 디스크 캐시는 다음 재구축까지 유지됩니다." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/nl/embeddings.json b/src/i18n/locales/nl/embeddings.json index 4beceff5ba..b98b5149ac 100644 --- a/src/i18n/locales/nl/embeddings.json +++ b/src/i18n/locales/nl/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Wachten op {{service}}; de indexering wordt automatisch hervat", "providerWaitingUntil": "Wachten op {{service}}; de indexering wordt automatisch hervat om {{time}}", - "parserFailed": "Code-indexering is gestopt tijdens het parseren van {{filePath}} ({{category}}). Herstel het bestand en start de indexering expliciet om het opnieuw te proberen, of schakel indexering uit. Dit is geen model- of providerfout.", - "indexingFilesRetrying": "Mislukte bestanden worden opnieuw geprobeerd. Een gedeeltelijke index is beschikbaar.", "indexingFilesFailed": "Er zijn nog bestandsfouten. Geen automatische nieuwe pogingen in afwachting. Een gedeeltelijke index is beschikbaar.", "indexingStopping": "Indexering wordt gestopt...", "indexingFailedNoBlocks": "Indexering mislukt: Geen codeblokken werden succesvol geïndexeerd. Dit duidt meestal op een embedder configuratieprobleem.", @@ -71,7 +69,8 @@ "unknownError": "Onbekende fout", "indexingRequiresWorkspace": "Indexering vereist een geopende workspace map", "indexingStopped": "Indexering gestopt door gebruiker.", - "indexingStoppedPartial": "Indexering gestopt. Gedeeltelijke indexgegevens bewaard." + "indexingStoppedPartial": "Indexering gestopt. Gedeeltelijke indexgegevens bewaard.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Semble-binary downloaden...", @@ -80,5 +79,6 @@ "downloadFailed": "Downloaden van semble mislukt: {{errorMessage}}", "checkFailed": "Semble-controle mislukt: {{errorMessage}}", "providerReset": "Semble-provider gereset. Schijfcache blijft behouden tot de volgende heropbouw." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/pl/embeddings.json b/src/i18n/locales/pl/embeddings.json index 1ab81cbafe..d7ebfa3576 100644 --- a/src/i18n/locales/pl/embeddings.json +++ b/src/i18n/locales/pl/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione", "providerWaitingUntil": "Oczekiwanie na {{service}}; indeksowanie zostanie automatycznie wznowione o {{time}}", - "parserFailed": "Indeksowanie kodu zatrzymało się podczas parsowania {{filePath}} ({{category}}). Popraw plik i ręcznie uruchom indeksowanie, aby spróbować ponownie, lub wyłącz indeksowanie. To nie jest błąd modelu ani dostawcy.", - "indexingFilesRetrying": "Ponawianie prób dla plików z błędami. Dostępny jest częściowy indeks.", "indexingFilesFailed": "Nadal występują błędy plików. Brak oczekujących automatycznych ponowień. Dostępny jest częściowy indeks.", "indexingStopping": "Zatrzymywanie indeksowania...", "indexingFailedNoBlocks": "Indeksowanie nie powiodło się: Żadne bloki kodu nie zostały pomyślnie zaindeksowane. To zwykle wskazuje na problem z konfiguracją embeddera.", @@ -71,7 +69,8 @@ "unknownError": "Nieznany błąd", "indexingRequiresWorkspace": "Indeksowanie wymaga otwartego folderu workspace", "indexingStopped": "Indeksowanie zatrzymane przez użytkownika.", - "indexingStoppedPartial": "Indeksowanie zatrzymane. Częściowe dane indeksu zachowane." + "indexingStoppedPartial": "Indeksowanie zatrzymane. Częściowe dane indeksu zachowane.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Pobieranie binariów semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Nie udało się pobrać semble: {{errorMessage}}", "checkFailed": "Sprawdzenie semble nie powiodło się: {{errorMessage}}", "providerReset": "Dostawca Semble zresetowany. Pamięć podręczna na dysku pozostaje do kolejnej przebudowy." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/pt-BR/embeddings.json b/src/i18n/locales/pt-BR/embeddings.json index aeba6c2f34..e9f4561c9f 100644 --- a/src/i18n/locales/pt-BR/embeddings.json +++ b/src/i18n/locales/pt-BR/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Aguardando {{service}}; a indexação é retomada automaticamente", "providerWaitingUntil": "Aguardando {{service}}; a indexação é retomada automaticamente às {{time}}", - "parserFailed": "A indexação do código parou durante a análise de {{filePath}} ({{category}}). Corrige o arquivo e inicia explicitamente a indexação para tentar de novo, ou desativa a indexação. Não é um erro do modelo nem do provedor.", - "indexingFilesRetrying": "Tentando novamente os arquivos com falhas. Um índice parcial está disponível.", "indexingFilesFailed": "Ainda há falhas em arquivos. Nenhuma nova tentativa automática pendente. Um índice parcial está disponível.", "indexingStopping": "Parando a indexação...", "indexingFailedNoBlocks": "Indexação falhou: Nenhum bloco de código foi indexado com sucesso. Isso geralmente indica um problema de configuração do embedder.", @@ -71,7 +69,8 @@ "unknownError": "Erro desconhecido", "indexingRequiresWorkspace": "A indexação requer uma pasta de workspace aberta", "indexingStopped": "Indexação interrompida pelo usuário.", - "indexingStoppedPartial": "Indexação interrompida. Dados de índice parciais preservados." + "indexingStoppedPartial": "Indexação interrompida. Dados de índice parciais preservados.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Baixando o binário do semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Falha ao baixar semble: {{errorMessage}}", "checkFailed": "Verificação do semble falhou: {{errorMessage}}", "providerReset": "Provedor Semble redefinido. O cache em disco permanece até a próxima reconstrução." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/ru/embeddings.json b/src/i18n/locales/ru/embeddings.json index ee1b7aed0f..bb667b9976 100644 --- a/src/i18n/locales/ru/embeddings.json +++ b/src/i18n/locales/ru/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Ожидание {{service}}; индексация возобновится автоматически", "providerWaitingUntil": "Ожидание {{service}}; индексация возобновится автоматически в {{time}}", - "parserFailed": "Индексация кода остановлена при разборе {{filePath}} ({{category}}). Исправь файл и явно запусти индексацию для повторной попытки или отключи её. Это не ошибка модели или провайдера.", - "indexingFilesRetrying": "Повторные попытки для файлов с ошибками. Доступен частичный индекс.", "indexingFilesFailed": "Остались ошибки файлов. Автоматических повторных попыток не ожидается. Доступен частичный индекс.", "indexingStopping": "Остановка индексации...", "indexingFailedNoBlocks": "Индексация не удалась: Ни один блок кода не был успешно проиндексирован. Это обычно указывает на проблему конфигурации эмбеддера.", @@ -71,7 +69,8 @@ "unknownError": "Неизвестная ошибка", "indexingRequiresWorkspace": "Для индексации требуется открытая папка рабочего пространства", "indexingStopped": "Индексация остановлена пользователем.", - "indexingStoppedPartial": "Индексация остановлена. Частичные данные индекса сохранены." + "indexingStoppedPartial": "Индексация остановлена. Частичные данные индекса сохранены.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Загрузка бинарного файла semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Не удалось загрузить semble: {{errorMessage}}", "checkFailed": "Проверка semble не удалась: {{errorMessage}}", "providerReset": "Провайдер Semble сброшен. Дисковый кеш сохраняется до следующей пересборки." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/tr/embeddings.json b/src/i18n/locales/tr/embeddings.json index 0c62e2f6a4..3afc96150f 100644 --- a/src/i18n/locales/tr/embeddings.json +++ b/src/i18n/locales/tr/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "{{service}} bekleniyor; dizinleme otomatik olarak devam eder", "providerWaitingUntil": "{{service}} bekleniyor; dizinleme {{time}} zamanında otomatik olarak devam eder", - "parserFailed": "{{filePath}} ayrıştırılırken kod indeksleme durdu ({{category}}). Dosyayı düzelt ve yeniden denemek için indekslemeyi açıkça başlat ya da indekslemeyi devre dışı bırak. Bu bir model veya sağlayıcı hatası değil.", - "indexingFilesRetrying": "Başarısız dosyalar yeniden deneniyor. Kısmi indeks kullanılabilir.", "indexingFilesFailed": "Dosya hataları devam ediyor. Bekleyen otomatik yeniden deneme yok. Kısmi indeks kullanılabilir.", "indexingStopping": "İndeksleme durduruluyor...", "indexingFailedNoBlocks": "İndeksleme başarısız: Hiçbir kod bloğu başarıyla indekslenemedi. Bu genellikle bir embedder yapılandırma sorunu olduğunu gösterir.", @@ -71,7 +69,8 @@ "unknownError": "Bilinmeyen hata", "indexingRequiresWorkspace": "İndeksleme açık bir workspace klasörü gerektirir", "indexingStopped": "İndeksleme kullanıcı tarafından durduruldu.", - "indexingStoppedPartial": "İndeksleme durduruldu. Kısmi indeks verileri korundu." + "indexingStoppedPartial": "İndeksleme durduruldu. Kısmi indeks verileri korundu.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "semble ikili dosyası indiriliyor...", @@ -80,5 +79,6 @@ "downloadFailed": "semble indirilemedi: {{errorMessage}}", "checkFailed": "Semble kontrolü başarısız: {{errorMessage}}", "providerReset": "Semble sağlayıcısı sıfırlandı. Diskteki önbellek bir sonraki yeniden yapılandırmaya kadar korunur." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/vi/embeddings.json b/src/i18n/locales/vi/embeddings.json index 44f624778c..5fe21bd5b9 100644 --- a/src/i18n/locales/vi/embeddings.json +++ b/src/i18n/locales/vi/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục", "providerWaitingUntil": "Đang chờ {{service}}; việc lập chỉ mục tự động tiếp tục lúc {{time}}", - "parserFailed": "Đã dừng lập chỉ mục mã khi phân tích {{filePath}} ({{category}}). Sửa tệp rồi chủ động bắt đầu lập chỉ mục để thử lại, hoặc tắt lập chỉ mục. Đây không phải lỗi mô hình hay nhà cung cấp.", - "indexingFilesRetrying": "Đang thử lại các tệp bị lỗi. Có thể dùng chỉ mục một phần.", "indexingFilesFailed": "Vẫn còn tệp bị lỗi. Không có lần thử lại tự động nào đang chờ. Có thể dùng chỉ mục một phần.", "indexingStopping": "Đang dừng lập chỉ mục...", "indexingFailedNoBlocks": "Lập chỉ mục thất bại: Không có khối mã nào được lập chỉ mục thành công. Điều này thường cho thấy vấn đề cấu hình embedder.", @@ -71,7 +69,8 @@ "unknownError": "Lỗi không xác định", "indexingRequiresWorkspace": "Lập chỉ mục yêu cầu một thư mục workspace đang mở", "indexingStopped": "Lập chỉ mục đã bị dừng bởi người dùng.", - "indexingStoppedPartial": "Lập chỉ mục đã dừng. Dữ liệu chỉ mục một phần được bảo toàn." + "indexingStoppedPartial": "Lập chỉ mục đã dừng. Dữ liệu chỉ mục một phần được bảo toàn.", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "Đang tải xuống tệp nhị phân semble...", @@ -80,5 +79,6 @@ "downloadFailed": "Tải xuống semble thất bại: {{errorMessage}}", "checkFailed": "Kiểm tra semble thất bại: {{errorMessage}}", "providerReset": "Nhà cung cấp Semble đã đặt lại. Bộ nhớ đệm trên đĩa vẫn còn cho đến lần xây dựng lại tiếp theo." - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/zh-CN/embeddings.json b/src/i18n/locales/zh-CN/embeddings.json index 053256c17f..912fcfc300 100644 --- a/src/i18n/locales/zh-CN/embeddings.json +++ b/src/i18n/locales/zh-CN/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "正在等待 {{service}}；索引将自动恢复", "providerWaitingUntil": "正在等待 {{service}}；索引将于 {{time}} 自动恢复", - "parserFailed": "解析 {{filePath}} 时，代码索引已停止（{{category}}）。修复文件后，手动启动索引以重试，或禁用索引。这不是模型或提供商错误。", - "indexingFilesRetrying": "正在重试失败的文件。部分索引可用。", "indexingFilesFailed": "仍有文件失败。没有待执行的自动重试。部分索引可用。", "indexingStopping": "正在停止索引...", "indexingFailedNoBlocks": "索引失败：没有代码块被成功索引。这通常表示 Embedder 配置问题。", @@ -71,7 +69,8 @@ "unknownError": "未知错误", "indexingRequiresWorkspace": "索引需要打开的工作区文件夹", "indexingStopped": "用户已停止索引。", - "indexingStoppedPartial": "索引已停止。部分索引数据已保留。" + "indexingStoppedPartial": "索引已停止。部分索引数据已保留。", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "正在下载 semble 二进制文件...", @@ -80,5 +79,6 @@ "downloadFailed": "下载 semble 失败：{{errorMessage}}", "checkFailed": "Semble 检查失败：{{errorMessage}}", "providerReset": "Semble 提供程序已重置。磁盘缓存将保留至下次重建。" - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/i18n/locales/zh-TW/embeddings.json b/src/i18n/locales/zh-TW/embeddings.json index d308f1f3f7..2e4f45ebb3 100644 --- a/src/i18n/locales/zh-TW/embeddings.json +++ b/src/i18n/locales/zh-TW/embeddings.json @@ -60,8 +60,6 @@ "orchestrator": { "providerWaiting": "正在等待 {{service}}；索引將自動恢復", "providerWaitingUntil": "正在等待 {{service}}；索引將於 {{time}} 自動恢復", - "parserFailed": "剖析 {{filePath}} 時，程式碼索引已停止（{{category}}）。修正檔案後，手動啟動索引以重試，或停用索引。這不是模型或供應商錯誤。", - "indexingFilesRetrying": "正在重試失敗的檔案。部分索引可供使用。", "indexingFilesFailed": "仍有檔案失敗。沒有待執行的自動重試。部分索引可供使用。", "indexingStopping": "正在停止索引...", "indexingFailedNoBlocks": "索引失敗：沒有程式碼區塊被成功索引。這通常表示 Embedder 設定問題。", @@ -71,7 +69,8 @@ "unknownError": "未知錯誤", "indexingRequiresWorkspace": "索引需要開啟的工作區資料夾", "indexingStopped": "使用者已停止索引。", - "indexingStoppedPartial": "索引已停止。部分索引資料已保留。" + "indexingStoppedPartial": "索引已停止。部分索引資料已保留。", + "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." }, "semble": { "downloadingBinary": "正在下載 semble 二進位檔...", @@ -80,5 +79,6 @@ "downloadFailed": "下載 semble 失敗：{{errorMessage}}", "checkFailed": "Semble 檢查失敗：{{errorMessage}}", "providerReset": "Semble 提供者已重設。磁碟快取將保留至下次重建。" - } + }, + "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." } diff --git a/src/services/code-index/__tests__/search-service.spec.ts b/src/services/code-index/__tests__/search-service.spec.ts index 345e3f591f..cd791b5508 100644 --- a/src/services/code-index/__tests__/search-service.spec.ts +++ b/src/services/code-index/__tests__/search-service.spec.ts @@ -6,7 +6,6 @@ import { SearchProviderRateLimitError } from "../shared/indexing-errors" import { CodeIndexStateManager, type IndexingState } from "../state-manager" import type { IEmbedder } from "../interfaces/embedder" import type { IVectorStore } from "../interfaces/vector-store" -import { EmbeddingError } from "../shared/indexing-errors" vi.mock("vscode", () => ({ EventEmitter: class { @@ -16,10 +15,9 @@ vi.mock("vscode", () => ({ }, })) vi.mock("../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: { captureEvent: vi.fn() } } })) -vi.mock("../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), info: vi.fn() } })) - -function prepare(state: IndexingState = "Indexed", failed = true, exists = true) { +function prepare(state: IndexingState, { failed = false, exists = true, enabled = true } = {}) { const failures = new FileFailureRegistry() if (failed) { for (const file of ["src/a.ts", "other/b.ts"]) { @@ -28,347 +26,107 @@ function prepare(state: IndexingState = "Indexed", failed = true, exists = true) } } const config = { - isFeatureEnabled: true, + isFeatureEnabled: enabled, isFeatureConfigured: true, currentSearchMinScore: 0.4, currentSearchMaxResults: 12, } const stateManager = new CodeIndexStateManager() - stateManager.setSystemState(state, "Original status") + stateManager.setSystemState(state, "status") const setSystemState = vi.spyOn(stateManager, "setSystemState") - const embedder = { rateLimitScope: "embedding:test", createEmbeddings: vi.fn() } + const embedder = { createEmbeddings: vi.fn() } embedder.createEmbeddings.mockResolvedValue({ embeddings: [[1, 0]] }) const store = { search: vi.fn().mockResolvedValue([]), collectionExists: vi.fn().mockResolvedValue(exists), } - const getCurrentStatus = vi.fn(() => stateManager.getCurrentStatus()) const pacers = new ProviderPacerRegistry() - const service = new CodeIndexSearchService( - config, - stateManager, - embedder, - store, - failures, - pacers, - getCurrentStatus, - ) - const statusBefore = stateManager.getCurrentStatus() - const failuresBefore = failures.list() - const registryMethods = ["record", "clear", "renew", "resetAllBudgets", "getRetryBudget"] as const - const registryWrites = registryMethods.map((method) => vi.spyOn(failures, method)) - function expectUnchanged() { - expect(setSystemState).not.toHaveBeenCalled() - expect(stateManager.getCurrentStatus()).toEqual(statusBefore) - expect(failures.list()).toEqual(failuresBefore) - for (const write of registryWrites) expect(write).not.toHaveBeenCalled() + const service = new CodeIndexSearchService(config, stateManager, embedder, store, failures, pacers) + return { service, failures, pacers, store, embedder, setSystemState } +} + +async function withWaitingPath(pacers: ProviderPacerRegistry, file: string) { + const controller = new AbortController() + const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs: 60_000 }) + const pending = pacers + .get("openai:1") + .run(file, controller.signal, () => Promise.reject(limited)) + .catch(() => {}) + await new Promise((resolve) => setTimeout(resolve, 0)) + return async () => { + controller.abort() + await pending } - return { service, failures, pacers, store, embedder, config, stateManager, getCurrentStatus, expectUnchanged } } -afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() -}) +afterEach(() => vi.restoreAllMocks()) -describe("CodeIndexSearchService", () => { - it.each(["Indexed", "Indexing", "Standby"] as const)( - "returns every failed path with matches or no matches in %s", - async (state) => { - const { service, store } = prepare(state) +describe("CodeIndexSearchService (CI-SS-01)", () => { + it.each([ + { state: "Indexed", failed: false, incomplete: false }, + { state: "Indexed", failed: true, incomplete: true }, + { state: "Indexing", failed: false, incomplete: true }, + { state: "Standby", failed: true, incomplete: true }, + ] as const)("$state with failures=$failed reports incomplete=$incomplete", async ({ state, failed, incomplete }) => { + const { service } = prepare(state, { failed }) + const response = await service.searchIndex("query") + expect(response.incomplete).toBe(incomplete) + expect(response.failedPaths).toEqual(failed ? ["src/a.ts", "other/b.ts"] : []) + expect(response.waitingPaths).toEqual([]) + }) + + it("reports waiting paths separately and deduplicated, for matches and empty results", async () => { + const { service, pacers, store } = prepare("Indexing", { failed: true }) + const releaseFirst = await withWaitingPath(pacers, "w.ts") + const releaseSecond = await withWaitingPath(pacers, "w.ts") + try { for (const results of [[], [{ id: "point", score: 0.9 }]]) { store.search.mockResolvedValueOnce(results) - await expect(service.searchIndex("query", "src/folder")).resolves.toEqual({ - results, - incomplete: true, - failedPaths: ["src/a.ts", "other/b.ts"], - waitingPaths: [], - }) - } - expect(store.search).toHaveBeenCalledWith([1, 0], path.normalize("src/folder"), 0.4, 12) - expect(store.collectionExists).toHaveBeenCalledTimes(state === "Standby" ? 2 : 0) - }, - ) - - it.each(["Indexed", "Indexing"] as const)("allows %s without known failures", async (state) => { - const { service, expectUnchanged } = prepare(state, false) - await expect(service.searchIndex("query")).resolves.toEqual({ - results: [], - incomplete: false, - failedPaths: [], - waitingPaths: [], - }) - expectUnchanged() - }) - - it.each([ - ["Standby", false, true], - ["Standby", true, false], - ["Error", true, true], - ["Stopping", true, true], - ] as const)("rejects %s / failures %s / collection %s", async (state, failed, exists) => { - const { service, embedder, store, expectUnchanged } = prepare(state, failed, exists) - await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.collectionExists).toHaveBeenCalledTimes(state === "Standby" && failed ? 1 : 0) - expectUnchanged() - }) - - it.each(["isFeatureEnabled", "isFeatureConfigured"] as const)("rejects when %s is false", async (setting) => { - const { service, config, embedder, expectUnchanged } = prepare() - config[setting] = false - await expect(service.searchIndex("query")).rejects.toThrow("disabled or not configured") - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expectUnchanged() - }) - - it("uses the manager's computed status instead of raw progress state", async () => { - const { service, getCurrentStatus, embedder, stateManager } = prepare("Indexing") - getCurrentStatus.mockReturnValue({ ...stateManager.getCurrentStatus(), systemStatus: "Error" }) - await expect(service.searchIndex("query")).rejects.toThrow("Current state: Error") - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it.each(["Error", "Stopping"] as const)("does not bypass %s reached during collection lookup", async (state) => { - const { service, store, getCurrentStatus, stateManager, embedder } = prepare("Standby") - store.collectionExists.mockImplementation(async () => { - getCurrentStatus.mockReturnValue({ ...stateManager.getCurrentStatus(), systemStatus: state }) - return true - }) - await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - }) - - it.each([true, false])("preserves cancellation during collection lookup, rejects=%s", async (rejects) => { - const { service, store, embedder, expectUnchanged } = prepare("Standby") - const controller = new AbortController() - const reason = Object.freeze({ cancelled: true }) - store.collectionExists.mockImplementation(async () => { - controller.abort(reason) - if (rejects) throw new Error("collection lookup failed") - return true - }) - await expect(service.searchIndex("query", undefined, controller.signal)).rejects.toBe(reason) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expectUnchanged() - }) - - it.each([true, false])("snapshots failures at response time, recovery=%s", async (recovered) => { - const { service, failures, store } = prepare() - store.search.mockImplementation(async () => { - failures.clear("src/a.ts") - failures.clear("other/b.ts") - if (!recovered) { - failures.renew("pending.ts", "new") - failures.record("pending.ts", "new", "embed", "waiting", 1, "retrying", "server") + const response = await service.searchIndex("query") + expect(response.results).toEqual(results) + expect(response).toMatchObject({ incomplete: true, waitingPaths: ["w.ts"] }) + expect(response.failedPaths).toEqual(["src/a.ts", "other/b.ts"]) } - return [] - }) - await expect(service.searchIndex("query", "src")).resolves.toEqual({ - results: [], - incomplete: !recovered, - failedPaths: recovered ? [] : ["pending.ts"], - waitingPaths: [], - }) - }) - - it("supports the state-manager fallback when no status supplier is provided", async () => { - const { config, stateManager, embedder, store, failures, pacers } = prepare() - const service = new CodeIndexSearchService(config, stateManager, embedder, store, failures, pacers) - await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: true }) - }) - - it.each(["Indexed", "Indexing", "Standby"] as const)( - "keeps %s health unchanged on store failure", - async (state) => { - const { service, store, expectUnchanged } = prepare(state) - const error = new Error("store unavailable") - store.search.mockRejectedValue(error) - await expect(service.searchIndex("query")).rejects.toBe(error) - expect(store.search).toHaveBeenCalledOnce() - expectUnchanged() - }, - ) - - it("rejects an empty embedding without changing indexing health", async () => { - const { service, embedder, store, expectUnchanged } = prepare() - embedder.createEmbeddings.mockResolvedValue({ embeddings: [] }) - await expect(service.searchIndex("query")).rejects.toThrow("Failed to generate embedding") - expect(store.search).not.toHaveBeenCalled() - expectUnchanged() - }) - - it.each([401, 408, 409])("does not retry permanent query errors (%s)", async (status) => { - const { service, embedder, expectUnchanged } = prepare() - const error = Object.freeze(Object.assign(new Error("unavailable"), { status })) - embedder.createEmbeddings.mockRejectedValue(error) - await expect(service.searchIndex("query")).rejects.toMatchObject({ - cause: error, - classification: "permanent", - attempts: 1, - }) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(error).not.toHaveProperty("attempts") - expectUnchanged() - }) - - it.each([503])("uses one two-retry budget per query (%s)", async (status) => { - vi.useFakeTimers() - const { service, embedder, expectUnchanged } = prepare() - const error = Object.assign(new Error("unavailable"), { status }) - embedder.createEmbeddings.mockRejectedValue(error) - for (const expectedCalls of [3, 6]) { - const outcome = service.searchIndex("query").catch((failure: unknown) => failure) - await vi.advanceTimersByTimeAsync(499) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls - 2) - await vi.advanceTimersByTimeAsync(1) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls - 1) - await vi.advanceTimersByTimeAsync(1000) - await expect(outcome).resolves.toMatchObject({ cause: error, attempts: 3 }) - expect(embedder.createEmbeddings).toHaveBeenCalledTimes(expectedCalls) - } - expectUnchanged() - }) - - it.each([60_000, 61_000, 120_000])( - "reports a query 429 immediately with full %s ms pause", - async (retryAfterMs) => { - vi.useFakeTimers() - const { service, embedder, pacers, expectUnchanged } = prepare() - const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs }) - embedder.createEmbeddings.mockRejectedValue(error) - await expect(service.searchIndex("query")).rejects.toMatchObject({ - name: SearchProviderRateLimitError.name, - message: "embeddings:searchProviderRateLimited", - cause: error, - }) - expect(pacers.get(embedder.rateLimitScope).state().pausedUntil).toBe(Date.now() + retryAfterMs) - expect(pacers.waitingPaths()).toEqual([]) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expectUnchanged() - }, - ) - - it("does not wait for provider pauses or indexing permits", async () => { - vi.useFakeTimers() - const { service, embedder, pacers } = prepare() - const pacer = pacers.get(embedder.rateLimitScope) - pacer.reportRateLimit( - new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 120_000 }), - pacer.markIssued(), - ) - await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: true }) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - }) - - it("bypasses saturated indexing permits and marks each query request issued", async () => { - const { service, embedder, pacers, expectUnchanged } = prepare("Indexed", false) - const pacer = pacers.get(embedder.rateLimitScope) - let release!: () => void - const held = new Promise((resolve) => { - release = resolve - }) - const indexing = Array.from({ length: pacer.state().limit }, (_, position) => - pacer.run(`file-${position}.ts`, undefined, () => held), - ) - await Promise.resolve() - const markIssued = vi.spyOn(pacer, "markIssued") - try { - await expect(service.searchIndex("query")).resolves.toMatchObject({ incomplete: false }) - expect(embedder.createEmbeddings).toHaveBeenCalledOnce() - expect(markIssued).toHaveBeenCalledOnce() - expectUnchanged() } finally { - release() - await Promise.all(indexing) + await releaseFirst() + await releaseSecond() } }) - it.each(["Standby", "Error", "Stopping"] as const)("waiting-only search gate preserves %s", async (state) => { - const { service, pacers, embedder } = prepare(state, false) - vi.spyOn(pacers, "waitingPaths").mockReturnValue(["waiting.ts"]) - if (state === "Standby") { - await expect(service.searchIndex("query")).resolves.toEqual({ - results: [], - incomplete: true, - failedPaths: [], - waitingPaths: ["waiting.ts"], - }) - } else { - await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - } + it("reports whole-workspace failures for a directory-scoped search", async () => { + const { service, store } = prepare("Indexed", { failed: true }) + const response = await service.searchIndex("query", "src") + expect(store.search).toHaveBeenCalledWith([1, 0], path.normalize("src"), 0.4, 12) + expect(response.failedPaths).toEqual(["src/a.ts", "other/b.ts"]) }) - it("rechecks waiting-only availability after collection lookup", async () => { - const { service, pacers, store, embedder } = prepare("Standby", false) - const waiting = vi.spyOn(pacers, "waitingPaths").mockReturnValue(["waiting.ts"]) - store.collectionExists.mockImplementation(async () => { - waiting.mockReturnValue([]) - return true - }) - await expect(service.searchIndex("query")).rejects.toThrow("Current state: Standby") + it.each([ + { state: "Error", failed: true, exists: true }, + { state: "Stopping", failed: true, exists: true }, + { state: "Standby", failed: false, exists: true }, + { state: "Standby", failed: true, exists: false }, + ] as const)("rejects search in $state (failures=$failed, collection=$exists)", async ({ state, failed, exists }) => { + const { service, embedder } = prepare(state, { failed, exists }) + await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) expect(embedder.createEmbeddings).not.toHaveBeenCalled() }) - it("snapshots waiting paths only after the store responds", async () => { - const { service, pacers, store } = prepare("Indexed", false) - const waiting = vi.spyOn(pacers, "waitingPaths").mockReturnValue([]) - store.search.mockImplementation(async () => { - waiting.mockReturnValue(["late.ts"]) - return [] - }) - await expect(service.searchIndex("query")).resolves.toEqual({ - results: [], - incomplete: true, - failedPaths: [], - waitingPaths: ["late.ts"], - }) - }) - - it("cancels a retry wait with unchanged reason and no later request", async () => { - vi.useFakeTimers() - const { service, embedder, store, expectUnchanged } = prepare() - embedder.createEmbeddings.mockRejectedValue(new EmbeddingError("unavailable", { status: 503 })) - const controller = new AbortController() - const reason = Object.freeze({ cancelled: true }) - const outcome = service.searchIndex("query", undefined, controller.signal).catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - controller.abort(reason) - await expect(outcome).resolves.toBe(reason) - await vi.advanceTimersByTimeAsync(120_000) - expect(embedder.createEmbeddings).toHaveBeenCalledExactlyOnceWith(["query"], undefined, controller.signal) - expect(store.search).not.toHaveBeenCalled() - expect(vi.getTimerCount()).toBe(0) - expectUnchanged() + it("does not set Error when a query fails", async () => { + const { service, embedder, setSystemState } = prepare("Indexed") + embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("bad"), { status: 400 })) + await expect(service.searchIndex("query")).rejects.toThrow() + expect(setSystemState).not.toHaveBeenCalled() }) - it("does not issue requests for an already cancelled caller", async () => { - const { service, embedder, store, expectUnchanged } = prepare() - const signal = AbortSignal.abort(new Error("caller ended")) - await expect(service.searchIndex("query", undefined, signal)).rejects.toBe(signal.reason) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - expect(store.search).not.toHaveBeenCalled() - expectUnchanged() + it("turns a query rate limit into SearchProviderRateLimitError without waiting", async () => { + const { service, embedder } = prepare("Indexed") + embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("limited"), { status: 429 })) + await expect(service.searchIndex("query")).rejects.toBeInstanceOf(SearchProviderRateLimitError) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() }) - it.each(["embed", "search"] as const)("preserves cancellation after in-flight %s finishes", async (activity) => { - const { service, embedder, store, expectUnchanged } = prepare() - const controller = new AbortController() - const reason = new Error("caller ended") - if (activity === "embed") { - embedder.createEmbeddings.mockImplementation(async () => { - controller.abort(reason) - return { embeddings: [[1, 0]] } - }) - } else { - store.search.mockImplementation(async () => { - controller.abort(reason) - return [] - }) - } - await expect(service.searchIndex("query", undefined, controller.signal)).rejects.toBe(reason) - expect(store.search).toHaveBeenCalledTimes(activity === "embed" ? 0 : 1) - expectUnchanged() + it("rejects when the feature is disabled", async () => { + const { service } = prepare("Indexed", { enabled: false }) + await expect(service.searchIndex("query")).rejects.toThrow("disabled or not configured") }) }) diff --git a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts index e3973d4de0..838fffdc6e 100644 --- a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts +++ b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts @@ -10,7 +10,7 @@ describe("WorkspaceIndexingEnablementManager", () => { isFeatureEnabled: true, isFeatureConfigured: true, initialize: vi.fn().mockResolvedValue({ requiresRestart: false }), - startIndexing: vi.fn<(contextProxy?: ContextProxy) => Promise>().mockResolvedValue(undefined), + startIndexing: vi.fn().mockResolvedValue(undefined), stopIndexing: vi.fn(), getCurrentStatus: vi.fn().mockReturnValue({ systemStatus: "Standby", @@ -21,9 +21,6 @@ describe("WorkspaceIndexingEnablementManager", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: false, - failedFiles: [], - failedFileSummary: undefined, - providerWaits: [], }), } return { @@ -37,21 +34,15 @@ describe("WorkspaceIndexingEnablementManager", () => { } } - it("persists enablement and awaits one start before publishing", async () => { + it("persists enablement, initializes and starts in order without awaiting indexing", async () => { const { manager, indexing, provider } = setup() - let resolveStart: () => void = () => {} - manager.startIndexing.mockReturnValue(new Promise((resolve) => (resolveStart = resolve))) - const pending = indexing.setEnabled(true, provider) - await Promise.resolve() + manager.startIndexing.mockReturnValue(new Promise(() => {})) + await indexing.setEnabled(true, provider) expect(manager.setWorkspaceEnabled).toHaveBeenCalledWith(true) - expect(manager.startIndexing).toHaveBeenCalledExactlyOnceWith(provider.contextProxy) - expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.startIndexing) - expect(manager.initialize).not.toHaveBeenCalled() + expect(manager.initialize).toHaveBeenCalledWith(provider.contextProxy) + expect(manager.setWorkspaceEnabled).toHaveBeenCalledBefore(manager.initialize) + expect(manager.initialize).toHaveBeenCalledBefore(manager.startIndexing) expect(manager.stopIndexing).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - resolveStart() - await pending - expect(provider.postMessageToWebview).toHaveBeenCalledOnce() }) it("persists disablement before stopping", async () => { @@ -74,14 +65,11 @@ describe("WorkspaceIndexingEnablementManager", () => { expect(manager.stopIndexing).not.toHaveBeenCalled() }) - it("propagates start failures without publishing success", async () => { + it("logs background indexing failures", async () => { const { manager, indexing, provider } = setup() - const error = new Error("failed") - manager.startIndexing.mockRejectedValue(error) - await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) - expect(manager.startIndexing).toHaveBeenCalledOnce() - expect(manager.initialize).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() + manager.startIndexing.mockRejectedValue(new Error("failed")) + await expect(indexing.setEnabled(true, provider)).resolves.toBeUndefined() + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: failed") }) it.each([true, false])("publishes the current status after setting enabled=%s", async (enabled) => { @@ -101,12 +89,15 @@ describe("WorkspaceIndexingEnablementManager", () => { await expect(indexing.setEnabled(false, provider)).rejects.toBe(error) }) - it("propagates persistence failures without starting", async () => { - const { manager, indexing, provider } = setup() - const error = new Error("failed") - manager.setWorkspaceEnabled.mockRejectedValue(error) - await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) - expect(manager.startIndexing).not.toHaveBeenCalled() - expect(provider.postMessageToWebview).not.toHaveBeenCalled() - }) + it.each(["setWorkspaceEnabled", "initialize"] as const)( + "propagates %s failures without starting", + async (method) => { + const { manager, indexing, provider } = setup() + const error = new Error("failed") + manager[method].mockRejectedValue(error) + await expect(indexing.setEnabled(true, provider)).rejects.toBe(error) + expect(manager.startIndexing).not.toHaveBeenCalled() + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + }, + ) }) diff --git a/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts b/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts deleted file mode 100644 index 1aaf0b5d45..0000000000 --- a/src/services/code-index/embedders/__tests__/bedrock-pacing.spec.ts +++ /dev/null @@ -1,70 +0,0 @@ -import "../../__tests__/mock-code-index-log" -import { createServer } from "node:http2" -import { BedrockEmbedder } from "../bedrock" -import { pacedTransportWriter, waitForTransport } from "./paced-transport-writer" - -vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) -vi.mock("../../../../utils/networkProxy", () => ({ createProxyRoutingRequestHandler: () => undefined })) - -beforeEach(() => { - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }) - vi.spyOn(Math, "random").mockReturnValue(0) -}) -afterEach(() => { - vi.clearAllTimers() - vi.useRealTimers() - vi.restoreAllMocks() - vi.unstubAllEnvs() -}) - -it("paces four HTTP/2 throttles without exhaustion and then completes", async () => { - let requests = 0 - const server = createServer() - server.on("stream", (stream) => { - requests++ - if (requests <= 4) { - stream.respond({ - ":status": 429, - "content-type": "application/json", - "x-amzn-errortype": "ThrottlingException", - }) - stream.end(JSON.stringify({ message: "limited", __type: "ThrottlingException" })) - } else { - stream.respond({ ":status": 200, "content-type": "application/json" }) - stream.end(JSON.stringify({ embedding: [1, 0], inputTextTokenCount: 1 })) - } - }) - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) - const address = server.address() - if (!address || typeof address === "string") throw new Error("Expected a TCP test listener") - vi.stubEnv("AWS_ACCESS_KEY_ID", "test-access-key") - vi.stubEnv("AWS_SECRET_ACCESS_KEY", "test-secret-key") - vi.stubEnv("AWS_SESSION_TOKEN", "") - vi.stubEnv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", `http://127.0.0.1:${address.port}`) - const embedder = new BedrockEmbedder("us-east-1") - const controller = new AbortController() - const { replace, failures, cache, pacers } = pacedTransportWriter() - const pending = replace(embedder, controller.signal).catch((error: unknown) => error) - try { - for (const [index, delay] of [500, 1000, 2000, 4000].entries()) { - await waitForTransport(() => { - expect(requests).toBe(index + 1) - expect(pacers.get(embedder.rateLimitScope).state().pausedUntil).toBe(Date.now() + delay) - expect(pacers.waitingPaths()).toEqual(["test.ts"]) - }) - expect(failures.list()).toEqual([]) - vi.advanceTimersByTime(delay - 1) - expect(requests).toBe(index + 1) - vi.advanceTimersByTime(1) - } - await pending - expect(requests).toBe(5) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() - } finally { - controller.abort() - await pending - embedder["bedrockClient"].destroy() - await new Promise((resolve) => server.close(() => resolve())) - } -}) diff --git a/src/services/code-index/embedders/__tests__/bedrock.spec.ts b/src/services/code-index/embedders/__tests__/bedrock.spec.ts index 8aa29e1d36..b671f029fc 100644 --- a/src/services/code-index/embedders/__tests__/bedrock.spec.ts +++ b/src/services/code-index/embedders/__tests__/bedrock.spec.ts @@ -1,8 +1,9 @@ -import { codeIndexLog } from "../../code-index-log" -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import { BedrockRuntimeClient, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime" import { BedrockEmbedder } from "../bedrock" +import { ProviderPacerRegistry } from "../../provider-pacer" + +const pacers = new ProviderPacerRegistry() import { MAX_ITEM_TOKENS } from "../../constants" import { createProxyRoutingRequestHandler } from "../../../../utils/networkProxy" @@ -67,8 +68,8 @@ vitest.mock("../../../../i18n", () => ({ // Mock console methods const consoleMocks = { - error: vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}), - warn: vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}), + error: vitest.spyOn(console, "error").mockImplementation(function () {}), + warn: vitest.spyOn(console, "warn").mockImplementation(function () {}), } describe("BedrockEmbedder", () => { @@ -89,7 +90,7 @@ describe("BedrockEmbedder", () => { return { send: mockSend } as unknown as BedrockRuntimeClient }) - embedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0") + embedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0", pacers) }) afterEach(() => { @@ -107,13 +108,13 @@ describe("BedrockEmbedder", () => { }) it("should require region", () => { - expect(() => new BedrockEmbedder("", "profile", "model")).toThrow( + expect(() => new BedrockEmbedder("", "profile", "model", pacers)).toThrow( "Region is required for AWS Bedrock embedder", ) }) it("should use profile for credentials", () => { - const profileEmbedder = new BedrockEmbedder("us-west-2", "dev-profile") + const profileEmbedder = new BedrockEmbedder("us-west-2", "dev-profile", undefined, pacers) expect(profileEmbedder).toBeDefined() }) @@ -134,7 +135,7 @@ describe("BedrockEmbedder", () => { } vitest.mocked(createProxyRoutingRequestHandler).mockReturnValue(handler) - new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0") + new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0", pacers) const clientConfig = vitest.mocked(BedrockRuntimeClient).mock.calls.at(-1)?.[0] expect(clientConfig?.requestHandler).toBe(handler) @@ -146,7 +147,7 @@ describe("BedrockEmbedder", () => { it("should keep the client default handler when no proxy is configured", () => { vitest.mocked(createProxyRoutingRequestHandler).mockReturnValue(undefined) - new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0") + new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0", pacers) expect(BedrockRuntimeClient).toHaveBeenLastCalledWith( expect.not.objectContaining({ requestHandler: expect.anything() }), @@ -224,7 +225,7 @@ describe("BedrockEmbedder", () => { }) it("should handle Cohere model format", async () => { - const cohereEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "cohere.embed-english-v3") + const cohereEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "cohere.embed-english-v3", pacers) const testTexts = ["Hello world"] const mockResponse = { body: new TextEncoder().encode( @@ -254,11 +255,7 @@ describe("BedrockEmbedder", () => { }) it("should create embeddings with Nova multimodal model", async () => { - const novaMultimodalEmbedder = new BedrockEmbedder( - "us-east-1", - "test-profile", - "amazon.nova-2-multimodal-embeddings-v1:0", - ) + const novaMultimodalEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.nova-2-multimodal-embeddings-v1:0", pacers) const testTexts = ["Hello world"] const mockResponse = { body: new TextEncoder().encode( @@ -303,11 +300,7 @@ describe("BedrockEmbedder", () => { }) it("should handle Nova multimodal model with multiple texts", async () => { - const novaMultimodalEmbedder = new BedrockEmbedder( - "us-east-1", - "test-profile", - "amazon.nova-2-multimodal-embeddings-v1:0", - ) + const novaMultimodalEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.nova-2-multimodal-embeddings-v1:0", pacers) const testTexts = ["Hello world", "Another text"] const mockResponses = [ { @@ -459,7 +452,7 @@ describe("BedrockEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) // Verify warning was logged - expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) // Verify only normal texts were processed expect(mockSend).toHaveBeenCalledTimes(2) @@ -472,7 +465,7 @@ describe("BedrockEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(codeIndexLog.warn).toHaveBeenCalledTimes(2) + expect(console.warn).toHaveBeenCalledTimes(2) expect(mockSend).not.toHaveBeenCalled() expect(result).toEqual({ embeddings: [], @@ -517,7 +510,7 @@ describe("BedrockEmbedder", () => { ) expect(mockSend).toHaveBeenCalledTimes(1) - expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) }) @@ -535,7 +528,7 @@ describe("BedrockEmbedder", () => { "Failed to create embeddings: API connection failed", ) - expect(codeIndexLog.error).toHaveBeenCalledWith( + expect(console.error).toHaveBeenCalledWith( expect.stringContaining("Bedrock embedder error"), expect.any(Error), ) @@ -676,6 +669,42 @@ describe("BedrockEmbedder", () => { expect(result.error).toBe("Connection failed") }) + describe("CI-EM-02: temporary failures", () => { + const okResponse = { body: new TextEncoder().encode(JSON.stringify({ embedding: [0.1] })) } + const serverError = () => Object.assign(new Error("Service unavailable"), { $metadata: { httpStatusCode: 503 } }) + const throttle = () => Object.assign(new Error("Slow down"), { name: "ThrottlingException" }) + + beforeEach(() => vitest.useFakeTimers()) + afterEach(() => vitest.useRealTimers()) + + it.each([ + ["two server failures", [serverError(), serverError()], 1500], + ["a throttle", [throttle()], 1000], + ])("recovers after %s and reports success", async (_label, failures, waitMs) => { + for (const failure of failures) mockSend.mockRejectedValueOnce(failure) + mockSend.mockResolvedValueOnce(okResponse) + const pending = embedder.validateConfiguration() + await vitest.advanceTimersByTimeAsync(waitMs) + await expect(pending).resolves.toEqual({ valid: true }) + expect(mockSend).toHaveBeenCalledTimes(failures.length + 1) + }) + + it("stops after three requests and returns a validation failure", async () => { + mockSend.mockRejectedValue(serverError()) + const pending = embedder.validateConfiguration() + await vitest.advanceTimersByTimeAsync(1500) + const result = await pending + expect(result.valid).toBe(false) + expect(mockSend).toHaveBeenCalledTimes(3) + }) + + it("does not retry a permanent failure", async () => { + mockSend.mockRejectedValue(Object.assign(new Error("Denied"), { name: "AccessDeniedException" })) + await expect(embedder.validateConfiguration()).resolves.toMatchObject({ valid: false }) + expect(mockSend).toHaveBeenCalledTimes(1) + }) + }) + it("should fail validation with generic error", async () => { const genericError = new Error("Unknown error") mockSend.mockRejectedValue(genericError) diff --git a/src/services/code-index/embedders/__tests__/cancellation.spec.ts b/src/services/code-index/embedders/__tests__/cancellation.spec.ts deleted file mode 100644 index d5e5016c75..0000000000 --- a/src/services/code-index/embedders/__tests__/cancellation.spec.ts +++ /dev/null @@ -1,261 +0,0 @@ -vi.mock("../../../../utils/networkProxy", () => ({ createProxyRoutingRequestHandler: vi.fn() })) -vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { instance: { captureEvent: vi.fn() } }, -})) -import { clearAllMocks } from "../../../../test-utils/reset" -import { OpenAI } from "openai" -import { BedrockRuntimeClient } from "@aws-sdk/client-bedrock-runtime" -import { OpenAiEmbedder } from "../openai" -import { OpenAICompatibleEmbedder } from "../openai-compatible" -import { OpenRouterEmbedder } from "../openrouter" -import { GeminiEmbedder } from "../gemini" -import { MistralEmbedder } from "../mistral" -import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" -import { CodeIndexOllamaEmbedder } from "../ollama" -import { BedrockEmbedder } from "../bedrock" -import type { IEmbedder } from "../../interfaces/embedder" -import { waitForRetry } from "../cancellation" - -const { request } = vi.hoisted(() => ({ request: vi.fn() })) - -vi.mock("openai", async (importOriginal) => ({ - ...(await importOriginal()), - OpenAI: vi.fn().mockImplementation(function () { - return { embeddings: { create: request } } - }), -})) -vi.mock("../../../../api/providers/openai-native", () => ({ - OpenAiNativeHandler: class { - constructor(protected options: object) {} - }, -})) -vi.mock("@aws-sdk/client-bedrock-runtime", () => ({ - BedrockRuntimeClient: vi.fn().mockImplementation(function () { - return { send: request } - }), - InvokeModelCommand: vi.fn().mockImplementation(function (input: unknown) { - return { input } - }), -})) -vi.mock("@aws-sdk/credential-providers", () => ({ - fromIni: vi.fn(), - fromNodeProviderChain: vi.fn(), -})) -vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) -vi.mock("../../../../shared/embeddingModels", () => ({ - getDefaultModelId: () => "test-model", - getModelQueryPrefix: () => "", -})) -vi.mock("../../constants", async (importOriginal) => ({ - ...(await importOriginal()), - MAX_BATCH_TOKENS: 4, -})) - -type Provider = { - name: string - create: () => IEmbedder - transport: "sdk" | "fetch" | "ollama" | "bedrock" -} - -const providers: Provider[] = [ - { name: "OpenAI", create: () => new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), transport: "sdk" }, - { - name: "OpenAI-compatible", - create: () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key"), - transport: "sdk", - }, - { name: "OpenRouter", create: () => new OpenRouterEmbedder("test-key"), transport: "sdk" }, - { name: "Gemini", create: () => new GeminiEmbedder("test-key"), transport: "sdk" }, - { name: "Mistral", create: () => new MistralEmbedder("test-key"), transport: "sdk" }, - { name: "Vercel", create: () => new VercelAiGatewayEmbedder("test-key"), transport: "sdk" }, - { name: "Ollama", create: () => new CodeIndexOllamaEmbedder({}), transport: "ollama" }, - { name: "Bedrock", create: () => new BedrockEmbedder("us-east-1"), transport: "bedrock" }, - { - name: "OpenAI-compatible direct fetch", - create: () => new OpenAICompatibleEmbedder("https://example.test/v1/embeddings", "test-key"), - transport: "fetch", - }, -] - -function deferred() { - let resolve!: (value: Value) => void - let reject!: (reason: unknown) => void - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise - reject = rejectPromise - }) - return { promise, resolve, reject } -} - -function successfulResponse(transport: Provider["transport"]) { - const data = { data: [{ embedding: [0.25, 0.5] }], embeddings: [[0.25, 0.5]] } - if (transport === "fetch" || transport === "ollama") { - return { ok: true, json: async () => data } - } - if (transport === "bedrock") { - return { body: new TextEncoder().encode(JSON.stringify({ embedding: [0.25, 0.5] })) } - } - return data -} - -beforeEach(() => { - vi.useFakeTimers() - request.mockReset() - vi.stubGlobal("fetch", request) - vi.spyOn(console, "warn").mockImplementation(() => {}) - vi.spyOn(console, "error").mockImplementation(() => {}) -}) - -afterEach(() => { - vi.unstubAllGlobals() - clearAllMocks() - vi.useRealTimers() -}) - -describe.each(providers)("$name cancellation", ({ create, transport }) => { - it("forwards the caller signal to the transport", async () => { - const controller = new AbortController() - request.mockResolvedValue(successfulResponse(transport)) - await create().createEmbeddings(["first"], undefined, controller.signal) - const options: { signal: AbortSignal; abortSignal: AbortSignal } = request.mock.calls[0][1] - if (transport === "bedrock") { - expect(options.abortSignal).toBe(controller.signal) - } else if (transport === "ollama") { - expect(options.signal.aborted).toBe(false) - controller.abort() - expect(options.signal.aborted).toBe(true) - expect(options.signal.reason).toBe(controller.signal.reason) - } else { - expect(options.signal).toBe(controller.signal) - } - }) - - it("refuses an already-aborted call without issuing a request", async () => { - const controller = new AbortController() - controller.abort(Object.freeze({ stopped: true })) - await expect(create().createEmbeddings(["first"], undefined, controller.signal)).rejects.toBe( - controller.signal.reason, - ) - expect(request).not.toHaveBeenCalled() - }) - - it.each(["success", "failure", "cancellation-error"])( - "discards late %s and issues no queued batch or item", - async (outcome) => { - const controller = new AbortController() - const pending = deferred() - const started = deferred() - request.mockImplementation(() => { - started.resolve() - return pending.promise - }) - const result = create().createEmbeddings(["first123", "second12", "third123"], undefined, controller.signal) - await started.promise - controller.abort(Object.freeze({ stopped: true })) - const requestOptions: { signal: AbortSignal; abortSignal: AbortSignal } = request.mock.calls[0][1] - const forwardedSignal = transport === "bedrock" ? requestOptions.abortSignal : requestOptions.signal - expect(forwardedSignal.aborted).toBe(true) - expect(forwardedSignal.reason).toBe(controller.signal.reason) - if (outcome === "success") { - pending.resolve(successfulResponse(transport)) - } else { - pending.reject( - outcome === "failure" ? new Error("Late failure") : new DOMException("Late abort", "AbortError"), - ) - } - await expect(result).rejects.toBe(controller.signal.reason) - expect(request).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - }, - ) - it.each([ - [429, "rate-limit"], - [503, "server"], - [408, "permanent"], - [409, "permanent"], - [401, "permanent"], - ] as const)("classifies HTTP %d without retrying", async (status, classification) => { - const failure = Object.assign(new Error("request failed"), { - status, - headers: new Headers({ "retry-after": "2" }), - $metadata: { httpStatusCode: status }, - }) - if (transport === "fetch" || transport === "ollama") { - request.mockResolvedValue(new Response("request failed", { status, headers: failure.headers })) - } else { - request.mockRejectedValue(failure) - } - const embedder = create() - await expect(embedder.createEmbeddings(["first"])).rejects.toMatchObject({ - classification, - statusCode: status, - retryAfterMs: 2000, - }) - expect(request).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - if (transport === "sdk") { - expect(OpenAI).toHaveBeenLastCalledWith(expect.objectContaining({ maxRetries: 0 })) - } else if (transport === "bedrock") { - expect(BedrockRuntimeClient).toHaveBeenLastCalledWith(expect.objectContaining({ maxAttempts: 1 })) - } - }) - - it("classifies transport connection failures without retrying", async () => { - request.mockRejectedValue(Object.assign(new Error("connection failed"), { code: "ECONNRESET" })) - await expect(create().createEmbeddings(["first"])).rejects.toMatchObject({ classification: "connection" }) - expect(request).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - }) -}) - -describe("cancellable wait", () => { - it("refuses a previously aborted signal", () => { - const controller = new AbortController() - controller.abort() - expect(() => waitForRetry(100, controller.signal)).toThrow(controller.signal.reason) - expect(vi.getTimerCount()).toBe(0) - }) - - it("removes its abort listener when the wait completes", async () => { - const controller = new AbortController() - const addListener = vi.spyOn(controller.signal, "addEventListener") - const removeListener = vi.spyOn(controller.signal, "removeEventListener") - const result = waitForRetry(100, controller.signal) - await vi.advanceTimersByTimeAsync(100) - await result - expect(addListener).toHaveBeenCalledOnce() - expect(removeListener).toHaveBeenCalledWith("abort", addListener.mock.calls[0][1]) - expect(vi.getTimerCount()).toBe(0) - }) -}) - -it("Ollama keeps its timeout when a caller signal is supplied and cleans up after rejection", async () => { - const controller = new AbortController() - request.mockImplementation( - (_url: unknown, options: { signal: AbortSignal }) => - new Promise((_resolve, reject) => { - options.signal.addEventListener("abort", () => reject(options.signal.reason), { once: true }) - }), - ) - const result = new CodeIndexOllamaEmbedder({}).createEmbeddings(["first"], undefined, controller.signal) - const assertion = expect(result).rejects.toThrow("embeddings:validation.connectionFailed") - await vi.advanceTimersByTimeAsync(60_000) - await assertion - expect(controller.signal.aborted).toBe(false) - const options: { signal: AbortSignal } = request.mock.calls[0][1] - expect(options.signal.aborted).toBe(true) - expect(vi.getTimerCount()).toBe(0) -}) - -it("cancels a wait with the unchanged reason and removes its timer and listener", async () => { - const controller = new AbortController() - const addListener = vi.spyOn(controller.signal, "addEventListener") - const removeListener = vi.spyOn(controller.signal, "removeEventListener") - const reason = Object.freeze({ stopped: true }) - const pending = waitForRetry(60_000, controller.signal) - controller.abort(reason) - await expect(pending).rejects.toBe(reason) - expect(addListener).toHaveBeenCalledOnce() - expect(removeListener).toHaveBeenCalledWith("abort", addListener.mock.calls[0][1]) - expect(vi.getTimerCount()).toBe(0) -}) diff --git a/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts new file mode 100644 index 0000000000..a807b8743a --- /dev/null +++ b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts @@ -0,0 +1,26 @@ +import { EmbedderFactory } from "../embedder-factory" +import { BedrockEmbedder } from "../bedrock" +import { ProviderPacerRegistry } from "../../provider-pacer" +import type { CodeIndexConfig } from "../../interfaces/config" + +vi.mock("../bedrock", () => ({ BedrockEmbedder: vi.fn() })) + +describe("EmbedderFactory (CI-EM-03)", () => { + const config: CodeIndexConfig = { + isConfigured: true, + embedderProvider: "bedrock", + modelId: "amazon.titan-embed-text-v2:0", + bedrockOptions: { region: "us-east-1", profile: "dev" }, + } + + it("passes the workspace pacer registry through to BedrockEmbedder", () => { + const pacers = new ProviderPacerRegistry() + new EmbedderFactory().create(config, pacers) + expect(BedrockEmbedder).toHaveBeenLastCalledWith("us-east-1", "dev", "amazon.titan-embed-text-v2:0", pacers) + }) + + it("still creates a Bedrock embedder when no registry is supplied", () => { + new EmbedderFactory().create(config) + expect(vi.mocked(BedrockEmbedder).mock.lastCall?.[3]).toBeInstanceOf(ProviderPacerRegistry) + }) +}) diff --git a/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts b/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts index b568436ce3..7128695e8f 100644 --- a/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts +++ b/src/services/code-index/embedders/__tests__/embedder-validation-manager.spec.ts @@ -29,15 +29,6 @@ describe("EmbedderValidationManager", () => { }, ) - it("rethrows rate limits unchanged without permanent-error telemetry", async () => { - const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) - const error = Object.assign(new Error("limited"), { status: 429, headers: { "retry-after": "120" } }) - vi.mocked(embedder.validateConfiguration).mockRejectedValue(error) - - await expect(manager.validateEmbedder(embedder)).rejects.toBe(error) - expect(TelemetryService.instance.captureEvent).not.toHaveBeenCalled() - }) - it("preserves an exception's message and reports its stack", async () => { const embedder = new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }) const error = new Error("Connection failed") diff --git a/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts index 76d0411c14..996e1cb398 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible-rate-limit.spec.ts @@ -1,4 +1,3 @@ -vi.mock("../../code-index-log", () => ({ codeIndexLog: { info: vi.fn(), error: vi.fn(), warn: vi.fn() } })) vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: { captureEvent: vi.fn() } }, })) diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index afe5621c5d..08380d0e80 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -1,5 +1,3 @@ -import { codeIndexLog } from "../../code-index-log" -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import type { MockedClass, MockedFunction } from "vitest" import { OpenAI } from "openai" import { OpenAICompatibleEmbedder } from "../openai-compatible" @@ -66,8 +64,8 @@ describe("OpenAICompatibleEmbedder", () => { beforeEach(() => { clearAllMocks() - vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}) - vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}) + vitest.spyOn(console, "warn").mockImplementation(function () {}) + vitest.spyOn(console, "error").mockImplementation(function () {}) // Setup mock OpenAI instance mockEmbeddingsCreate = vitest.fn() @@ -371,7 +369,7 @@ describe("OpenAICompatibleEmbedder", () => { await embedder.createEmbeddings(testTexts) // Should warn about oversized text - expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("exceeds maximum token limit")) // Should only process normal texts (1 call for 2 normal texts batched together) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) @@ -430,7 +428,7 @@ describe("OpenAICompatibleEmbedder", () => { ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) - expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) it("should throw error immediately on non-retryable errors", async () => { @@ -462,7 +460,7 @@ describe("OpenAICompatibleEmbedder", () => { "Failed to create embeddings: API connection failed", ) - expect(codeIndexLog.error).toHaveBeenCalledWith( + expect(console.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI Compatible embedder error"), apiError, ) @@ -478,7 +476,7 @@ describe("OpenAICompatibleEmbedder", () => { "Failed to create embeddings: Batch processing failed", ) - expect(codeIndexLog.error).toHaveBeenCalledWith( + expect(console.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI Compatible embedder error"), expect.any(Error), ) diff --git a/src/services/code-index/embedders/__tests__/openai.spec.ts b/src/services/code-index/embedders/__tests__/openai.spec.ts index 3fa2513175..5918ce83c3 100644 --- a/src/services/code-index/embedders/__tests__/openai.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai.spec.ts @@ -1,5 +1,3 @@ -import { codeIndexLog } from "../../code-index-log" -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import type { MockedClass } from "vitest" import { OpenAI } from "openai" @@ -37,8 +35,8 @@ vitest.mock("../../../../i18n", () => ({ // Mock console methods const consoleMocks = { - error: vitest.spyOn(codeIndexLog, "error").mockImplementation(function () {}), - warn: vitest.spyOn(codeIndexLog, "warn").mockImplementation(function () {}), + error: vitest.spyOn(console, "error").mockImplementation(function () {}), + warn: vitest.spyOn(console, "warn").mockImplementation(function () {}), } describe("OpenAiEmbedder", () => { @@ -214,7 +212,7 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) // Verify warning was logged - expect(codeIndexLog.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) // Verify only normal texts were processed expect(mockEmbeddingsCreate).toHaveBeenCalledWith( @@ -268,7 +266,7 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(codeIndexLog.warn).toHaveBeenCalledTimes(2) + expect(console.warn).toHaveBeenCalledTimes(2) expect(mockEmbeddingsCreate).not.toHaveBeenCalled() expect(result).toEqual({ embeddings: [], @@ -313,7 +311,7 @@ describe("OpenAiEmbedder", () => { ) expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) - expect(codeIndexLog.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) + expect(console.warn).not.toHaveBeenCalledWith(expect.stringContaining("Rate limit hit")) }) it("should throw error immediately on non-retryable errors", async () => { @@ -345,7 +343,7 @@ describe("OpenAiEmbedder", () => { "Failed to create embeddings: API connection failed", ) - expect(codeIndexLog.error).toHaveBeenCalledWith( + expect(console.error).toHaveBeenCalledWith( expect.stringContaining("OpenAI embedder error"), expect.any(Error), ) diff --git a/src/services/code-index/embedders/__tests__/openrouter.spec.ts b/src/services/code-index/embedders/__tests__/openrouter.spec.ts index e6080b1d1b..088334ad57 100644 --- a/src/services/code-index/embedders/__tests__/openrouter.spec.ts +++ b/src/services/code-index/embedders/__tests__/openrouter.spec.ts @@ -1,5 +1,3 @@ -import { codeIndexLog } from "../../code-index-log" -vi.mock("../../code-index-log", () => ({ codeIndexLog: { error: vi.fn(), warn: vi.fn() } })) import type { MockedClass } from "vitest" import { describe, it, expect, beforeEach, vi } from "vitest" import { OpenAI } from "openai" @@ -47,8 +45,8 @@ describe("OpenRouterEmbedder", () => { beforeEach(() => { clearAllMocks() - vi.spyOn(codeIndexLog, "warn").mockImplementation(function () {}) - vi.spyOn(codeIndexLog, "error").mockImplementation(function () {}) + vi.spyOn(console, "warn").mockImplementation(function () {}) + vi.spyOn(console, "error").mockImplementation(function () {}) // Setup mock OpenAI instance mockEmbeddingsCreate = vi.fn() diff --git a/src/services/code-index/embedders/__tests__/paced-transport-writer.ts b/src/services/code-index/embedders/__tests__/paced-transport-writer.ts deleted file mode 100644 index 37c078ec46..0000000000 --- a/src/services/code-index/embedders/__tests__/paced-transport-writer.ts +++ /dev/null @@ -1,82 +0,0 @@ -import path from "node:path" -import { setTimeout as sleep } from "node:timers/promises" -import { FileIndexWriter } from "../../file-completion" -import { FileFailureRegistry } from "../../file-failure-registry" -import { IndexWriteGate } from "../../index-write-gate" -import { ProviderPacerRegistry } from "../../provider-pacer" -import type { CacheManager } from "../../cache-manager" -import type { GenerationAllocator } from "../../generation-allocator" -import type { IEmbedder, IVectorStore, CodeBlock } from "../../interfaces" - -export function pacedTransportWriter(store?: IVectorStore) { - const workspace = path.resolve("/workspace") - const file = path.join(workspace, "test.ts") - const failures = new FileFailureRegistry() - const pacers = new ProviderPacerRegistry() - const cache = { - invalidateFile: vi.fn().mockResolvedValue(undefined), - completeFile: vi.fn(), - completeRemoval: vi.fn(), - } - const vectorStore: IVectorStore = store ?? { - rateLimitScope: "qdrant:test", - initialize: vi.fn().mockResolvedValue(false), - upsertPoints: vi.fn().mockResolvedValue(undefined), - deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), - deletePointsByMultipleFilePaths: vi - .fn() - .mockResolvedValue(undefined), - countPointsForFile: vi.fn().mockResolvedValue({ current: 1, other: 0 }), - search: vi.fn().mockResolvedValue([]), - clearCollection: vi.fn().mockResolvedValue(undefined), - deleteCollection: vi.fn().mockResolvedValue(undefined), - collectionExists: vi.fn().mockResolvedValue(true), - hasIndexedData: vi.fn().mockResolvedValue(true), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - } - const gate = new IndexWriteGate() - // The cache supplies invalidation/completion; the allocator supplies only allocate. - const writer = new FileIndexWriter( - cache as unknown as CacheManager, - vectorStore, - gate, - { allocate: async () => 1 } as unknown as GenerationAllocator, - workspace, - failures, - pacers, - ) - const block: CodeBlock = { - file_path: file, - identifier: null, - type: "function", - start_line: 1, - end_line: 1, - content: "test", - fileHash: "revision", - segmentHash: "segment", - } - return { - writer, - gate, - pacers, - failures, - cache, - file, - replace: (embedder: IEmbedder, signal = new AbortController().signal) => - writer.replace(file, "revision", [block], embedder, signal), - } -} - -export async function waitForTransport(assertion: () => void): Promise { - const deadline = performance.now() + 2000 - while (true) { - try { - assertion() - return - } catch (error) { - if (performance.now() >= deadline) throw error - await sleep(1) - } - } -} diff --git a/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts b/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts deleted file mode 100644 index 27f793909b..0000000000 --- a/src/services/code-index/embedders/__tests__/rate-limit-delay.spec.ts +++ /dev/null @@ -1,130 +0,0 @@ -import "../../__tests__/mock-code-index-log" -import { OpenAiEmbedder } from "../openai" -import { OpenAICompatibleEmbedder } from "../openai-compatible" -import { OpenRouterEmbedder } from "../openrouter" -import { GeminiEmbedder } from "../gemini" -import { MistralEmbedder } from "../mistral" -import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" -import { CodeIndexOllamaEmbedder } from "../ollama" -import { pacedTransportWriter } from "./paced-transport-writer" -import { codeIndexLog } from "../../code-index-log" - -vi.mock("../../../../api/providers/openai-native", () => ({ - OpenAiNativeHandler: class { - constructor(protected options: unknown) {} - }, -})) -vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) - -const factories = [ - ["OpenAI", () => new OpenAiEmbedder({ openAiNativeApiKey: "test-key" })], - ["compatible SDK", () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key")], - ["compatible fetch", () => new OpenAICompatibleEmbedder("https://example.test/embeddings", "test-key")], - ["OpenRouter", () => new OpenRouterEmbedder("test-key")], - ["Gemini", () => new GeminiEmbedder("test-key")], - ["Mistral", () => new MistralEmbedder("test-key")], - ["Vercel", () => new VercelAiGatewayEmbedder("test-key")], - ["Ollama", () => new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "https://example.test" })], -] as const - -beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(new Date("2026-11-11T12:00:00Z")) - vi.spyOn(Math, "random").mockReturnValue(0) - vi.mocked(codeIndexLog.info).mockClear() -}) -afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -function response(status = 200, headers: Record = {}) { - return new Response( - JSON.stringify( - status === 200 - ? { data: [{ embedding: [1, 0] }], embeddings: [[1, 0]] } - : { error: { message: "limited" } }, - ), - { - status, - headers: { "content-type": "application/json", ...headers }, - }, - ) -} - -describe.each(factories)("T25 %s paced transport", (_name, createEmbedder) => { - it.each( - [2, 60, 120].flatMap((seconds) => [ - { header: "retry-after", value: String(seconds), delay: seconds * 1000 }, - { header: "retry-after-ms", value: String(seconds * 1000), delay: seconds * 1000 }, - { - header: "retry-after", - value: new Date(Date.parse("2026-11-11T12:00:00Z") + seconds * 1000).toUTCString(), - delay: seconds * 1000, - }, - ]), - )("honors the entire $header: $value", async ({ header, value, delay }) => { - const request = vi - .fn() - .mockImplementationOnce(async () => response(429, { [header]: value })) - .mockImplementation(async () => response()) - vi.stubGlobal("fetch", request) - const { replace, failures, cache } = pacedTransportWriter() - const pending = replace(createEmbedder()) - await vi.advanceTimersByTimeAsync(0) - expect(request).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync(delay - 1) - expect(request).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(1) - await pending - expect(request).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() - }) - - it.each([undefined, "invalid", "-1", "Tue, 10 Nov 2026 12:00:00 GMT"])( - "recovers after four limits with growing fallback for %s and no budget charge", - async (header) => { - let issued = 0 - const request = vi.fn(async () => - ++issued <= 4 ? response(429, header === undefined ? {} : { "retry-after": header }) : response(), - ) - vi.stubGlobal("fetch", request) - const { replace, failures, cache } = pacedTransportWriter() - const pending = replace(createEmbedder()) - await vi.advanceTimersByTimeAsync(0) - for (const [index, delay] of [500, 1000, 2000, 4000].entries()) { - expect(request).toHaveBeenCalledTimes(index + 1) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(delay - 1) - expect(request).toHaveBeenCalledTimes(index + 1) - await vi.advanceTimersByTimeAsync(1) - } - await pending - expect(request).toHaveBeenCalledTimes(5) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() - expect(codeIndexLog.info).toHaveBeenCalledTimes(4) - }, - ) - - it("cancels a provider wait without another transport request or failure entry", async () => { - const request = vi.fn(async () => response(429, { "retry-after": "120" })) - vi.stubGlobal("fetch", request) - const controller = new AbortController() - const { replace, failures, pacers } = pacedTransportWriter() - const pending = replace(createEmbedder(), controller.signal).catch((error: unknown) => error) - await vi.advanceTimersByTimeAsync(0) - expect(request).toHaveBeenCalledOnce() - expect(pacers.waitingPaths()).toEqual(["test.ts"]) - expect(failures.list()).toEqual([]) - controller.abort() - expect(await pending).toBe(controller.signal.reason) - await vi.advanceTimersByTimeAsync(240_000) - expect(request).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([]) - }) -}) diff --git a/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts b/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts deleted file mode 100644 index 6616bbf72f..0000000000 --- a/src/services/code-index/embedders/__tests__/rate-limit-scopes.spec.ts +++ /dev/null @@ -1,71 +0,0 @@ -import "../../__tests__/mock-code-index-log" -import { OpenAiEmbedder } from "../openai" -import { OpenAICompatibleEmbedder } from "../openai-compatible" -import { OpenRouterEmbedder } from "../openrouter" -import { GeminiEmbedder } from "../gemini" -import { MistralEmbedder } from "../mistral" -import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" -import { CodeIndexOllamaEmbedder } from "../ollama" -import { BedrockEmbedder } from "../bedrock" -import { ProviderPacerRegistry } from "../../provider-pacer" - -vi.mock("../../../../api/providers/openai-native", () => ({ - OpenAiNativeHandler: class { - constructor(protected options: unknown) {} - }, -})) -vi.mock("@aws-sdk/client-bedrock-runtime", () => ({ BedrockRuntimeClient: vi.fn(), InvokeModelCommand: vi.fn() })) -vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) - -const factories = [ - ["openai", (key: string) => new OpenAiEmbedder({ openAiNativeApiKey: key })], - ["openai-compatible", (key: string) => new OpenAICompatibleEmbedder("https://example.test/v1", key)], - ["openrouter", (key: string) => new OpenRouterEmbedder(key)], - ["gemini", (key: string) => new GeminiEmbedder(key)], - ["mistral", (key: string) => new MistralEmbedder(key)], - ["vercel-ai-gateway", (key: string) => new VercelAiGatewayEmbedder(key)], -] as const - -it.each(factories)("%s shares identical allowances and isolates credentials", (provider, createEmbedder) => { - const first = createEmbedder("first-secret") - const recreated = createEmbedder("first-secret") - const other = createEmbedder("second-secret") - expect(first.rateLimitScope).toMatch(new RegExp(`^${provider}:[a-f0-9]{64}$`)) - expect(first.rateLimitScope).toBe(recreated.rateLimitScope) - expect(first.rateLimitScope).not.toBe(other.rateLimitScope) - expect(first.rateLimitScope).not.toContain("first-secret") - const registry = new ProviderPacerRegistry() - expect(registry.get(first.rateLimitScope)).toBe(registry.get(recreated.rateLimitScope)) - expect(registry.get(first.rateLimitScope)).not.toBe(registry.get(other.rateLimitScope)) -}) - -it("normalizes equivalent URLs while isolating different endpoints", () => { - const first = new OpenAICompatibleEmbedder("https://EXAMPLE.test:443/v1/", "key") - const same = new OpenAICompatibleEmbedder("https://example.test/v1", "key") - const other = new OpenAICompatibleEmbedder("https://example.test/v2", "key") - expect(first.rateLimitScope).toBe(same.rateLimitScope) - expect(first.rateLimitScope).not.toBe(other.rateLimitScope) -}) - -it("Ollama scopes depend only on the normalized endpoint", () => { - const first = new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "http://LOCALHOST:11434/" }) - const same = new CodeIndexOllamaEmbedder({ ollamaModelId: "another-model" }) - const other = new CodeIndexOllamaEmbedder({ ollamaBaseUrl: "http://localhost:11435" }) - expect(first.rateLimitScope).toBe(same.rateLimitScope) - expect(first.rateLimitScope).not.toBe(other.rateLimitScope) -}) - -it("Bedrock isolates region and profile, not model", () => { - const first = new BedrockEmbedder("us-east-1", "profile") - expect(first.rateLimitScope).toBe(new BedrockEmbedder("us-east-1", "profile", "other-model").rateLimitScope) - expect(first.rateLimitScope).not.toBe(new BedrockEmbedder("us-west-2", "profile").rateLimitScope) - expect(first.rateLimitScope).not.toBe(new BedrockEmbedder("us-east-1", "other-profile").rateLimitScope) - expect(first.rateLimitScope).toMatch(/^bedrock:[a-f0-9]{64}$/) - expect(first.rateLimitScope).not.toContain("profile") -}) - -it("wrapped providers retain their own provider identity", () => { - const gemini = new GeminiEmbedder("key") - const compatible = new OpenAICompatibleEmbedder("https://generativelanguage.googleapis.com/v1beta/openai/", "key") - expect(gemini.rateLimitScope).not.toBe(compatible.rateLimitScope) -}) diff --git a/src/services/code-index/embedders/bedrock.ts b/src/services/code-index/embedders/bedrock.ts index 3496dd4552..c2f8e683e3 100644 --- a/src/services/code-index/embedders/bedrock.ts +++ b/src/services/code-index/embedders/bedrock.ts @@ -1,5 +1,3 @@ -import { codeIndexLog } from "../code-index-log" -import { makeRateLimitScope } from "../provider-pacer" import { BedrockRuntimeClient, InvokeModelCommand, InvokeModelCommandInput } from "@aws-sdk/client-bedrock-runtime" import { fromIni, fromNodeProviderChain } from "@aws-sdk/credential-providers" import { createProxyRoutingRequestHandler } from "../../../utils/networkProxy" @@ -8,40 +6,38 @@ import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId } from "../../../shared/embeddingModels" import { Package } from "../../../shared/package" import { t } from "../../../i18n" -import { withValidationErrorHandling, formatEmbeddingError } from "../shared/validation-helpers" +import { withValidationErrorHandling, toEmbeddingError } from "../shared/validation-helpers" +import { EmbeddingError } from "../shared/indexing-errors" +import { retryWithPolicy } from "../shared/retry-policy" +import { makeRateLimitScope, type ProviderPacerRegistry } from "../provider-pacer" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" /** - * Amazon Bedrock implementation of the embedder interface with batching + * Amazon Bedrock implementation of the embedder interface with batching and rate limiting */ export class BedrockEmbedder implements IEmbedder { - readonly rateLimitScope: string private bedrockClient: BedrockRuntimeClient private readonly defaultModelId: string + readonly rateLimitScope: string /** * Creates a new Amazon Bedrock embedder * @param region AWS region for Bedrock service (required) * @param profile AWS profile name for credentials (optional - uses default credential chain if not provided) * @param modelId Optional model ID override + * @param pacers Shared rate-limit waits; validation requests honor the same pause as indexing */ constructor( - private readonly region: string, - private readonly profile?: string, - modelId?: string, + private readonly region: string, + private readonly profile: string | undefined, + modelId: string | undefined, + private readonly pacers: ProviderPacerRegistry, ) { if (!region) { throw new Error("Region is required for AWS Bedrock embedder") } - this.rateLimitScope = makeRateLimitScope( - "bedrock", - undefined, - undefined, - JSON.stringify([region, profile ?? ""]), - ) - // Initialize the Bedrock client with credentials // If profile is specified, use it; otherwise use default credential chain const credentials = this.profile ? fromIni({ profile: this.profile }) : fromNodeProviderChain() @@ -53,23 +49,24 @@ export class BedrockEmbedder implements IEmbedder { this.bedrockClient = new BedrockRuntimeClient({ userAgentAppId: `ZooCode#${Package.version}`, - maxAttempts: 1, region: this.region, credentials, + // Retries are owned by the indexing retry policy; SDK retries would multiply its allowance. + maxAttempts: 1, ...(requestHandler && { requestHandler }), }) this.defaultModelId = modelId || getDefaultModelId("bedrock") + this.rateLimitScope = makeRateLimitScope("bedrock", undefined, undefined, `${region}|${profile ?? ""}`) } /** - * Creates embeddings for the given texts with batching + * Creates embeddings for the given texts with batching and rate limiting * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { const modelToUse = model || this.defaultModelId const allEmbeddings: number[][] = [] @@ -77,7 +74,6 @@ export class BedrockEmbedder implements IEmbedder { const remainingTexts = [...texts] while (remainingTexts.length > 0) { - signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -87,7 +83,7 @@ export class BedrockEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -113,7 +109,7 @@ export class BedrockEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) + const batchResult = await this._embedBatch(currentBatch, modelToUse) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -123,42 +119,31 @@ export class BedrockEmbedder implements IEmbedder { return { embeddings: allEmbeddings, usage } } + /** Sends one request per text; retries and rate-limit waits belong to the caller's retry policy and pacer. */ private async _embedBatch( batchTexts: string[], model: string, - signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { try { - signal?.throwIfAborted() const embeddings: number[][] = [] - let totalPromptTokens = 0 let totalTokens = 0 - // Amazon Titan models typically need one request per text. + // Amazon Titan models typically don't support batch embedding in a single request. for (const text of batchTexts) { - signal?.throwIfAborted() - const embedding = await this._invokeEmbeddingModel(text, model, signal) + const embedding = await this._invokeEmbeddingModel(text, model) embeddings.push(embedding.embedding) - totalPromptTokens += embedding.inputTextTokenCount || 0 totalTokens += embedding.inputTextTokenCount || 0 } - return { - embeddings, - usage: { - promptTokens: totalPromptTokens, - totalTokens, - }, - } + return { embeddings, usage: { promptTokens: totalTokens, totalTokens } } } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, location: "BedrockEmbedder:_embedBatch", }) - codeIndexLog.error("Bedrock embedder error:", error) - throw formatEmbeddingError(error) + console.error("Bedrock embedder error:", error) + throw toEmbeddingError(asRateLimit(error)) } } @@ -171,7 +156,6 @@ export class BedrockEmbedder implements IEmbedder { private async _invokeEmbeddingModel( text: string, model: string, - signal?: AbortSignal, ): Promise<{ embedding: number[]; inputTextTokenCount?: number }> { let requestBody: any const modelId = model @@ -224,9 +208,7 @@ export class BedrockEmbedder implements IEmbedder { const command = new InvokeModelCommand(params) - signal?.throwIfAborted() - const response = await this.bedrockClient.send(command, ...(signal ? [{ abortSignal: signal }] : [])) - signal?.throwIfAborted() + const response = await this.bedrockClient.send(command) // Parse the response const responseBody = JSON.parse(new TextDecoder().decode(response.body)) @@ -270,8 +252,18 @@ export class BedrockEmbedder implements IEmbedder { async validateConfiguration(): Promise<{ valid: boolean; error?: string }> { return withValidationErrorHandling(async () => { try { - // Test with a minimal embedding request - const result = await this._invokeEmbeddingModel("test", this.defaultModelId) + // Temporary server, connection and throttling failures recover instead of failing validation. + const result = await retryWithPolicy( + () => + this.pacers.get(this.rateLimitScope).run(undefined, undefined, async () => { + try { + return await this._invokeEmbeddingModel("test", this.defaultModelId) + } catch (error) { + throw asRateLimit(error) + } + }), + { budget: { retriesUsed: 0 } }, + ) // Check if we got a valid response if (!result.embedding || result.embedding.length === 0) { @@ -282,7 +274,8 @@ export class BedrockEmbedder implements IEmbedder { } return { valid: true } - } catch (error: any) { + } catch (caught: any) { + const error = caught?.cause ?? caught // Check for specific AWS errors if (error.name === "UnrecognizedClientException") { return { @@ -322,3 +315,11 @@ export class BedrockEmbedder implements IEmbedder { } } } + +// The AWS SDK reports throttling by name; give it a 429 so the shared classifier treats it as a rate limit. +function asRateLimit(error: unknown): unknown { + if (error instanceof Error && error.name === "ThrottlingException") { + return new EmbeddingError(error.message, error, { classification: "rate-limit", statusCode: 429 }) + } + return error +} diff --git a/src/services/code-index/embedders/embedder-factory.ts b/src/services/code-index/embedders/embedder-factory.ts index 31d6e8a754..3dddac4ab6 100644 --- a/src/services/code-index/embedders/embedder-factory.ts +++ b/src/services/code-index/embedders/embedder-factory.ts @@ -4,6 +4,7 @@ import { t } from "../../../i18n" import type { CodeIndexConfig } from "../interfaces/config" import type { IEmbedder } from "../interfaces" import type { IEmbedderFactory } from "../interfaces/embedder-factory" +import type { ProviderPacerRegistry } from "../provider-pacer" import { OpenAiEmbedderFactory } from "./factories/openai-embedder-factory" import { OllamaEmbedderFactory } from "./factories/ollama-embedder-factory" import { OpenAICompatibleEmbedderFactory } from "./factories/openai-compatible-embedder-factory" @@ -31,7 +32,7 @@ export class EmbedderFactory implements IEmbedderFactory { [providerIdentifiers.openrouter]: new OpenRouterEmbedderFactory(), } - public create(config: CodeIndexConfig): IEmbedder { + public create(config: CodeIndexConfig, pacers?: ProviderPacerRegistry): IEmbedder { const provider = config.embedderProvider // Semble is a self-contained search binary, not a vector embedder. @@ -46,6 +47,6 @@ export class EmbedderFactory implements IEmbedderFactory { throw new Error(t("embeddings:serviceFactory.invalidEmbedderType", { embedderProvider: provider })) } - return this.embedderFactories[provider].create(config) + return this.embedderFactories[provider].create(config, pacers) } } diff --git a/src/services/code-index/embedders/embedder-validation-manager.ts b/src/services/code-index/embedders/embedder-validation-manager.ts index 92fd31e662..84dd73aac6 100644 --- a/src/services/code-index/embedders/embedder-validation-manager.ts +++ b/src/services/code-index/embedders/embedder-validation-manager.ts @@ -2,14 +2,12 @@ import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" import { t } from "../../../i18n" import type { IEmbedder } from "../interfaces" -import { classifyIndexingError } from "../shared/indexing-errors" export class EmbedderValidationManager { public async validateEmbedder(embedder: IEmbedder): Promise<{ valid: boolean; error?: string }> { try { return await embedder.validateConfiguration() } catch (error) { - if (classifyIndexingError(error) === "rate-limit") throw error TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts b/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts index 5a93c5619b..4de2de4b16 100644 --- a/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts +++ b/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts @@ -1,14 +1,16 @@ import type { CodeIndexConfig } from "../../interfaces/config" import type { IEmbedderFactory } from "../../interfaces/embedder-factory" +import { ProviderPacerRegistry } from "../../provider-pacer" import { BedrockEmbedder } from "../bedrock" import { requireSetting } from "./require-setting" export class BedrockEmbedderFactory implements IEmbedderFactory { - create({ bedrockOptions, modelId }: CodeIndexConfig): BedrockEmbedder { + create({ bedrockOptions, modelId }: CodeIndexConfig, pacers = new ProviderPacerRegistry()): BedrockEmbedder { return new BedrockEmbedder( requireSetting(bedrockOptions?.region, "embeddings:serviceFactory.bedrockConfigMissing"), bedrockOptions?.profile, modelId, + pacers, ) } } diff --git a/src/services/code-index/embedders/gemini.ts b/src/services/code-index/embedders/gemini.ts index 29f8e1f2b6..e09e37f2ff 100644 --- a/src/services/code-index/embedders/gemini.ts +++ b/src/services/code-index/embedders/gemini.ts @@ -1,5 +1,4 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" -import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { GEMINI_MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -17,8 +16,6 @@ import { TelemetryService } from "@roo-code/telemetry" * migrated to gemini-embedding-001 for backward compatibility. */ export class GeminiEmbedder implements IEmbedder { - readonly rateLimitScope: string - private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/" private static readonly DEFAULT_MODEL = "gemini-embedding-001" @@ -31,6 +28,10 @@ export class GeminiEmbedder implements IEmbedder { } private readonly modelId: string + get rateLimitScope(): string { + return this.openAICompatibleEmbedder.rateLimitScope + } + /** * Migrates deprecated model IDs to their replacements. * @param modelId The model ID to potentially migrate @@ -50,8 +51,6 @@ export class GeminiEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } - this.rateLimitScope = makeRateLimitScope("gemini", GeminiEmbedder.GEMINI_BASE_URL, apiKey) - // Migrate deprecated models to their replacements silently const migratedModelId = modelId ? GeminiEmbedder.migrateModelId(modelId) : undefined @@ -73,14 +72,12 @@ export class GeminiEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/mistral.ts b/src/services/code-index/embedders/mistral.ts index 174b9424aa..a6895d3b50 100644 --- a/src/services/code-index/embedders/mistral.ts +++ b/src/services/code-index/embedders/mistral.ts @@ -1,5 +1,4 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" -import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -14,13 +13,15 @@ import { TelemetryService } from "@roo-code/telemetry" * - codestral-embed-2505 (dimension: 1536) */ export class MistralEmbedder implements IEmbedder { - readonly rateLimitScope: string - private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly MISTRAL_BASE_URL = "https://api.mistral.ai/v1" private static readonly DEFAULT_MODEL = "codestral-embed-2505" private readonly modelId: string + get rateLimitScope(): string { + return this.openAICompatibleEmbedder.rateLimitScope + } + /** * Creates a new Mistral embedder * @param apiKey The Mistral API key for authentication @@ -31,8 +32,6 @@ export class MistralEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } - this.rateLimitScope = makeRateLimitScope("mistral", MistralEmbedder.MISTRAL_BASE_URL, apiKey) - // Use provided model or default this.modelId = modelId || MistralEmbedder.DEFAULT_MODEL @@ -51,14 +50,12 @@ export class MistralEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/embedders/ollama.ts b/src/services/code-index/embedders/ollama.ts index 56710209fe..c75ce632b8 100644 --- a/src/services/code-index/embedders/ollama.ts +++ b/src/services/code-index/embedders/ollama.ts @@ -1,12 +1,11 @@ -import { codeIndexLog } from "../code-index-log" -import { EmbeddingError } from "../shared/indexing-errors" -import { makeRateLimitScope } from "../provider-pacer" import { ApiHandlerOptions } from "../../../shared/api" import { EmbedderInfo, EmbeddingResponse, IEmbedder } from "../interfaces" import { getModelQueryPrefix } from "../../../shared/embeddingModels" import { MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" import { withValidationErrorHandling, sanitizeErrorMessage } from "../shared/validation-helpers" +import { EmbeddingError } from "../shared/indexing-errors" +import { makeRateLimitScope } from "../provider-pacer" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" @@ -18,9 +17,9 @@ const OLLAMA_VALIDATION_TIMEOUT_MS = 30000 // 30 seconds for validation requests * Implements the IEmbedder interface using a local Ollama instance. */ export class CodeIndexOllamaEmbedder implements IEmbedder { - readonly rateLimitScope: string private readonly baseUrl: string private readonly defaultModelId: string + readonly rateLimitScope: string constructor(options: ApiHandlerOptions) { // Ensure ollamaBaseUrl and ollamaModelId exist on ApiHandlerOptions or add defaults @@ -30,8 +29,8 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { baseUrl = baseUrl.replace(/\/+$/, "") this.baseUrl = baseUrl - this.rateLimitScope = makeRateLimitScope("ollama", baseUrl) this.defaultModelId = options.ollamaModelId || "nomic-embed-text:latest" + this.rateLimitScope = makeRateLimitScope("ollama", baseUrl) } /** @@ -40,10 +39,9 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { * @param model - Optional model ID to override the default. * @returns A promise that resolves to an EmbeddingResponse containing the embeddings and usage data. */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { const modelToUse = model || this.defaultModelId - const url = `${this.baseUrl}/api/embed` + const url = `${this.baseUrl}/api/embed` // Endpoint as specified // Apply model-specific query prefix if required const queryPrefix = getModelQueryPrefix("ollama", modelToUse) @@ -56,7 +54,7 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -70,11 +68,14 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { }) : texts - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) - const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal - try { + // Note: Standard Ollama API uses 'prompt' for single text, not 'input' for array. + // Implementing based on user's specific request structure. + + // Add timeout to prevent indefinite hanging + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const response = await fetch(url, { method: "POST", headers: { @@ -82,11 +83,11 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { }, body: JSON.stringify({ model: modelToUse, - input: processedTexts, + input: processedTexts, // Using 'input' as requested }), - signal: requestSignal, + signal: controller.signal, }) - signal?.throwIfAborted() + clearTimeout(timeoutId) if (!response.ok) { let errorBody = t("embeddings:ollama.couldNotReadErrorBody") @@ -95,19 +96,18 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { } catch { // Ignore error reading body } - throw new EmbeddingError( - t("embeddings:ollama.requestFailed", { - status: response.status, - statusText: response.statusText, - errorBody, - }), - response, - ) + // Keep status and headers so a 5xx or 429 is retried or paced instead of failing at once. + const message = t("embeddings:ollama.requestFailed", { + status: response.status, + statusText: response.statusText, + errorBody, + }) + throw new EmbeddingError(message, { status: response.status, headers: response.headers }) } const data = await response.json() - signal?.throwIfAborted() + // Extract embeddings using 'embeddings' key as requested const embeddings = data.embeddings if (!embeddings || !Array.isArray(embeddings)) { throw new Error(t("embeddings:ollama.invalidResponseStructure")) @@ -117,29 +117,28 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { embeddings: embeddings, } } catch (error: any) { - signal?.throwIfAborted() + if (error instanceof EmbeddingError) throw error + // Capture telemetry before reformatting the error TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)), stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined, location: "OllamaEmbedder:createEmbeddings", }) - codeIndexLog.error("Ollama embedding failed:", error) + // Log the original error for debugging purposes + console.error("Ollama embedding failed:", error) + + // Handle specific error types with better messages if (error.name === "AbortError") { - throw new EmbeddingError(t("embeddings:validation.connectionFailed"), error, { - classification: controller.signal.aborted ? "connection" : "permanent", - }) + throw new Error(t("embeddings:validation.connectionFailed")) } else if (error.message?.includes("fetch failed") || error.code === "ECONNREFUSED") { - throw new EmbeddingError(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl }), error) + throw new Error(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl })) } else if (error.code === "ENOTFOUND") { - throw new EmbeddingError(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl }), error) + throw new Error(t("embeddings:ollama.hostNotFound", { baseUrl: this.baseUrl })) } - throw new EmbeddingError(t("embeddings:ollama.embeddingFailed", { message: error.message }), error, { - retryAfterMs: error instanceof EmbeddingError ? error.retryAfterMs : undefined, - }) - } finally { - clearTimeout(timeoutId) + // Re-throw a more specific error for the caller + throw new Error(t("embeddings:ollama.embeddingFailed", { message: error.message })) } } diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 341ec28357..2125992904 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -1,17 +1,17 @@ -import { codeIndexLog } from "../code-index-log" -import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId, getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" -import { withValidationErrorHandling, HttpError, formatEmbeddingError } from "../shared/validation-helpers" +import { withValidationErrorHandling, HttpError, toEmbeddingError } from "../shared/validation-helpers" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" +import { makeRateLimitScope } from "../provider-pacer" interface EmbeddingItem { embedding: string | number[] + [key: string]: any } interface OpenAIEmbeddingResponse { @@ -23,18 +23,18 @@ interface OpenAIEmbeddingResponse { } /** - * OpenAI Compatible implementation of the embedder interface with batching. + * OpenAI Compatible implementation of the embedder interface with batching and rate limiting. * This embedder allows using any OpenAI-compatible API endpoint by specifying a custom baseURL. */ export class OpenAICompatibleEmbedder implements IEmbedder { - readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string private readonly baseUrl: string private readonly apiKey: string private readonly isFullUrl: boolean private readonly maxItemTokens: number + readonly rateLimitScope: string /** * Creates a new OpenAI Compatible embedder @@ -53,14 +53,14 @@ export class OpenAICompatibleEmbedder implements IEmbedder { this.baseUrl = baseUrl this.apiKey = apiKey - this.rateLimitScope = makeRateLimitScope("openai-compatible", baseUrl, apiKey) // Wrap OpenAI client creation to handle invalid API key characters try { + // Retries are owned by the indexing retry policy; SDK retries would multiply its allowance. this.embeddingsClient = new OpenAI({ baseURL: baseUrl, - maxRetries: 0, apiKey: apiKey, + maxRetries: 0, }) } catch (error) { // Use the error handler to transform ByteString conversion errors @@ -71,16 +71,16 @@ export class OpenAICompatibleEmbedder implements IEmbedder { // Cache the URL type check for performance this.isFullUrl = this.isFullEndpointUrl(baseUrl) this.maxItemTokens = maxItemTokens || MAX_ITEM_TOKENS + this.rateLimitScope = makeRateLimitScope("openai-compatible", baseUrl, apiKey) } /** - * Creates embeddings for the given texts with batching + * Creates embeddings for the given texts with batching and rate limiting * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -94,7 +94,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -113,7 +113,6 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { - signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -123,7 +122,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > this.maxItemTokens) { - codeIndexLog.warn( + console.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -149,7 +148,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) + const batchResult = await this._embedBatch(currentBatch, modelToUse) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -193,11 +192,9 @@ export class OpenAICompatibleEmbedder implements IEmbedder { url: string, batchTexts: string[], model: string, - signal?: AbortSignal, ): Promise { const response = await fetch(url, { method: "POST", - ...(signal ? { signal } : {}), headers: { "Content-Type": "application/json", // Azure OpenAI uses 'api-key' header, while OpenAI uses 'Authorization' @@ -225,10 +222,11 @@ export class OpenAICompatibleEmbedder implements IEmbedder { // Ignore text parsing errors errorText = `Error ${status}` } - const error = new Error(`HTTP ${status}: ${errorText}`) as HttpError - error.status = status || response?.status || 0 - error.headers = response?.headers - throw error + // Headers carry Retry-After for the shared rate-limit wait. + throw Object.assign(new Error(`HTTP ${status}: ${errorText}`), { + status: status || response?.status || 0, + headers: response?.headers, + }) } try { @@ -240,68 +238,45 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } } + /** + /** Sends one request; retries and rate-limit waits belong to the caller's retry policy and pacer. */ private async _embedBatch( batchTexts: string[], model: string, - signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { - // Use cached value for performance - const isFullUrl = this.isFullUrl - try { - signal?.throwIfAborted() - let response: OpenAIEmbeddingResponse - - if (isFullUrl) { - response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model, signal) - } else { - response = (await this.embeddingsClient.embeddings.create( - { + const response: OpenAIEmbeddingResponse = this.isFullUrl + ? await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model) + : ((await this.embeddingsClient.embeddings.create({ input: batchTexts, model: model, - // Base64 with local decoding bypasses the OpenAI v4.78.1 numeric-array parser's - // 256-dimension truncation, which breaks larger models. + // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 + // when processing numeric arrays, which breaks compatibility with models using larger dimensions. + // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. encoding_format: "base64", - }, - ...(signal ? [{ signal }] : []), - )) as OpenAIEmbeddingResponse - } - - signal?.throwIfAborted() - const processedEmbeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding === "string") { - const buffer = Buffer.from(item.embedding, "base64") + })) as OpenAIEmbeddingResponse) - const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) - - return { - ...item, - embedding: Array.from(float32Array), - } - } - return item + const embeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding !== "string") return item.embedding + const buffer = Buffer.from(item.embedding, "base64") + return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)) }) - response.data = processedEmbeddings - - const embeddings = response.data.map((item) => item.embedding as number[]) - return { - embeddings: embeddings, + embeddings, usage: { promptTokens: response.usage?.prompt_tokens || 0, totalTokens: response.usage?.total_tokens || 0, }, } } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, location: "OpenAICompatibleEmbedder:_embedBatch", }) - codeIndexLog.error("OpenAI Compatible embedder error:", error) - throw formatEmbeddingError(error) + console.error("OpenAI Compatible embedder error:", error) + throw toEmbeddingError(error) } } @@ -359,4 +334,5 @@ export class OpenAICompatibleEmbedder implements IEmbedder { name: "openai-compatible", } } + } diff --git a/src/services/code-index/embedders/openai.ts b/src/services/code-index/embedders/openai.ts index 5119873c00..0138516def 100644 --- a/src/services/code-index/embedders/openai.ts +++ b/src/services/code-index/embedders/openai.ts @@ -1,5 +1,3 @@ -import { codeIndexLog } from "../code-index-log" -import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { OpenAiNativeHandler } from "../../../api/providers/openai-native" import { ApiHandlerOptions } from "../../../shared/api" @@ -7,18 +5,19 @@ import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces" import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" -import { withValidationErrorHandling, formatEmbeddingError } from "../shared/validation-helpers" +import { withValidationErrorHandling, toEmbeddingError } from "../shared/validation-helpers" +import { makeRateLimitScope } from "../provider-pacer" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" /** - * OpenAI implementation of the embedder interface with batching + * OpenAI implementation of the embedder interface with batching and rate limiting */ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { - readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string + readonly rateLimitScope: string /** * Creates a new OpenAI embedder @@ -27,10 +26,10 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { constructor(options: ApiHandlerOptions & { openAiEmbeddingModelId?: string }) { super(options) const apiKey = this.options.openAiNativeApiKey ?? "not-provided" - this.rateLimitScope = makeRateLimitScope("openai", "https://api.openai.com/v1", apiKey) // Wrap OpenAI client creation to handle invalid API key characters try { + // Retries are owned by the indexing retry policy; SDK retries would multiply its allowance. this.embeddingsClient = new OpenAI({ apiKey, maxRetries: 0 }) } catch (error) { // Use the error handler to transform ByteString conversion errors @@ -38,16 +37,16 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { } this.defaultModelId = options.openAiEmbeddingModelId || "text-embedding-3-small" + this.rateLimitScope = makeRateLimitScope("openai", "https://api.openai.com/v1", apiKey) } /** - * Creates embeddings for the given texts with batching + * Creates embeddings for the given texts with batching and rate limiting * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -61,7 +60,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -80,7 +79,6 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { - signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -90,7 +88,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -116,7 +114,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) + const batchResult = await this._embedBatch(currentBatch, modelToUse) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -126,22 +124,17 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { return { embeddings: allEmbeddings, usage } } + /** Sends one request; retries and rate-limit waits belong to the caller's retry policy and pacer. */ private async _embedBatch( batchTexts: string[], model: string, - signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { try { - signal?.throwIfAborted() - const response = await this.embeddingsClient.embeddings.create( - { - input: batchTexts, - model: model, - }, - signal ? { signal } : undefined, - ) + const response = await this.embeddingsClient.embeddings.create({ + input: batchTexts, + model: model, + }) - signal?.throwIfAborted() return { embeddings: response.data.map((item) => item.embedding), usage: { @@ -150,14 +143,13 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { }, } } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, location: "OpenAiEmbedder:_embedBatch", }) - codeIndexLog.error("OpenAI embedder error:", error) - throw formatEmbeddingError(error) + console.error("OpenAI embedder error:", error) + throw toEmbeddingError(error) } } diff --git a/src/services/code-index/embedders/openrouter.ts b/src/services/code-index/embedders/openrouter.ts index a2b7d0f0f8..e82a3cb737 100644 --- a/src/services/code-index/embedders/openrouter.ts +++ b/src/services/code-index/embedders/openrouter.ts @@ -1,20 +1,20 @@ -import { codeIndexLog } from "../code-index-log" -import { makeRateLimitScope } from "../provider-pacer" import { OpenAI } from "openai" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_BATCH_TOKENS, MAX_ITEM_TOKENS } from "../constants" import { getDefaultModelId, getModelQueryPrefix } from "../../../shared/embeddingModels" import { t } from "../../../i18n" -import { withValidationErrorHandling, formatEmbeddingError } from "../shared/validation-helpers" +import { withValidationErrorHandling, toEmbeddingError } from "../shared/validation-helpers" import { TelemetryEventName } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { handleOpenAIError } from "../../../api/providers/utils/error-handler" +import { makeRateLimitScope } from "../provider-pacer" // Default provider name when no specific provider is selected export const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]" interface EmbeddingItem { embedding: string | number[] + [key: string]: any } interface OpenRouterEmbeddingResponse { @@ -26,17 +26,17 @@ interface OpenRouterEmbeddingResponse { } /** - * OpenRouter implementation of the embedder interface with batching. + * OpenRouter implementation of the embedder interface with batching and rate limiting. * OpenRouter provides an OpenAI-compatible API that gives access to hundreds of models * through a single endpoint, automatically handling fallbacks and cost optimization. */ export class OpenRouterEmbedder implements IEmbedder { - readonly rateLimitScope: string private embeddingsClient: OpenAI private readonly defaultModelId: string private readonly maxItemTokens: number private readonly baseUrl: string = "https://openrouter.ai/api/v1" private readonly specificProvider?: string + readonly rateLimitScope: string /** * Creates a new OpenRouter embedder @@ -50,8 +50,6 @@ export class OpenRouterEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } - this.rateLimitScope = makeRateLimitScope("openrouter", this.baseUrl, apiKey) - // Only set specificProvider if it's not the default value this.specificProvider = specificProvider && specificProvider !== OPENROUTER_DEFAULT_PROVIDER_NAME ? specificProvider : undefined @@ -60,8 +58,9 @@ export class OpenRouterEmbedder implements IEmbedder { try { this.embeddingsClient = new OpenAI({ baseURL: this.baseUrl, - maxRetries: 0, apiKey: apiKey, + // Retries are owned by the indexing retry policy; SDK retries would multiply its allowance. + maxRetries: 0, defaultHeaders: { "HTTP-Referer": "https://github.com/Zoo-Code-Org/Zoo-Code", "X-Title": "Zoo Code", @@ -74,16 +73,16 @@ export class OpenRouterEmbedder implements IEmbedder { this.defaultModelId = modelId || getDefaultModelId("openrouter") this.maxItemTokens = maxItemTokens || MAX_ITEM_TOKENS + this.rateLimitScope = makeRateLimitScope("openrouter", this.baseUrl, apiKey) } /** - * Creates embeddings for the given texts with batching + * Creates embeddings for the given texts with batching and rate limiting * @param texts Array of text strings to embed * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -97,7 +96,7 @@ export class OpenRouterEmbedder implements IEmbedder { const prefixedText = `${queryPrefix}${text}` const estimatedTokens = Math.ceil(prefixedText.length / 4) if (estimatedTokens > MAX_ITEM_TOKENS) { - codeIndexLog.warn( + console.warn( t("embeddings:textWithPrefixExceedsTokenLimit", { index, estimatedTokens, @@ -116,7 +115,6 @@ export class OpenRouterEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { - signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -126,7 +124,7 @@ export class OpenRouterEmbedder implements IEmbedder { const itemTokens = Math.ceil(text.length / 4) if (itemTokens > this.maxItemTokens) { - codeIndexLog.warn( + console.warn( t("embeddings:textExceedsTokenLimit", { index: i, itemTokens, @@ -152,7 +150,7 @@ export class OpenRouterEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatch(currentBatch, modelToUse, signal) + const batchResult = await this._embedBatch(currentBatch, modelToUse) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -162,23 +160,20 @@ export class OpenRouterEmbedder implements IEmbedder { return { embeddings: allEmbeddings, usage } } + /** Sends one request; retries and rate-limit waits belong to the caller's retry policy and pacer. */ private async _embedBatch( batchTexts: string[], model: string, - signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { try { - signal?.throwIfAborted() - const requestParams: OpenAI.EmbeddingCreateParams & { - provider?: { order: string[]; only: string[]; allow_fallbacks: boolean } - } = { + const requestParams: OpenAI.EmbeddingCreateParams & { provider?: unknown } = { input: batchTexts, model: model, - // Base64 with local decoding bypasses the OpenAI v4.78.1 numeric-array parser's - // 256-dimension truncation, which breaks larger models. + // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 + // when processing numeric arrays, which breaks compatibility with models using larger dimensions. + // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. encoding_format: "base64", } - if (this.specificProvider) { requestParams.provider = { order: [this.specificProvider], @@ -189,47 +184,33 @@ export class OpenRouterEmbedder implements IEmbedder { const response = (await this.embeddingsClient.embeddings.create( requestParams, - ...(signal ? [{ signal }] : []), - )) as OpenRouterEmbeddingResponse + )) as unknown as OpenRouterEmbeddingResponse - signal?.throwIfAborted() - const processedEmbeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding === "string") { - const buffer = Buffer.from(item.embedding, "base64") - - const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) - - return { - ...item, - embedding: Array.from(float32Array), - } - } - return item + const embeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding !== "string") return item.embedding + const buffer = Buffer.from(item.embedding, "base64") + return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)) }) - response.data = processedEmbeddings - - const embeddings = response.data.map((item) => item.embedding as number[]) - return { - embeddings: embeddings, + embeddings, usage: { promptTokens: response.usage?.prompt_tokens || 0, totalTokens: response.usage?.total_tokens || 0, }, } } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, location: "OpenRouterEmbedder:_embedBatch", }) - codeIndexLog.error("OpenRouter embedder error:", error) - throw formatEmbeddingError(error) + console.error("OpenRouter embedder error:", error) + throw toEmbeddingError(error) } } + /** * Validates the OpenRouter embedder configuration by testing API connectivity * @returns Promise resolving to validation result with success status and optional error message @@ -242,9 +223,7 @@ export class OpenRouterEmbedder implements IEmbedder { const modelToUse = this.defaultModelId // Build the request parameters - const requestParams: OpenAI.EmbeddingCreateParams & { - provider?: { order: string[]; only: string[]; allow_fallbacks: boolean } - } = { + const requestParams: any = { input: testTexts, model: modelToUse, encoding_format: "base64", @@ -292,4 +271,5 @@ export class OpenRouterEmbedder implements IEmbedder { name: "openrouter", } } + } diff --git a/src/services/code-index/embedders/vercel-ai-gateway.ts b/src/services/code-index/embedders/vercel-ai-gateway.ts index f2314d6c37..715b7c2244 100644 --- a/src/services/code-index/embedders/vercel-ai-gateway.ts +++ b/src/services/code-index/embedders/vercel-ai-gateway.ts @@ -1,5 +1,4 @@ import { OpenAICompatibleEmbedder } from "./openai-compatible" -import { makeRateLimitScope } from "../provider-pacer" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_ITEM_TOKENS } from "../constants" import { t } from "../../../i18n" @@ -23,13 +22,15 @@ import { TelemetryService } from "@roo-code/telemetry" * - mistral/mistral-embed (dimension: 1024) */ export class VercelAiGatewayEmbedder implements IEmbedder { - readonly rateLimitScope: string - private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder private static readonly VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1" private static readonly DEFAULT_MODEL = "openai/text-embedding-3-large" private readonly modelId: string + get rateLimitScope(): string { + return this.openAICompatibleEmbedder.rateLimitScope + } + /** * Creates a new Vercel AI Gateway embedder * @param apiKey The Vercel AI Gateway API key for authentication @@ -40,12 +41,6 @@ export class VercelAiGatewayEmbedder implements IEmbedder { throw new Error(t("embeddings:validation.apiKeyRequired")) } - this.rateLimitScope = makeRateLimitScope( - "vercel-ai-gateway", - VercelAiGatewayEmbedder.VERCEL_AI_GATEWAY_BASE_URL, - apiKey, - ) - // Use provided model or default this.modelId = modelId || VercelAiGatewayEmbedder.DEFAULT_MODEL @@ -64,14 +59,12 @@ export class VercelAiGatewayEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { - signal?.throwIfAborted() + async createEmbeddings(texts: string[], model?: string): Promise { try { // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) + return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) } catch (error) { - signal?.throwIfAborted() TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, diff --git a/src/services/code-index/interfaces/embedder-factory.ts b/src/services/code-index/interfaces/embedder-factory.ts index 013b3f372b..36aeca9501 100644 --- a/src/services/code-index/interfaces/embedder-factory.ts +++ b/src/services/code-index/interfaces/embedder-factory.ts @@ -1,7 +1,8 @@ import type { CodeIndexConfig } from "./config" import type { IEmbedder } from "./index" +import type { ProviderPacerRegistry } from "../provider-pacer" /** Creates a vector embedder, validating the provider's required configuration. */ export interface IEmbedderFactory { - create(config: CodeIndexConfig): IEmbedder + create(config: CodeIndexConfig, pacers?: ProviderPacerRegistry): IEmbedder } diff --git a/src/services/code-index/interfaces/embedder.ts b/src/services/code-index/interfaces/embedder.ts index f6a2f79a91..38c227db19 100644 --- a/src/services/code-index/interfaces/embedder.ts +++ b/src/services/code-index/interfaces/embedder.ts @@ -3,15 +3,13 @@ * This interface is implemented by both OpenAI and Ollama embedders. */ export interface IEmbedder { - readonly rateLimitScope: string - /** * Creates embeddings for the given texts. * @param texts Array of text strings to create embeddings for * @param model Optional model ID to use for embeddings * @returns Promise resolving to an EmbeddingResponse */ - createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise + createEmbeddings(texts: string[], model?: string): Promise /** * Validates the embedder configuration by testing connectivity and credentials. @@ -20,6 +18,9 @@ export interface IEmbedder { validateConfiguration(): Promise<{ valid: boolean; error?: string }> get embedderInfo(): EmbedderInfo + + /** Requests with the same scope share one provider allowance and its rate-limit wait. */ + readonly rateLimitScope: string } export interface EmbeddingResponse { diff --git a/src/services/code-index/interfaces/vector-store-factory.ts b/src/services/code-index/interfaces/vector-store-factory.ts index 4a4361ef35..57fde591f6 100644 --- a/src/services/code-index/interfaces/vector-store-factory.ts +++ b/src/services/code-index/interfaces/vector-store-factory.ts @@ -1,6 +1,7 @@ import type { CodeIndexConfig } from "./config" import type { IVectorStore } from "./index" +import type { ProviderPacerRegistry } from "../provider-pacer" export interface IVectorStoreFactory { - create(config: CodeIndexConfig, workspacePath: string): IVectorStore + create(config: CodeIndexConfig, workspacePath: string, pacers?: ProviderPacerRegistry): IVectorStore } diff --git a/src/services/code-index/interfaces/vector-store.ts b/src/services/code-index/interfaces/vector-store.ts index 254476d4fc..22d5f4307a 100644 --- a/src/services/code-index/interfaces/vector-store.ts +++ b/src/services/code-index/interfaces/vector-store.ts @@ -8,13 +8,15 @@ export type PointStruct = { } export interface IVectorStore { - readonly rateLimitScope: string - /** * Initializes the vector store + * @param signal Ends retry and rate-limit waits during collection and index creation * @returns Promise resolving to boolean indicating if a new collection was created */ - initialize(): Promise + initialize(signal?: AbortSignal): Promise + + /** Requests with the same scope share one service allowance and its rate-limit wait. */ + readonly rateLimitScope: string /** * Upserts points into the vector store @@ -38,17 +40,16 @@ export interface IVectorStore { ): Promise /** - * Deletes legacy points and generations below the mandatory cutoff for this exact file. + * Deletes points by file path + * @param filePath Path of the file to delete points for */ - deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise + deletePointsByFilePath(filePath: string): Promise /** - * Applies the same generation fence to every exact file path. + * Deletes points by multiple file paths + * @param filePaths Array of file paths to delete points for */ - deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise - - /** Other includes legacy points without a generation. Both counts are exact. */ - countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> + deletePointsByMultipleFilePaths(filePaths: string[]): Promise /** * Clears all points from the collection diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index ac6bdaaf8b..789ec8c9b9 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -367,6 +367,8 @@ export class CodeIndexManager { reason: failure.reason, state: failure.state, })) + const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles[0] + const failedFileSummary = representative && { ...representative, count: failedFiles.length } const providerWaits = this.pacers.providerWaits() let systemStatus: IndexingState = status.systemStatus let message = status.message @@ -386,6 +388,7 @@ export class CodeIndexManager { systemStatus, message, failedFiles, + failedFileSummary, providerWaits, workspacePath: this.workspacePath, workspaceEnabled: this.isWorkspaceEnabled, diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts index ece1da5403..1706100a3c 100644 --- a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -11,7 +11,7 @@ import { SOURCE_MISSING, isQuotaError, } from "../indexing-errors" -import { formatEmbeddingError } from "../validation-helpers" +import { toEmbeddingError } from "../validation-helpers" import { IndexingRetryError } from "../indexing-errors" describe("quota 429 classification", () => { @@ -136,7 +136,7 @@ describe("indexing error classification", () => { it("classifies raw errors before translated formatting", () => { const cause = new APIConnectionTimeoutError() - const error = formatEmbeddingError(cause) + const error = toEmbeddingError(cause) expect(error).toBeInstanceOf(EmbeddingError) expect(error.message).toBe("embeddings:failedWithError") expect(error.cause).toBe(cause) @@ -180,7 +180,7 @@ it.each(["2", "invalid", undefined])( (value) => { const get = vi.fn((name: string) => (name === "retry-after" ? value : undefined)) const original = { status: 429, headers: { get } } - const formatted = formatEmbeddingError(original) + const formatted = toEmbeddingError(original) const retried = new IndexingRetryError(formatted, 3, "rate-limit", "embed") expect(retried.retryAfterMs).toBe(value === "2" ? 2000 : undefined) expect(formatted.retryAfterMs).toBe(retried.retryAfterMs) diff --git a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts index 5263d1299c..bf6c732a92 100644 --- a/src/services/code-index/shared/__tests__/validation-helpers.spec.ts +++ b/src/services/code-index/shared/__tests__/validation-helpers.spec.ts @@ -1,58 +1,4 @@ -import * as fs from "fs" -import * as path from "path" -import { formatEmbeddingError, sanitizeErrorMessage } from "../validation-helpers" -import { EmbeddingError } from "../indexing-errors" -import { t } from "../../../../i18n" - -vi.mock("../../../../i18n", () => ({ t: vi.fn((key: string) => key) })) - -describe("formatEmbeddingError", () => { - it.each([ - [401, "permanent", "embeddings:authenticationFailed"], - [429, "rate-limit", "embeddings:failedWithStatus"], - [503, "server", "embeddings:failedWithStatus"], - ] as const)("keeps classification and delay for HTTP %s as a single attempt", (status, classification, key) => { - const cause = Object.assign(new Error("failure"), { status, headers: new Headers({ "retry-after": "2" }) }) - const error = formatEmbeddingError(cause) - expect(error).toBeInstanceOf(EmbeddingError) - expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: 2000, cause }) - expect(error.message).toBe(key) - if (status !== 401) { - expect(t).toHaveBeenLastCalledWith(key, { attempts: 1, statusCode: status, errorMessage: "failure" }) - } - }) - - it("preserves connection errors with no headers", () => { - const cause = new TypeError("fetch failed", { cause: new Error("network") }) - const error = formatEmbeddingError(cause) - expect(error).toMatchObject({ classification: "connection", cause }) - expect(error.retryAfterMs).toBeUndefined() - expect(t).toHaveBeenLastCalledWith("embeddings:failedWithError", { attempts: 1, errorMessage: "fetch failed" }) - }) - - it("supplies every placeholder used by the failure messages in all locales", () => { - formatEmbeddingError(Object.assign(new Error("failure"), { status: 503 })) - const statusArgs = vi.mocked(t).mock.lastCall?.[1] ?? {} - formatEmbeddingError(new TypeError("fetch failed")) - const errorArgs = vi.mocked(t).mock.lastCall?.[1] ?? {} - const localesRoot = path.resolve(__dirname, "../../../../i18n/locales") - const locales = fs.readdirSync(localesRoot) - expect(locales).toHaveLength(18) - for (const locale of locales) { - const messages: Record = JSON.parse( - fs.readFileSync(path.join(localesRoot, locale, "embeddings.json"), "utf8"), - ) - for (const [key, args] of [ - ["failedWithStatus", statusArgs], - ["failedWithError", errorArgs], - ] as const) { - const placeholders = [...String(messages[key]).matchAll(/\{\{(\w+)\}\}/g)].map((match) => match[1]) - expect(placeholders.length, `${locale} ${key}`).toBeGreaterThan(0) - for (const placeholder of placeholders) expect(args, `${locale} ${key}`).toHaveProperty(placeholder) - } - } - }) -}) +import { sanitizeErrorMessage } from "../validation-helpers" describe("sanitizeErrorMessage", () => { it("should sanitize Unix-style file paths", () => { @@ -117,15 +63,11 @@ describe("sanitizeErrorMessage", () => { expect(sanitizeErrorMessage(input)).toBe(expected) }) - it.each([ - [null, "null"], - [undefined, "undefined"], - [123, "123"], - [{}, "[object Object]"], - ])("handles runtime non-string input %o", (input, expected) => { - // Exercise untyped callers without weakening the production string contract. - const sanitizeRuntimeInput = sanitizeErrorMessage as unknown as (value: unknown) => string - expect(sanitizeRuntimeInput(input)).toBe(expected) + it("should handle non-string inputs gracefully", () => { + expect(sanitizeErrorMessage(null as any)).toBe("null") + expect(sanitizeErrorMessage(undefined as any)).toBe("undefined") + expect(sanitizeErrorMessage(123 as any)).toBe("123") + expect(sanitizeErrorMessage({} as any)).toBe("[object Object]") }) it("should preserve non-sensitive error messages", () => { @@ -148,34 +90,3 @@ describe("sanitizeErrorMessage", () => { expect(sanitizeErrorMessage(input)).toBe(expected) }) }) - -describe("validation rate limits", () => { - it("rethrows the original 429 before custom handling", async () => { - const { withValidationErrorHandling } = await import("../validation-helpers") - const error = new EmbeddingError("limited", { status: 429 }, { retryAfterMs: 61_000 }) - const handler = vi.fn() - await expect( - withValidationErrorHandling( - async () => { - throw error - }, - "openai", - { - beforeStandardHandling: handler, - }, - ), - ).rejects.toBe(error) - expect(handler).not.toHaveBeenCalled() - }) - it("still accepts valid configuration and converts permanent failure", async () => { - const { withValidationErrorHandling } = await import("../validation-helpers") - await expect(withValidationErrorHandling(async () => ({ valid: true }), "openai")).resolves.toEqual({ - valid: true, - }) - await expect( - withValidationErrorHandling(async () => { - throw new Error("bad key") - }, "openai"), - ).resolves.toEqual({ valid: false, error: "bad key" }) - }) -}) diff --git a/src/services/code-index/shared/validation-helpers.ts b/src/services/code-index/shared/validation-helpers.ts index 392c56940e..12614f8365 100644 --- a/src/services/code-index/shared/validation-helpers.ts +++ b/src/services/code-index/shared/validation-helpers.ts @@ -1,6 +1,7 @@ import { t } from "../../../i18n" import { serializeError } from "serialize-error" -import { classifyIndexingError, EmbeddingError, getIndexingErrorStatusCode } from "./indexing-errors" +import { EmbeddingError, isQuotaError } from "./indexing-errors" +import { FILE_RETRY_ATTEMPTS } from "../constants" /** * Sanitizes error messages by removing sensitive information like file paths and URLs @@ -49,7 +50,6 @@ export function sanitizeErrorMessage(errorMessage: string): string { */ export interface HttpError extends Error { status?: number - headers?: Headers response?: { status?: number } @@ -200,7 +200,6 @@ export async function withValidationErrorHandling ({ t: (key: string) => key })) -const now = Date.parse("2026-11-11T12:00:00Z") -const operations = ["upsert", "delete", "verify"] as const -const matchesOperation = (url: string, operation: (typeof operations)[number]) => - operation === "verify" - ? url.includes("/count") - : operation === "delete" - ? url.includes("/delete") - : /\/points\?/.test(url) - -beforeEach(() => { - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }) - vi.setSystemTime(now) - vi.spyOn(Math, "random").mockReturnValue(0) - nock.enableNetConnect("127.0.0.1") -}) -afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() - nock.disableNetConnect() -}) - -const scenarios = [ - ...operations.flatMap((activity) => - [7, 60, 61, 120].map((seconds) => ({ - activity, - scenario: `numeric ${seconds} seconds, four limits`, - header: String(seconds) as string | undefined, - delay: seconds * 1000, - limits: 4, - })), - ), - { - activity: "upsert", - scenario: "HTTP date 7 seconds ahead", - header: new Date(now + 7000).toUTCString(), - delay: 7000, - limits: 1, - }, - { - activity: "upsert", - scenario: "HTTP date 120 seconds ahead", - header: new Date(now + 120_000).toUTCString(), - delay: 120_000, - limits: 1, - }, - { - activity: "upsert", - scenario: "past date", - header: new Date(now - 7000).toUTCString(), - delay: undefined, - limits: 1, - }, - { activity: "upsert", scenario: "numeric 61 seconds", header: "61", delay: 61_000, limits: 1 }, - { activity: "upsert", scenario: "missing header", header: undefined, delay: undefined, limits: 1 }, -] as const - -it.each(scenarios)("T26/T32 HTTP $activity $scenario", async ({ activity, header, delay, limits }) => { - let requests = 0 - let healthComplete!: () => void - const health = new Promise((resolve) => { - healthComplete = resolve - }) - const perAttempt = activity === "verify" ? 2 : 1 - const server = createServer((request, response) => { - response.setHeader("content-type", "application/json") - if (request.url === "/") { - response.end(JSON.stringify({ title: "qdrant", version: "1.15.0" }), healthComplete) - return - } - const matches = matchesOperation(request.url!, activity) - if (matches) requests++ - let body = "" - request.on("data", (chunk) => { - body += chunk - }) - request.on("end", () => { - if (matches && requests <= limits * perAttempt) { - response.writeHead(429, header === undefined ? {} : { "retry-after": header }) - response.end(JSON.stringify({ status: { error: "limited" }, time: 0 })) - } else { - const count = request.url!.includes("/count") - ? JSON.parse(body).filter.must.length === 2 - ? 1 - : 0 - : undefined - response.end( - JSON.stringify({ - result: count === undefined ? { status: "completed", operation_id: 1 } : { count }, - status: "ok", - time: 0, - }), - ) - } - }) - }) - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) - const endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` - const store = new QdrantVectorStore(path.resolve("/workspace"), endpoint, 2) - await health - const method = - activity === "verify" ? "countPointsForFile" : activity === "delete" ? "deletePointsByFilePath" : "upsertPoints" - const operation = vi.spyOn(store, method) - const { replace, failures, cache, pacers, gate } = pacedTransportWriter(store) - const grace = vi.spyOn(gate, "waitForGrace") - const controller = new AbortController() - const embedding = { - rateLimitScope: "embedder:test", - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[1, 0]] }), - validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), - embedderInfo: { name: "openai" }, - } satisfies IEmbedder - const pending = replace(embedding, controller.signal).catch((error: unknown) => error) - try { - for (const index of Array.from({ length: limits }, (_, index) => index)) { - await waitForTransport(() => { - expect(operation).toHaveBeenCalledTimes(index + 1) - expect(pacers.get(store.rateLimitScope).state().pausedUntil).toBe(Date.now() + (delay ?? 500)) - expect(pacers.get(store.rateLimitScope).state().waitingPaths).toEqual(["test.ts"]) - }) - await waitForTransport(() => expect(requests).toBe((index + 1) * perAttempt)) - const error = (await operation.mock.results[index].value.catch( - (error: unknown) => error, - )) as VectorStoreError - expect(error).toBeInstanceOf(VectorStoreError) - expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429 }) - if (index === 0) { - expect(error.retryAfterMs).toBe(delay) - if (header !== undefined && /^\d+$/.test(header)) { - expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) - expect(error.cause).toMatchObject({ retry_after: Number(header) }) - } else { - const rawError = error.cause as Error & { status: number; headers: Headers } - expect(rawError).toMatchObject({ name: "Error", status: 429 }) - expect(rawError.headers.get("retry-after")).toBe(header ?? null) - } - } - expect(failures.list()).toEqual([]) - expect(cache.completeFile).not.toHaveBeenCalled() - vi.advanceTimersByTime((delay ?? 500) - 1) - expect(requests).toBe((index + 1) * perAttempt) - vi.advanceTimersByTime(1) - } - await waitForTransport(() => expect(requests).toBe((limits + 1) * perAttempt)) - await operation.mock.results[limits].value - // Successful probes stage subsequent cleanup and verification requests. - await waitForTransport(() => { - if (cache.completeFile.mock.calls.length + grace.mock.calls.length === 0) { - vi.advanceTimersByTime(125) - } - expect(cache.completeFile.mock.calls.length + grace.mock.calls.length).toBe(1) - }) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - await waitForTransport(() => expect(cache.completeFile).toHaveBeenCalledOnce()) - await expect(pending).resolves.toBeUndefined() - expect(requests).toBe((limits + 1) * perAttempt) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() - expect(embedding.createEmbeddings).toHaveBeenCalledOnce() - } finally { - controller.abort() - await pending - server.closeAllConnections() - await new Promise((resolve) => server.close(() => resolve())) - } -}) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts index 1499b7d133..2208a6103d 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts @@ -1,150 +1,65 @@ -import "../../__tests__/mock-code-index-log" import path from "node:path" import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" import { QdrantVectorStore } from "../qdrant-client" -import { pacedTransportWriter } from "../../embedders/__tests__/paced-transport-writer" import { VectorStoreError } from "../../shared/indexing-errors" -import type { IEmbedder } from "../../interfaces" -import { LATE_WRITE_GRACE_MS } from "../../index-write-gate" vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) -const now = Date.parse("2026-11-11T12:00:00Z") -beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(now) - vi.spyOn(Math, "random").mockReturnValue(0) -}) + afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() vi.unstubAllGlobals() + vi.restoreAllMocks() }) -const operations = ["upsert", "delete", "verify"] as const -const matchesOperation = (url: string, operation: (typeof operations)[number]) => - operation === "verify" - ? url.includes("/count") - : operation === "delete" - ? url.includes("/delete") - : /\/points\?/.test(url) - -function installTransport(activity: (typeof operations)[number], header: string | undefined, limitedAttempts = 1) { - const requests = vi.fn() +function respondToUpsert(retryAfter: string | undefined) { vi.stubGlobal( "fetch", - vi.fn(async (input: RequestInfo | URL, options?: RequestInit) => { + vi.fn(async (input: RequestInfo | URL) => { const url = input instanceof Request ? input.url : String(input) - const matches = matchesOperation(url, activity) - if (matches) requests() - const limited = matches && requests.mock.calls.length <= limitedAttempts * (activity === "verify" ? 2 : 1) - const body = input instanceof Request ? await input.clone().text() : String(options?.body ?? "{}") - const count = url.includes("/count") ? (JSON.parse(body).filter.must.length === 2 ? 1 : 0) : undefined + const limited = /\/points\?/.test(url) return new Response( JSON.stringify( - limited - ? { status: { error: "limited" }, time: 0 } - : url.endsWith("/") - ? { title: "qdrant", version: "1.15.0" } - : { - result: count === undefined ? { status: "completed", operation_id: 1 } : { count }, - status: "ok", - time: 0, - }, + limited ? { status: { error: "limited" }, time: 0 } : { result: { status: "completed" }, status: "ok" }, ), { status: limited ? 429 : 200, headers: { "content-type": "application/json", - ...(header === undefined ? {} : { "retry-after": header }), + ...(retryAfter === undefined ? {} : { "retry-after": retryAfter }), }, }, ) }), ) - return requests } -const embedder = () => - ({ - rateLimitScope: "embedder:test", - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[1, 0]] }), - validateConfiguration: vi.fn().mockResolvedValue({ valid: true }), - embedderInfo: { name: "openai" }, - }) satisfies IEmbedder +const point = { id: "1", vector: [1, 0], payload: { filePath: "a.ts" } } -describe.each(operations)("T26 Qdrant %s pacing", (activity) => { - it.each([7, 60, 61, 120])( - "honors %s seconds across four limits and completes without failures", - async (seconds) => { - const requests = installTransport(activity, String(seconds), 4) - const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) - const { replace, failures, cache } = pacedTransportWriter(store) - const embedding = embedder() - const pending = replace(embedding) - await vi.advanceTimersByTimeAsync(0) - const perAttempt = activity === "verify" ? 2 : 1 - for (const index of [0, 1, 2, 3]) { - expect(requests).toHaveBeenCalledTimes((index + 1) * perAttempt) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(seconds * 1000 - 1) - expect(requests).toHaveBeenCalledTimes((index + 1) * perAttempt) - await vi.advanceTimersByTimeAsync(1) - } - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - await pending - expect(requests).toHaveBeenCalledTimes(5 * perAttempt) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() - expect(embedding.createEmbeddings).toHaveBeenCalledOnce() - }, - ) -}) +describe("Qdrant Retry-After (CI-QD-02)", () => { + it.each([ + ["61", 61_000], + ["120", 120_000], + ])("reads the whole %s-second header, not its first character", async (header, expected) => { + respondToUpsert(header) + const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const error = await store.upsertPoints([point]).catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(VectorStoreError) + expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429, retryAfterMs: expected }) + expect((error as VectorStoreError).cause).toBeInstanceOf(QdrantClientResourceExhaustedError) + }) -it.each([ - { scenario: "HTTP date 7 seconds ahead", header: new Date(now + 7000).toUTCString(), delay: 7000 }, - { scenario: "HTTP date 120 seconds ahead", header: new Date(now + 120_000).toUTCString(), delay: 120_000 }, - { scenario: "past HTTP date", header: new Date(now - 7000).toUTCString(), delay: undefined }, - { scenario: "numeric 61 seconds", header: "61", delay: 61_000 }, - { scenario: "missing header", header: undefined, delay: undefined }, -])("T32 $scenario preserves the raw error and waits without a cap", async ({ header, delay }) => { - const requests = installTransport("upsert", header) - const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) - const upsert = vi.spyOn(store, "upsertPoints") - const { replace, failures, cache } = pacedTransportWriter(store) - const pending = replace(embedder()) - await vi.advanceTimersByTimeAsync(0) - const result = upsert.mock.results[0] - const error = (await result.value.catch((error: unknown) => error)) as VectorStoreError - expect(error).toBeInstanceOf(VectorStoreError) - expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429 }) - expect(error.retryAfterMs).toBe(delay) - if (header === "61") { - expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) - expect(error.cause).toMatchObject({ retry_after: 61 }) - } else { - const rawError = error.cause as Error & { status: number; headers: Headers } - expect(rawError).toMatchObject({ name: "Error", status: 429 }) - expect(rawError.headers.get("retry-after")).toBe(header ?? null) - } - expect(requests).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([]) - await vi.advanceTimersByTimeAsync((delay ?? 500) - 1) - expect(requests).toHaveBeenCalledOnce() - await vi.advanceTimersByTimeAsync(1) - await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) - await pending - expect(requests).toHaveBeenCalledTimes(2) - expect(failures.list()).toEqual([]) - expect(cache.completeFile).toHaveBeenCalledOnce() -}) + it("classifies a 429 without a header as a rate limit with no delay", async () => { + respondToUpsert(undefined) + const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const error = await store.upsertPoints([point]).catch((caught: unknown) => caught) + expect(error).toMatchObject({ classification: "rate-limit" }) + expect((error as VectorStoreError).retryAfterMs).toBeUndefined() + }) -it("scopes Qdrant by normalized URL, independently of workspace and key", () => { - installTransport("upsert", undefined, 0) - const first = new QdrantVectorStore("/first", "http://EXAMPLE.test:6333/", 2, "first-key") - const same = new QdrantVectorStore("/second", "example.test:6333", 2, "second-key") - const other = new QdrantVectorStore("/first", "http://example.test:6334", 2) - expect(first.rateLimitScope).toBe(same.rateLimitScope) - expect(first.rateLimitScope).not.toBe(other.rateLimitScope) - expect(first.rateLimitScope).toMatch(/^qdrant:[a-f0-9]{64}$/) + it("scopes Qdrant by normalized URL, independently of workspace and key", () => { + const first = new QdrantVectorStore("/first", "http://EXAMPLE.test:6333/", 2, "first-key") + const same = new QdrantVectorStore("/second", "example.test:6333", 2, "second-key") + const other = new QdrantVectorStore("/first", "http://example.test:6334", 2) + expect(first.rateLimitScope).toBe(same.rateLimitScope) + expect(first.rateLimitScope).not.toBe(other.rateLimitScope) + }) }) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts deleted file mode 100644 index c3418ed20e..0000000000 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.retry-contract.spec.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { createServer, type Server } from "node:http" -import { createRequire } from "node:module" -import path from "node:path" -import nock from "nock" -import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" -import { QdrantVectorStore } from "../qdrant-client" -import { VectorStoreError } from "../../shared/indexing-errors" -import { retryWithPolicy, type RetryFailure } from "../../shared/retry-policy" -import { waitForRetry } from "../../embedders/cancellation" -import { clearAllMocks } from "../../../../test-utils/reset" - -vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) -vi.mock("../../code-index-log", () => ({ - codeIndexLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, -})) -vi.mock("../../embedders/cancellation", () => ({ waitForRetry: vi.fn().mockResolvedValue(undefined) })) - -let server: Server -let store: QdrantVectorStore -let responseStatus: number -let retryAfter: string | undefined -let requests: Array<{ method: string | undefined; url: string; body: unknown }> -const workspacePath = path.resolve("qdrant-contract-workspace") - -beforeEach(async () => { - clearAllMocks() - nock.enableNetConnect("127.0.0.1") - responseStatus = 200 - retryAfter = undefined - requests = [] - server = createServer((incoming, outgoing) => { - const chunks: Buffer[] = [] - incoming.on("data", (chunk: Buffer) => chunks.push(chunk)) - incoming.on("end", () => { - const body = Buffer.concat(chunks).toString() - const url = incoming.url ?? "" - if (url === "/") { - outgoing.writeHead(200, { "content-type": "application/json" }) - outgoing.end(JSON.stringify({ title: "qdrant", version: "1.20.0" })) - return - } - requests.push({ method: incoming.method, url, body: body ? JSON.parse(body) : undefined }) - outgoing.writeHead(responseStatus, { - "content-type": "application/json", - ...(retryAfter === undefined ? {} : { "retry-after": retryAfter }), - }) - const result = url.includes("/count") ? { count: 2 } : { status: "completed", operation_id: 1 } - outgoing.end( - JSON.stringify({ result, status: responseStatus === 200 ? "ok" : { error: "failure" }, time: 0 }), - ) - }) - }) - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) - const address = server.address() - if (!address || typeof address === "string") throw new Error("Expected a TCP test listener") - store = new QdrantVectorStore(workspacePath, `http://127.0.0.1:${address.port}`, 2) -}) - -afterEach(async () => { - vi.restoreAllMocks() - server.closeAllConnections() - await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) -}) - -function operation(activity: "upsert" | "delete" | "verify"): Promise { - if (activity === "upsert") { - return store.upsertPoints([ - { id: "00000000-0000-4000-8000-000000000001", vector: [1, 0], payload: { filePath: "file.ts" } }, - ]) - } - if (activity === "delete") return store.deletePointsByFilePath("file.ts", 2) - return store.countPointsForFile("file.ts", 2) -} - -it("runs the exact pinned package rather than a stale install", () => { - const require = createRequire(import.meta.url) - const installed: unknown = require( - path.resolve(path.dirname(require.resolve("@qdrant/js-client-rest")), "../../package.json"), - ) - expect(installed).toMatchObject({ version: "1.20.0" }) -}) - -it.each([ - ["upsert", 60], - ["delete", 60], - ["verify", 60], - ["upsert", 61], - ["delete", 61], - ["verify", 61], -] as const)("%s preserves the complete Retry-After %s header through real HTTP", async (activity, seconds) => { - responseStatus = 429 - retryAfter = String(seconds) - const failures: RetryFailure[] = [] - const budget = { retriesUsed: 2 } - const error: unknown = await retryWithPolicy(() => operation(activity), { - activity, - budget, - onFailure: (failure) => failures.push(failure), - }).catch((failure: unknown) => failure) - expect(waitForRetry).not.toHaveBeenCalled() - expect(failures).toEqual([]) - expect(budget.retriesUsed).toBe(2) - expect(error).toBeInstanceOf(VectorStoreError) - if (!(error instanceof VectorStoreError)) throw new Error("Expected a classified vector-store error") - expect(error).toMatchObject({ retryAfterMs: seconds * 1000, classification: "rate-limit", statusCode: 429 }) - expect(error.cause).toBeInstanceOf(QdrantClientResourceExhaustedError) - expect(error.cause).toMatchObject({ retry_after: seconds }) - // Verify issues two concurrent counts; the rejection can arrive before the second reaches the server. - await vi.waitFor(() => expect(requests).toHaveLength(activity === "verify" ? 2 : 1)) -}) - -it.each(["upsert", "delete", "verify"] as const)("%s is a single attempt without the policy", async (activity) => { - responseStatus = 503 - await expect(operation(activity)).rejects.toMatchObject({ classification: "server", statusCode: 503 }) - // Verify issues two concurrent counts; the rejection can arrive before the second reaches the server. - await vi.waitFor(() => expect(requests).toHaveLength(activity === "verify" ? 2 : 1)) - expect(waitForRetry).not.toHaveBeenCalled() -}) - -it.each([400, 401, 403, 408, 409])("HTTP %s is permanent and never retried", async (status) => { - responseStatus = status - await expect(retryWithPolicy(() => operation("upsert"))).rejects.toMatchObject({ - classification: "permanent", - attempts: 1, - statusCode: status, - }) - expect(requests).toHaveLength(1) - expect(waitForRetry).not.toHaveBeenCalled() -}) - -it("deletes exact file paths only below the generation cutoff", async () => { - await store.deletePointsByMultipleFilePaths([path.join(workspacePath, "file.ts"), "other.ts"], 2) - expect(requests).toHaveLength(1) - expect(requests[0].body).toEqual({ - filter: { - must: [ - { key: "filePath", match: { any: ["file.ts", "other.ts"] } }, - { should: [{ key: "generation", range: { lt: 2 } }, { is_empty: { key: "generation" } }] }, - ], - }, - }) - expect(requests[0].url).toContain("wait=true") -}) - -it("counts the exact current generation separately from all other points", async () => { - await expect(store.countPointsForFile(path.join(workspacePath, "file.ts"), 2)).resolves.toEqual({ - current: 2, - other: 2, - }) - expect(requests.map(({ body }) => body)).toEqual([ - { - filter: { - must: [ - { key: "filePath", match: { value: "file.ts" } }, - { key: "generation", match: { value: 2 } }, - ], - }, - exact: true, - }, - { - filter: { - must: [{ key: "filePath", match: { value: "file.ts" } }], - must_not: [{ key: "generation", match: { value: 2 } }], - }, - exact: true, - }, - ]) -}) - -it.each([-1, 0.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])( - "rejects unsafe generation %s before either delete entry point", - async (generation) => { - await expect(store.deletePointsByFilePath("file.ts", generation)).rejects.toThrow("generation cutoff") - await expect(store.deletePointsByMultipleFilePaths(["file.ts"], generation)).rejects.toThrow( - "generation cutoff", - ) - expect(requests).toHaveLength(0) - }, -) - -it("accepts generation zero through both delete entry points and skips an empty file list", async () => { - await store.deletePointsByFilePath("file.ts", 0) - await store.deletePointsByMultipleFilePaths(["file.ts"], 0) - await store.deletePointsByMultipleFilePaths([], 0) - expect(requests).toHaveLength(2) -}) - -it("creates the required verification indexes before the optional indexes", async () => { - await store["_createPayloadIndexes"]() - expect(requests.slice(0, 2).map(({ body }) => body)).toEqual([ - { field_name: "filePath", field_schema: "keyword" }, - { field_name: "generation", field_schema: "integer" }, - ]) - expect(requests).toHaveLength(8) - expect(requests.every(({ url }) => url.includes("wait=true"))).toBe(true) -}) - -it.each(["filePath", "generation"])("does not swallow a required %s index failure", async (fieldName) => { - const createIndex = vi.spyOn(store["client"], "createPayloadIndex") - if (fieldName === "generation") createIndex.mockResolvedValueOnce({ status: "completed", operation_id: 1 }) - const error = new Error("required index failed") - createIndex.mockRejectedValueOnce(error) - await expect(store["_createPayloadIndexes"]()).rejects.toBe(error) - expect(createIndex).toHaveBeenCalledTimes(fieldName === "generation" ? 2 : 1) -}) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index e397425188..c184896e91 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -11,15 +11,6 @@ import { clearAllMocks } from "../../../../test-utils/reset" vitest.mock("@qdrant/js-client-rest") vitest.mock("crypto") vitest.mock("../../../../utils/path") -// The store logs through codeIndexLog; forward to console so the existing console spies observe it. -vitest.mock("../../code-index-log", () => ({ - codeIndexLog: { - info: (...values: unknown[]) => console.log(...values), - warn: (...values: unknown[]) => console.warn(...values), - error: (...values: unknown[]) => console.error(...values), - debug: () => {}, - }, -})) vitest.mock("../../../../i18n", () => ({ t: (key: string, params?: any) => { // Mock translation function that includes parameters for testing @@ -559,16 +550,14 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", - wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", - wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) }) it("should not create a new collection if one exists with matching vectorSize and return false", async () => { // Mock getCollection to return existing collection info with matching vector size @@ -595,16 +584,14 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", - wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", - wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) }) it("should recreate collection if it exists but vectorSize mismatches and return true", async () => { const differentVectorSize = 768 @@ -654,16 +641,14 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", - wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", - wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) ;(console.warn as any).mockRestore() // Restore console.warn }) it("should log warning for non-404 errors but still create collection", async () => { @@ -677,7 +662,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled() - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) expect(console.warn).toHaveBeenCalledWith( expect.stringContaining(`Warning during getCollectionInfo for "${expectedCollectionName}"`), genericError.message, @@ -705,48 +690,48 @@ describe("QdrantVectorStore", () => { expect(console.error).toHaveBeenCalledTimes(1) // Only the outer try/catch ;(console.error as any).mockRestore() }) - it("should log but not fail if payload index creation errors occur", async () => { - // Mock successful collection creation - mockQdrantClientInstance.getCollection.mockRejectedValue({ - response: { status: 404 }, - message: "Not found", + describe("CI-QD-01: collection and payload index creation", () => { + const serverError = () => Object.assign(new Error("unavailable"), { status: 503 }) + + beforeEach(() => { + vitest.useFakeTimers() + mockQdrantClientInstance.getCollection.mockRejectedValue({ response: { status: 404 }, message: "Not found" }) + mockQdrantClientInstance.createCollection.mockResolvedValue(true) + vitest.spyOn(console, "error").mockImplementation(function () {}) }) - mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) - - // Required verification indexes succeed; only the optional indexes fail. - const indexError = new Error("Index creation failed") - mockQdrantClientInstance.createPayloadIndex.mockImplementation( - async (_collection: string, options: { field_name: string }) => { - if (options.field_name === "filePath" || options.field_name === "generation") return {} - throw indexError + afterEach(() => vitest.useRealTimers()) + + it.each(["createCollection", "createPayloadIndex"] as const)( + "recovers when %s fails twice with a server error", + async (method) => { + mockQdrantClientInstance[method].mockRejectedValueOnce(serverError()).mockRejectedValueOnce(serverError()) + const pending = vectorStore.initialize() + await vitest.advanceTimersByTimeAsync(1500) + await expect(pending).resolves.toBe(true) }, ) - vitest.spyOn(console, "warn").mockImplementation(function () {}) // Suppress console.warn - - const result = await vectorStore.initialize() - // Should still return true since main collection setup succeeded - expect(result).toBe(true) - expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - - // Verify all payload index creations were attempted (2 required + type + 5 pathSegments) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + it("rejects when payload index creation exhausts its retries", async () => { + mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(serverError()) + const pending = vectorStore.initialize().catch((error: unknown) => error) + await vitest.advanceTimersByTimeAsync(1500) + expect(await pending).toBeInstanceOf(Error) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(3) + }) - // Verify warnings were logged for each failed index (now 6) - expect(console.warn).toHaveBeenCalledTimes(6) - // Verify warning for 'type' index - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Could not create payload index for type`), - indexError.message, - ) - for (let i = 0; i <= 4; i++) { - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Could not create payload index for pathSegments.${i}`), - indexError.message, + it("does not retry a permanent creation failure", async () => { + mockQdrantClientInstance.createCollection.mockRejectedValue( + Object.assign(new Error("bad request"), { status: 400 }), ) - } + await expect(vectorStore.initialize()).rejects.toThrow() + expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) + }) - ;(console.warn as any).mockRestore() + it("treats an existing payload index as success", async () => { + mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(new Error("index already exists")) + await expect(vectorStore.initialize()).resolves.toBe(true) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + }) }) it("should throw vectorDimensionMismatch error when deleteCollection fails during recreation", async () => { @@ -867,7 +852,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(2) expect(mockQdrantClientInstance.deleteCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) ;(console.warn as any).mockRestore() }) @@ -964,7 +949,7 @@ describe("QdrantVectorStore", () => { on_disk: true, }, }) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) ;(console.warn as any).mockRestore() }) @@ -1256,10 +1241,7 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.upsert.mockRejectedValue(upsertError) vitest.spyOn(console, "error").mockImplementation(function () {}) - await expect(vectorStore.upsertPoints(mockPoints)).rejects.toMatchObject({ - message: "Upsert failed", - cause: upsertError, - }) + await expect(vectorStore.upsertPoints(mockPoints)).rejects.toThrow(upsertError) expect(mockQdrantClientInstance.upsert).toHaveBeenCalledTimes(1) expect(console.error).toHaveBeenCalledWith("Failed to upsert points:", upsertError) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 38a8302521..af5476522f 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -1,6 +1,3 @@ -import { codeIndexLog } from "../code-index-log" -import { parseRetryAfterMs, VectorStoreError } from "../shared/indexing-errors" -import { makeRateLimitScope } from "../provider-pacer" import { QdrantClient, QdrantClientResourceExhaustedError, Schemas } from "@qdrant/js-client-rest" import { createHash } from "crypto" import * as path from "path" @@ -9,22 +6,29 @@ import { IVectorStore } from "../interfaces/vector-store" import { Payload, VectorStoreSearchResult } from "../interfaces" import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" import { t } from "../../../i18n" - -function normalizeVectorStoreError(error: unknown): VectorStoreError { - return new VectorStoreError( - error instanceof Error ? error.message : String(error), - error, - error instanceof QdrantClientResourceExhaustedError - ? { classification: "rate-limit", statusCode: 429, retryAfterMs: parseRetryAfterMs(error.retry_after) } - : undefined, - ) +import { VectorStoreError } from "../shared/indexing-errors" +import { ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" +import { retryWithPolicy } from "../shared/retry-policy" + +// The patched client reports Retry-After in whole seconds on retry_after (patches/@qdrant__js-client-rest@1.20.0.patch). +function toVectorStoreError(error: unknown): VectorStoreError { + if (error instanceof VectorStoreError) return error + const message = error instanceof Error ? error.message : String(error) + if (error instanceof QdrantClientResourceExhaustedError) { + const seconds = Number((error as QdrantClientResourceExhaustedError).retry_after) + return new VectorStoreError(message, error, { + classification: "rate-limit", + statusCode: 429, + retryAfterMs: Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : undefined, + }) + } + return new VectorStoreError(message, error) } /** * Qdrant implementation of the vector store interface */ export class QdrantVectorStore implements IVectorStore { - readonly rateLimitScope: string private readonly vectorSize!: number private readonly DISTANCE_METRIC = "Cosine" @@ -32,19 +36,26 @@ export class QdrantVectorStore implements IVectorStore { private readonly collectionName: string private readonly qdrantUrl: string = "http://localhost:6333" private readonly workspacePath: string + readonly rateLimitScope: string /** * Creates a new Qdrant vector store * @param workspacePath Path to the workspace * @param url Optional URL to the Qdrant server + * @param pacers Shared rate-limit waits for collection and index creation */ - constructor(workspacePath: string, url: string, vectorSize: number, apiKey?: string) { + constructor( + workspacePath: string, + url: string, + vectorSize: number, + apiKey?: string, + private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + ) { // Parse the URL to determine the appropriate QdrantClient configuration const parsedUrl = this.parseQdrantUrl(url) // Store the resolved URL for our property this.qdrantUrl = parsedUrl - this.rateLimitScope = makeRateLimitScope("qdrant", parsedUrl) this.workspacePath = workspacePath try { @@ -96,6 +107,15 @@ export class QdrantVectorStore implements IVectorStore { const hash = createHash("sha256").update(workspacePath).digest("hex") this.vectorSize = vectorSize this.collectionName = `ws-${hash.substring(0, 16)}` + this.rateLimitScope = makeRateLimitScope("qdrant", this.qdrantUrl) + } + + /** Runs one creation request with bounded server/connection retries and the shared rate-limit wait. */ + private create(request: () => Promise, signal?: AbortSignal): Promise { + return retryWithPolicy(() => this.pacers.get(this.rateLimitScope).run(undefined, signal, request), { + signal, + budget: { retriesUsed: 0 }, + }) } /** @@ -148,7 +168,7 @@ export class QdrantVectorStore implements IVectorStore { return collectionInfo } catch (error: unknown) { if (error instanceof Error) { - codeIndexLog.warn( + console.warn( `[QdrantVectorStore] Warning during getCollectionInfo for "${this.collectionName}". Collection may not exist or another error occurred:`, error.message, ) @@ -159,27 +179,32 @@ export class QdrantVectorStore implements IVectorStore { /** * Initializes the vector store + * @param signal Ends retry and rate-limit waits during collection and index creation * @returns Promise resolving to boolean indicating if a new collection was created */ - async initialize(): Promise { + async initialize(signal?: AbortSignal): Promise { let created = false try { const collectionInfo = await this.getCollectionInfo() if (collectionInfo === null) { // Collection info not retrieved (assume not found or inaccessible), create it - await this.client.createCollection(this.collectionName, { - vectors: { - size: this.vectorSize, - distance: this.DISTANCE_METRIC, - on_disk: true, - }, - hnsw_config: { - m: 64, - ef_construct: 512, - on_disk: true, - }, - }) + await this.create( + () => + this.client.createCollection(this.collectionName, { + vectors: { + size: this.vectorSize, + distance: this.DISTANCE_METRIC, + on_disk: true, + }, + hnsw_config: { + m: 64, + ef_construct: 512, + on_disk: true, + }, + }), + signal, + ) created = true } else { // Collection exists, check vector size @@ -208,11 +233,12 @@ export class QdrantVectorStore implements IVectorStore { } // Create payload indexes - await this._createPayloadIndexes() + await this._createPayloadIndexes(signal) return created } catch (error: any) { + if (signal?.aborted) throw error const errorMessage = error?.message || error - codeIndexLog.error( + console.error( `[QdrantVectorStore] Failed to initialize Qdrant collection "${this.collectionName}":`, errorMessage, ) @@ -235,7 +261,7 @@ export class QdrantVectorStore implements IVectorStore { * @returns Promise resolving to boolean indicating if a new collection was created */ private async _recreateCollectionWithNewDimension(existingVectorSize: number): Promise { - codeIndexLog.warn( + console.warn( `[QdrantVectorStore] Collection ${this.collectionName} exists with vector size ${existingVectorSize}, but expected ${this.vectorSize}. Recreating collection.`, ) @@ -244,10 +270,10 @@ export class QdrantVectorStore implements IVectorStore { try { // Step 1: Attempt to delete the existing collection - codeIndexLog.info(`[QdrantVectorStore] Deleting existing collection ${this.collectionName}...`) + console.log(`[QdrantVectorStore] Deleting existing collection ${this.collectionName}...`) await this.client.deleteCollection(this.collectionName) deletionSucceeded = true - codeIndexLog.info(`[QdrantVectorStore] Successfully deleted collection ${this.collectionName}`) + console.log(`[QdrantVectorStore] Successfully deleted collection ${this.collectionName}`) // Step 2: Wait a brief moment to ensure deletion is processed await new Promise((resolve) => setTimeout(resolve, 100)) @@ -259,7 +285,7 @@ export class QdrantVectorStore implements IVectorStore { } // Step 4: Create the new collection with correct dimensions - codeIndexLog.info( + console.log( `[QdrantVectorStore] Creating new collection ${this.collectionName} with vector size ${this.vectorSize}...`, ) recreationAttempted = true @@ -275,7 +301,7 @@ export class QdrantVectorStore implements IVectorStore { on_disk: true, }, }) - codeIndexLog.info(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) + console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) return true } catch (recreationError) { const errorMessage = recreationError instanceof Error ? recreationError.message : String(recreationError) @@ -290,7 +316,7 @@ export class QdrantVectorStore implements IVectorStore { contextualErrorMessage = `Deleted existing collection but failed to create new collection with vector size ${this.vectorSize}. ${errorMessage}` } - codeIndexLog.error( + console.error( `[QdrantVectorStore] CRITICAL: Failed to recreate collection ${this.collectionName} for dimension change (${existingVectorSize} -> ${this.vectorSize}). ${contextualErrorMessage}`, ) @@ -308,55 +334,23 @@ export class QdrantVectorStore implements IVectorStore { } /** - * Creates payload indexes for the collection, handling errors gracefully. + * Creates payload indexes. An existing index counts as success; any other failure that outlasts the + * retry policy rejects, so initialization is never reported as successful without its indexes. */ - private async _createPayloadIndexes(): Promise { - // Completion verification depends on these indexes; initialization must fail if creation fails. - await this.client.createPayloadIndex(this.collectionName, { - field_name: "filePath", - field_schema: "keyword", - wait: true, - }) - await this.client.createPayloadIndex(this.collectionName, { - field_name: "generation", - field_schema: "integer", - wait: true, - }) - - // Create index for the 'type' field to enable metadata filtering - try { - await this.client.createPayloadIndex(this.collectionName, { - field_name: "type", - field_schema: "keyword", - wait: true, - }) - } catch (indexError: any) { - const errorMessage = (indexError?.message || "").toLowerCase() - if (!errorMessage.includes("already exists")) { - codeIndexLog.warn( - `[QdrantVectorStore] Could not create payload index for type on ${this.collectionName}. Details:`, - indexError?.message || indexError, - ) - } - } - - // Create indexes for pathSegments fields - for (let segmentIndex = 0; segmentIndex <= 4; segmentIndex++) { - try { - await this.client.createPayloadIndex(this.collectionName, { - field_name: `pathSegments.${segmentIndex}`, - field_schema: "keyword", - wait: true, - }) - } catch (indexError: any) { - const errorMessage = (indexError?.message || "").toLowerCase() - if (!errorMessage.includes("already exists")) { - codeIndexLog.warn( - `[QdrantVectorStore] Could not create payload index for pathSegments.${segmentIndex} on ${this.collectionName}. Details:`, - indexError?.message || indexError, - ) + private async _createPayloadIndexes(signal?: AbortSignal): Promise { + const fields = ["type", ...Array.from({ length: 5 }, (_unused, index) => `pathSegments.${index}`)] + for (const fieldName of fields) { + await this.create(async () => { + try { + await this.client.createPayloadIndex(this.collectionName, { + field_name: fieldName, + field_schema: "keyword", + }) + } catch (indexError) { + const message = indexError instanceof Error ? indexError.message.toLowerCase() : "" + if (!message.includes("already exists")) throw indexError } - } + }, signal) } } @@ -398,8 +392,8 @@ export class QdrantVectorStore implements IVectorStore { wait: true, }) } catch (error) { - codeIndexLog.error("Failed to upsert points:", error) - throw normalizeVectorStoreError(error) + console.error("Failed to upsert points:", error) + throw toVectorStoreError(error) } } @@ -491,64 +485,83 @@ export class QdrantVectorStore implements IVectorStore { return filteredPoints as VectorStoreSearchResult[] } catch (error) { - codeIndexLog.error("Failed to search points:", error) + console.error("Failed to search points:", error) throw error } } - private relativeFilePath(filePath: string): string { - return path.normalize(path.isAbsolute(filePath) ? path.relative(this.workspacePath, filePath) : filePath) - } - - async deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise { - return this.deletePointsByMultipleFilePaths([filePath], beforeGeneration) + /** + * Deletes points by file path + * @param filePath Path of the file to delete points for + */ + async deletePointsByFilePath(filePath: string): Promise { + return this.deletePointsByMultipleFilePaths([filePath]) } - async deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise { - if (!Number.isSafeInteger(beforeGeneration) || beforeGeneration < 0) { - throw new Error("A non-negative safe integer generation cutoff is required") + async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { + if (filePaths.length === 0) { + return } - if (filePaths.length === 0) return - await this.client - .delete(this.collectionName, { - filter: { - must: [ - { - key: "filePath", - match: { any: filePaths.map((filePath) => this.relativeFilePath(filePath)) }, - }, - { - should: [ - { key: "generation", range: { lt: beforeGeneration } }, - { is_empty: { key: "generation" } }, - ], - }, - ], - }, + try { + // First check if the collection exists + const collectionExists = await this.collectionExists() + if (!collectionExists) { + console.warn( + `[QdrantVectorStore] Skipping deletion - collection "${this.collectionName}" does not exist`, + ) + return + } + + const workspaceRoot = this.workspacePath + + // Build filters using pathSegments to match the indexed fields + const filters = filePaths.map((filePath) => { + // IMPORTANT: Use the relative path to match what's stored in upsertPoints + // upsertPoints stores the relative filePath, not the absolute path + const relativePath = path.isAbsolute(filePath) ? path.relative(workspaceRoot, filePath) : filePath + + // Normalize the relative path + const normalizedRelativePath = path.normalize(relativePath) + + // Split the path into segments like we do in upsertPoints + const segments = normalizedRelativePath.split(path.sep).filter(Boolean) + + // Create a filter that matches all segments of the path + // This ensures we only delete points that match the exact file path + const mustConditions = segments.map((segment, index) => ({ + key: `pathSegments.${index}`, + match: { value: segment }, + })) + + return { must: mustConditions } + }) + + // Use 'should' to match any of the file paths (OR condition) + const filter = filters.length === 1 ? filters[0] : { should: filters } + + await this.client.delete(this.collectionName, { + filter, wait: true, }) - .catch((error: unknown) => { - throw normalizeVectorStoreError(error) + } catch (error: any) { + // Extract more detailed error information + const errorMessage = error?.message || String(error) + const errorStatus = error?.status || error?.response?.status || error?.statusCode + const errorDetails = error?.response?.data || error?.data || "" + + console.error(`[QdrantVectorStore] Failed to delete points by file paths:`, { + error: errorMessage, + status: errorStatus, + details: errorDetails, + collection: this.collectionName, + fileCount: filePaths.length, + // Include first few file paths for debugging (avoid logging too many) + samplePaths: filePaths.slice(0, 3), }) - } - - async countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> { - const fileCondition = { key: "filePath", match: { value: this.relativeFilePath(filePath) } } - const generationCondition = { key: "generation", match: { value: generation } } - const [current, other] = await Promise.all([ - this.client.count(this.collectionName, { - filter: { must: [fileCondition, generationCondition] }, - exact: true, - }), - this.client.count(this.collectionName, { - filter: { must: [fileCondition], must_not: [generationCondition] }, - exact: true, - }), - ]).catch((error: unknown) => { - throw normalizeVectorStoreError(error) - }) - return { current: current.count, other: other.count } + // A swallowed failure would let callers clear the file hash and lose the stale entries. + throw toVectorStoreError(error) + } } /** @@ -561,7 +574,7 @@ export class QdrantVectorStore implements IVectorStore { await this.client.deleteCollection(this.collectionName) } } catch (error) { - codeIndexLog.error(`[QdrantVectorStore] Failed to delete collection ${this.collectionName}:`, error) + console.error(`[QdrantVectorStore] Failed to delete collection ${this.collectionName}:`, error) throw error // Re-throw to allow calling code to handle it } } @@ -578,7 +591,7 @@ export class QdrantVectorStore implements IVectorStore { wait: true, }) } catch (error) { - codeIndexLog.error("Failed to clear collection:", error) + console.error("Failed to clear collection:", error) throw error } } @@ -622,12 +635,12 @@ export class QdrantVectorStore implements IVectorStore { // Backward compatibility: No marker exists (old index or pre-marker version) // Fall back to old logic - assume complete if collection has points - codeIndexLog.info( + console.log( "[QdrantVectorStore] No indexing metadata marker found. Using backward compatibility mode (checking points_count > 0).", ) return pointsCount > 0 } catch (error) { - codeIndexLog.warn("[QdrantVectorStore] Failed to check if collection has data:", error) + console.warn("[QdrantVectorStore] Failed to check if collection has data:", error) return false } } @@ -656,9 +669,9 @@ export class QdrantVectorStore implements IVectorStore { ], wait: true, }) - codeIndexLog.info("[QdrantVectorStore] Marked indexing as complete") + console.log("[QdrantVectorStore] Marked indexing as complete") } catch (error) { - codeIndexLog.error("[QdrantVectorStore] Failed to mark indexing as complete:", error) + console.error("[QdrantVectorStore] Failed to mark indexing as complete:", error) throw error } } @@ -687,9 +700,9 @@ export class QdrantVectorStore implements IVectorStore { ], wait: true, }) - codeIndexLog.info("[QdrantVectorStore] Marked indexing as incomplete (in progress)") + console.log("[QdrantVectorStore] Marked indexing as incomplete (in progress)") } catch (error) { - codeIndexLog.error("[QdrantVectorStore] Failed to mark indexing as incomplete:", error) + console.error("[QdrantVectorStore] Failed to mark indexing as incomplete:", error) throw error } } diff --git a/src/services/code-index/vector-store/vector-store-factory.ts b/src/services/code-index/vector-store/vector-store-factory.ts index 174f217108..2c8ad82bbb 100644 --- a/src/services/code-index/vector-store/vector-store-factory.ts +++ b/src/services/code-index/vector-store/vector-store-factory.ts @@ -4,9 +4,10 @@ import type { CodeIndexConfig } from "../interfaces/config" import type { IVectorStore } from "../interfaces" import type { IVectorStoreFactory } from "../interfaces/vector-store-factory" import { QdrantVectorStore } from "./qdrant-client" +import type { ProviderPacerRegistry } from "../provider-pacer" export class VectorStoreFactory implements IVectorStoreFactory { - public create(config: CodeIndexConfig, workspacePath: string): IVectorStore { + public create(config: CodeIndexConfig, workspacePath: string, pacers?: ProviderPacerRegistry): IVectorStore { if (config.embedderProvider === "semble") { throw new Error( "Semble provider handles its own vector storage. Do not call createVectorStore() for semble — use SembleProvider instead.", @@ -19,7 +20,7 @@ export class VectorStoreFactory implements IVectorStoreFactory { throw new Error(t("embeddings:serviceFactory.qdrantUrlMissing")) } - return new QdrantVectorStore(workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey) + return new QdrantVectorStore(workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey, pacers) } private resolveVectorSize(config: CodeIndexConfig): number { diff --git a/src/services/code-index/workspace-indexing-enablement-manager.ts b/src/services/code-index/workspace-indexing-enablement-manager.ts index 3552aab210..6edca361d8 100644 --- a/src/services/code-index/workspace-indexing-enablement-manager.ts +++ b/src/services/code-index/workspace-indexing-enablement-manager.ts @@ -9,6 +9,7 @@ export class WorkspaceIndexingEnablementManager { | "setWorkspaceEnabled" | "isFeatureEnabled" | "isFeatureConfigured" + | "initialize" | "startIndexing" | "stopIndexing" | "getCurrentStatus" @@ -21,7 +22,8 @@ export class WorkspaceIndexingEnablementManager { ): Promise { await this.manager.setWorkspaceEnabled(enabled) if (enabled && this.manager.isFeatureEnabled && this.manager.isFeatureConfigured) { - await this.manager.startIndexing(provider.contextProxy) + await this.manager.initialize(provider.contextProxy) + void this.manager.startIndexing().catch((error) => provider.log(`Indexing error: ${error}`)) } else if (!enabled) { await this.manager.stopIndexing() } diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx index 0f45860584..53240cf487 100644 --- a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx +++ b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx @@ -422,8 +422,8 @@ describe("CodeIndexPopover indexing reliability", () => { ).toBeInTheDocument() }) - it("renders all six failure activities and both retry states", () => { - const activities: FailedFileStatus["activity"][] = ["read", "parse", "embed", "upsert", "delete", "verify"] + it("renders all five failure activities and both retry states", () => { + const activities: FailedFileStatus["activity"][] = ["read", "parse", "embed", "upsert", "delete"] const failedFiles: FailedFileStatus[] = activities.map((activity, index) => ({ path: `src/${activity}.ts`, activity, diff --git a/webview-ui/src/i18n/locales/ca/settings.json b/webview-ui/src/i18n/locales/ca/settings.json index 0ad5bdcc8c..f7e9c23fcd 100644 --- a/webview-ui/src/i18n/locales/ca/settings.json +++ b/webview-ui/src/i18n/locales/ca/settings.json @@ -228,8 +228,7 @@ "parse": "Analitzar el fitxer", "embed": "Crear embeddings", "upsert": "Desar els blocs indexats", - "delete": "Eliminar els blocs indexats", - "verify": "Verificar el resultat indexat" + "delete": "Eliminar els blocs indexats" }, "embedderProviderLabel": "Proveïdor d'embeddings", "modelPlaceholder": "Introduïu el nom del model", diff --git a/webview-ui/src/i18n/locales/de/settings.json b/webview-ui/src/i18n/locales/de/settings.json index 3b5710d12b..76a3460d6e 100644 --- a/webview-ui/src/i18n/locales/de/settings.json +++ b/webview-ui/src/i18n/locales/de/settings.json @@ -187,8 +187,7 @@ "parse": "Datei parsen", "embed": "Embeddings erstellen", "upsert": "Indexierte Blöcke speichern", - "delete": "Indexierte Blöcke entfernen", - "verify": "Indexiertes Ergebnis überprüfen" + "delete": "Indexierte Blöcke entfernen" }, "enableLabel": "Codebase-Indexierung aktivieren", "enableDescription": "Aktiviere die Code-Indizierung für eine verbesserte Suche und ein besseres Kontextverständnis", diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index f1c6a6a600..24947a0907 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -260,8 +260,7 @@ "parse": "Parse file", "embed": "Create embeddings", "upsert": "Save indexed blocks", - "delete": "Remove indexed blocks", - "verify": "Verify indexed result" + "delete": "Remove indexed blocks" }, "enableLabel": "Enable Codebase Indexing", "enableDescription": "Enable code indexing for improved search and context understanding", diff --git a/webview-ui/src/i18n/locales/es/settings.json b/webview-ui/src/i18n/locales/es/settings.json index aa560ad957..b80332a214 100644 --- a/webview-ui/src/i18n/locales/es/settings.json +++ b/webview-ui/src/i18n/locales/es/settings.json @@ -187,8 +187,7 @@ "parse": "Analizar archivo", "embed": "Crear embeddings", "upsert": "Guardar bloques indexados", - "delete": "Eliminar bloques indexados", - "verify": "Verificar resultado indexado" + "delete": "Eliminar bloques indexados" }, "enableLabel": "Habilitar indexación de código", "enableDescription": "Habilita la indexación de código para mejorar la búsqueda y la comprensión del contexto", diff --git a/webview-ui/src/i18n/locales/fr/settings.json b/webview-ui/src/i18n/locales/fr/settings.json index c413f407cf..96bb80ee0b 100644 --- a/webview-ui/src/i18n/locales/fr/settings.json +++ b/webview-ui/src/i18n/locales/fr/settings.json @@ -187,8 +187,7 @@ "parse": "Analyser le fichier", "embed": "Créer les embeddings", "upsert": "Enregistrer les blocs indexés", - "delete": "Supprimer les blocs indexés", - "verify": "Vérifier le résultat indexé" + "delete": "Supprimer les blocs indexés" }, "enableLabel": "Activer l'indexation de la base de code", "enableDescription": "Activer l'indexation du code pour une recherche et une compréhension du contexte améliorées", diff --git a/webview-ui/src/i18n/locales/hi/settings.json b/webview-ui/src/i18n/locales/hi/settings.json index 27cf6f7f6f..5cb321319b 100644 --- a/webview-ui/src/i18n/locales/hi/settings.json +++ b/webview-ui/src/i18n/locales/hi/settings.json @@ -228,8 +228,7 @@ "parse": "फ़ाइल पार्स करना", "embed": "एम्बेडिंग बनाना", "upsert": "इंडेक्स किए गए ब्लॉक सहेजना", - "delete": "इंडेक्स किए गए ब्लॉक हटाना", - "verify": "इंडेक्स किए गए परिणाम की पुष्टि करना" + "delete": "इंडेक्स किए गए ब्लॉक हटाना" }, "embedderProviderLabel": "एम्बेडर प्रदाता", "modelPlaceholder": "मॉडल नाम दर्ज करें", diff --git a/webview-ui/src/i18n/locales/id/settings.json b/webview-ui/src/i18n/locales/id/settings.json index ec2412e12b..efbbd44cc4 100644 --- a/webview-ui/src/i18n/locales/id/settings.json +++ b/webview-ui/src/i18n/locales/id/settings.json @@ -228,8 +228,7 @@ "parse": "Urai file", "embed": "Buat embedding", "upsert": "Simpan blok yang diindeks", - "delete": "Hapus blok yang diindeks", - "verify": "Verifikasi hasil yang diindeks" + "delete": "Hapus blok yang diindeks" }, "embedderProviderLabel": "Provider Embedder", "modelPlaceholder": "Masukkan nama model", diff --git a/webview-ui/src/i18n/locales/it/settings.json b/webview-ui/src/i18n/locales/it/settings.json index c9ad3d5f89..a46cc8e6eb 100644 --- a/webview-ui/src/i18n/locales/it/settings.json +++ b/webview-ui/src/i18n/locales/it/settings.json @@ -228,8 +228,7 @@ "parse": "Analizzare il file", "embed": "Creare gli embedding", "upsert": "Salvare i blocchi indicizzati", - "delete": "Rimuovere i blocchi indicizzati", - "verify": "Verificare il risultato indicizzato" + "delete": "Rimuovere i blocchi indicizzati" }, "embedderProviderLabel": "Provider Embedder", "modelPlaceholder": "Inserisci il nome del modello", diff --git a/webview-ui/src/i18n/locales/ja/settings.json b/webview-ui/src/i18n/locales/ja/settings.json index 20ca24f1d0..a1bd21ce68 100644 --- a/webview-ui/src/i18n/locales/ja/settings.json +++ b/webview-ui/src/i18n/locales/ja/settings.json @@ -228,8 +228,7 @@ "parse": "ファイルの解析", "embed": "埋め込みの作成", "upsert": "インデックス化したブロックの保存", - "delete": "インデックス化したブロックの削除", - "verify": "インデックス化した結果の検証" + "delete": "インデックス化したブロックの削除" }, "embedderProviderLabel": "エンベッダープロバイダー", "modelPlaceholder": "モデル名を入力", diff --git a/webview-ui/src/i18n/locales/ko/settings.json b/webview-ui/src/i18n/locales/ko/settings.json index d47fd671a7..95b277ffcc 100644 --- a/webview-ui/src/i18n/locales/ko/settings.json +++ b/webview-ui/src/i18n/locales/ko/settings.json @@ -228,8 +228,7 @@ "parse": "파일 파싱", "embed": "임베딩 생성", "upsert": "인덱싱된 블록 저장", - "delete": "인덱싱된 블록 제거", - "verify": "인덱싱된 결과 검증" + "delete": "인덱싱된 블록 제거" }, "embedderProviderLabel": "임베더 제공자", "modelPlaceholder": "모델 이름을 입력하세요", diff --git a/webview-ui/src/i18n/locales/nl/settings.json b/webview-ui/src/i18n/locales/nl/settings.json index 7db47697a1..897d606028 100644 --- a/webview-ui/src/i18n/locales/nl/settings.json +++ b/webview-ui/src/i18n/locales/nl/settings.json @@ -228,8 +228,7 @@ "parse": "Bestand parseren", "embed": "Embeddings maken", "upsert": "Geïndexeerde blokken opslaan", - "delete": "Geïndexeerde blokken verwijderen", - "verify": "Geïndexeerd resultaat controleren" + "delete": "Geïndexeerde blokken verwijderen" }, "embedderProviderLabel": "Embedder Provider", "modelPlaceholder": "Voer modelnaam in", diff --git a/webview-ui/src/i18n/locales/pl/settings.json b/webview-ui/src/i18n/locales/pl/settings.json index cab9cf9e1e..7996814b1e 100644 --- a/webview-ui/src/i18n/locales/pl/settings.json +++ b/webview-ui/src/i18n/locales/pl/settings.json @@ -228,8 +228,7 @@ "parse": "Parsowanie pliku", "embed": "Tworzenie embeddingów", "upsert": "Zapisywanie zaindeksowanych bloków", - "delete": "Usuwanie zaindeksowanych bloków", - "verify": "Weryfikacja zaindeksowanego wyniku" + "delete": "Usuwanie zaindeksowanych bloków" }, "embedderProviderLabel": "Dostawca Embeddera", "modelPlaceholder": "Wprowadź nazwę modelu", diff --git a/webview-ui/src/i18n/locales/pt-BR/settings.json b/webview-ui/src/i18n/locales/pt-BR/settings.json index 020712a9af..14ac2a8b47 100644 --- a/webview-ui/src/i18n/locales/pt-BR/settings.json +++ b/webview-ui/src/i18n/locales/pt-BR/settings.json @@ -228,8 +228,7 @@ "parse": "Analisar arquivo", "embed": "Criar embeddings", "upsert": "Salvar blocos indexados", - "delete": "Remover blocos indexados", - "verify": "Verificar resultado indexado" + "delete": "Remover blocos indexados" }, "embedderProviderLabel": "Provedor de Embedder", "modelPlaceholder": "Insira o nome do modelo", diff --git a/webview-ui/src/i18n/locales/ru/settings.json b/webview-ui/src/i18n/locales/ru/settings.json index 86496f45e3..d33116dd7a 100644 --- a/webview-ui/src/i18n/locales/ru/settings.json +++ b/webview-ui/src/i18n/locales/ru/settings.json @@ -228,8 +228,7 @@ "parse": "Разбор файла", "embed": "Создание эмбеддингов", "upsert": "Сохранение проиндексированных блоков", - "delete": "Удаление проиндексированных блоков", - "verify": "Проверка результата индексации" + "delete": "Удаление проиндексированных блоков" }, "embedderProviderLabel": "Провайдер эмбеддера", "modelPlaceholder": "Введите название модели", diff --git a/webview-ui/src/i18n/locales/tr/settings.json b/webview-ui/src/i18n/locales/tr/settings.json index a234a66d24..7f29516c10 100644 --- a/webview-ui/src/i18n/locales/tr/settings.json +++ b/webview-ui/src/i18n/locales/tr/settings.json @@ -228,8 +228,7 @@ "parse": "Dosya ayrıştırma", "embed": "Gömme oluşturma", "upsert": "İndekslenen blokları kaydetme", - "delete": "İndekslenen blokları kaldırma", - "verify": "İndekslenen sonucu doğrulama" + "delete": "İndekslenen blokları kaldırma" }, "embedderProviderLabel": "Gömücü Sağlayıcı", "modelPlaceholder": "Model adını girin", diff --git a/webview-ui/src/i18n/locales/vi/settings.json b/webview-ui/src/i18n/locales/vi/settings.json index 5344ab8789..8c84fc8fbf 100644 --- a/webview-ui/src/i18n/locales/vi/settings.json +++ b/webview-ui/src/i18n/locales/vi/settings.json @@ -228,8 +228,7 @@ "parse": "Phân tích tệp", "embed": "Tạo embedding", "upsert": "Lưu các khối đã lập chỉ mục", - "delete": "Xóa các khối đã lập chỉ mục", - "verify": "Xác minh kết quả đã lập chỉ mục" + "delete": "Xóa các khối đã lập chỉ mục" }, "embedderProviderLabel": "Nhà cung cấp Embedder", "modelPlaceholder": "Nhập tên mô hình", diff --git a/webview-ui/src/i18n/locales/zh-CN/settings.json b/webview-ui/src/i18n/locales/zh-CN/settings.json index 445ae62dae..7a90bd887d 100644 --- a/webview-ui/src/i18n/locales/zh-CN/settings.json +++ b/webview-ui/src/i18n/locales/zh-CN/settings.json @@ -187,8 +187,7 @@ "parse": "解析文件", "embed": "创建嵌入向量", "upsert": "保存已索引的代码块", - "delete": "移除已索引的代码块", - "verify": "验证索引结果" + "delete": "移除已索引的代码块" }, "enableLabel": "启用代码库索引", "enableDescription": "启用代码索引以改进搜索和上下文理解", diff --git a/webview-ui/src/i18n/locales/zh-TW/settings.json b/webview-ui/src/i18n/locales/zh-TW/settings.json index 20964fc298..8fdc9f8f0d 100644 --- a/webview-ui/src/i18n/locales/zh-TW/settings.json +++ b/webview-ui/src/i18n/locales/zh-TW/settings.json @@ -214,8 +214,7 @@ "parse": "剖析檔案", "embed": "建立嵌入向量", "upsert": "儲存已索引的區塊", - "delete": "移除已索引的區塊", - "verify": "驗證索引結果" + "delete": "移除已索引的區塊" }, "enableLabel": "啟用程式碼庫索引", "enableDescription": "啟用程式碼索引以改進搜尋和上下文理解", From 51dfbbd071e04aa5b7f06ca067dd1efb4979616c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 13:45:41 -0800 Subject: [PATCH 23/42] test(code-index): visual specs for failed-file popover, provider-wait popover and amber badge --- webview-ui/playwright/gallery/stories.tsx | 16 ++++++ .../CodeIndexStatus.visual.fixture.tsx | 53 +++++++++++++++++++ .../chat/__tests__/CodeIndexStatus.visual.tsx | 23 ++++++++ 3 files changed, 92 insertions(+) create mode 100644 webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.fixture.tsx create mode 100644 webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.tsx diff --git a/webview-ui/playwright/gallery/stories.tsx b/webview-ui/playwright/gallery/stories.tsx index e832802dbc..c3b985b653 100644 --- a/webview-ui/playwright/gallery/stories.tsx +++ b/webview-ui/playwright/gallery/stories.tsx @@ -399,6 +399,22 @@ export const stories: Record = { ) }, + ...Object.fromEntries( + ["failed-files", "provider-wait", "amber-badge"].map((variant): [string, Story] => [ + `code-index-status-${variant}`, + async () => { + const [{ AppProviders }, { CodeIndexStatusFixture }] = await Promise.all([ + import("../AppProviders"), + import("@/components/chat/__tests__/CodeIndexStatus.visual.fixture"), + ]) + return ( + + + + ) + }, + ]), + ), "theme-token-cleanup": async () => { const [{ Checkbox }, { enabledChatControlClassName }] = await Promise.all([ import("@/components/ui/checkbox"), diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.fixture.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.fixture.tsx new file mode 100644 index 0000000000..7fad063c22 --- /dev/null +++ b/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.fixture.tsx @@ -0,0 +1,53 @@ +import { useEffect } from "react" + +import type { IndexingStatus } from "@roo-code/types" + +import { IndexingStatusBadge } from "../IndexingStatusBadge" + +const failedFile = { + path: "src/broken.ts", + activity: "embed", + reason: "Server error after 3 attempts", + state: "exhausted", +} as const + +export const codeIndexStatusFixtures: Record = { + "failed-files": { + systemStatus: "Standby", + processedItems: 41, + totalItems: 42, + currentItemUnit: "blocks", + failedFiles: [failedFile], + failedFileSummary: { ...failedFile, count: 1 }, + }, + "provider-wait": { + systemStatus: "Indexing", + processedItems: 10, + totalItems: 42, + currentItemUnit: "blocks", + providerWaits: [{ service: "OpenAI", waitingFiles: ["src/a.ts", "src/b.ts"] }], + }, + "amber-badge": { + systemStatus: "Indexed", + processedItems: 42, + totalItems: 42, + currentItemUnit: "blocks", + failedFiles: [failedFile], + }, +} + +function StatusFeeder({ status }: { status: IndexingStatus }) { + useEffect(() => { + window.postMessage({ type: "indexingStatusUpdate", values: status }, "*") + }, [status]) + return null +} + +export function CodeIndexStatusFixture({ variant }: { variant: string }) { + return ( +
+ + +
+ ) +} diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.tsx new file mode 100644 index 0000000000..78589825eb --- /dev/null +++ b/webview-ui/src/components/chat/__tests__/CodeIndexStatus.visual.tsx @@ -0,0 +1,23 @@ +import { expect, test } from "../../../../playwright/coverage-fixture" +import { mountedStory } from "../../../../playwright/mounted-story" + +const popoverCases = [ + { story: "code-index-status-failed-files", text: /broken\.ts/ }, + { story: "code-index-status-provider-wait", text: /OpenAI/ }, +] + +for (const { story, text } of popoverCases) { + test(`renders the ${story} popover`, async ({ mount, page }) => { + const component = mountedStory(await mount(story)) + await component.getByRole("button").first().click() + const popover = page.getByRole("dialog") + await expect(popover.getByText(text).first()).toBeVisible() + await expect(popover).toHaveScreenshot(`${story}.png`) + }) +} + +test("shows the amber badge for an idle partial index", async ({ mount }) => { + const component = mountedStory(await mount("code-index-status-amber-badge")) + await expect(component.locator(".bg-amber-500")).toBeVisible() + await expect(component).toHaveScreenshot("code-index-status-amber-badge.png") +}) From 88d62cfb5222b7747a9801c4e33168247f7527d1 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 13:46:45 -0800 Subject: [PATCH 24/42] style(code-index): apply prettier --- .../CodebaseSearchTool.workspace.spec.ts | 12 ++------ .../__tests__/search-service.spec.ts | 30 +++++++++++-------- .../embedders/__tests__/bedrock.spec.ts | 17 +++++++++-- src/services/code-index/embedders/bedrock.ts | 8 ++--- .../code-index/embedders/openai-compatible.ts | 1 - .../code-index/embedders/openrouter.ts | 2 -- .../code-index/shared/validation-helpers.ts | 3 +- .../qdrant-client.rate-limit.spec.ts | 4 ++- .../__tests__/qdrant-client.spec.ts | 9 ++++-- 9 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts index 808afa4916..475739a3f1 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.workspace.spec.ts @@ -190,16 +190,8 @@ describe("CodebaseSearchTool workspace selection", () => { await tool.execute({ query: "without an editor" }, task, callbacks) expect(taskManager.searchIndex).toHaveBeenCalledTimes(3) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith( - 1, - "before editor switch", - undefined - ) - expect(taskManager.searchIndex).toHaveBeenNthCalledWith( - 2, - "after editor switch", - undefined - ) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith(1, "before editor switch", undefined) + expect(taskManager.searchIndex).toHaveBeenNthCalledWith(2, "after editor switch", undefined) expect(taskManager.searchIndex).toHaveBeenNthCalledWith(3, "without an editor", undefined) expect(editorManager.searchIndex).not.toHaveBeenCalled() expect(getWorkspacePath).not.toHaveBeenCalled() diff --git a/src/services/code-index/__tests__/search-service.spec.ts b/src/services/code-index/__tests__/search-service.spec.ts index cd791b5508..49a157f14b 100644 --- a/src/services/code-index/__tests__/search-service.spec.ts +++ b/src/services/code-index/__tests__/search-service.spec.ts @@ -67,13 +67,16 @@ describe("CodeIndexSearchService (CI-SS-01)", () => { { state: "Indexed", failed: true, incomplete: true }, { state: "Indexing", failed: false, incomplete: true }, { state: "Standby", failed: true, incomplete: true }, - ] as const)("$state with failures=$failed reports incomplete=$incomplete", async ({ state, failed, incomplete }) => { - const { service } = prepare(state, { failed }) - const response = await service.searchIndex("query") - expect(response.incomplete).toBe(incomplete) - expect(response.failedPaths).toEqual(failed ? ["src/a.ts", "other/b.ts"] : []) - expect(response.waitingPaths).toEqual([]) - }) + ] as const)( + "$state with failures=$failed reports incomplete=$incomplete", + async ({ state, failed, incomplete }) => { + const { service } = prepare(state, { failed }) + const response = await service.searchIndex("query") + expect(response.incomplete).toBe(incomplete) + expect(response.failedPaths).toEqual(failed ? ["src/a.ts", "other/b.ts"] : []) + expect(response.waitingPaths).toEqual([]) + }, + ) it("reports waiting paths separately and deduplicated, for matches and empty results", async () => { const { service, pacers, store } = prepare("Indexing", { failed: true }) @@ -105,11 +108,14 @@ describe("CodeIndexSearchService (CI-SS-01)", () => { { state: "Stopping", failed: true, exists: true }, { state: "Standby", failed: false, exists: true }, { state: "Standby", failed: true, exists: false }, - ] as const)("rejects search in $state (failures=$failed, collection=$exists)", async ({ state, failed, exists }) => { - const { service, embedder } = prepare(state, { failed, exists }) - await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) - expect(embedder.createEmbeddings).not.toHaveBeenCalled() - }) + ] as const)( + "rejects search in $state (failures=$failed, collection=$exists)", + async ({ state, failed, exists }) => { + const { service, embedder } = prepare(state, { failed, exists }) + await expect(service.searchIndex("query")).rejects.toThrow(`Current state: ${state}`) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) it("does not set Error when a query fails", async () => { const { service, embedder, setSystemState } = prepare("Indexed") diff --git a/src/services/code-index/embedders/__tests__/bedrock.spec.ts b/src/services/code-index/embedders/__tests__/bedrock.spec.ts index b671f029fc..4ffdd647cb 100644 --- a/src/services/code-index/embedders/__tests__/bedrock.spec.ts +++ b/src/services/code-index/embedders/__tests__/bedrock.spec.ts @@ -255,7 +255,12 @@ describe("BedrockEmbedder", () => { }) it("should create embeddings with Nova multimodal model", async () => { - const novaMultimodalEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.nova-2-multimodal-embeddings-v1:0", pacers) + const novaMultimodalEmbedder = new BedrockEmbedder( + "us-east-1", + "test-profile", + "amazon.nova-2-multimodal-embeddings-v1:0", + pacers, + ) const testTexts = ["Hello world"] const mockResponse = { body: new TextEncoder().encode( @@ -300,7 +305,12 @@ describe("BedrockEmbedder", () => { }) it("should handle Nova multimodal model with multiple texts", async () => { - const novaMultimodalEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.nova-2-multimodal-embeddings-v1:0", pacers) + const novaMultimodalEmbedder = new BedrockEmbedder( + "us-east-1", + "test-profile", + "amazon.nova-2-multimodal-embeddings-v1:0", + pacers, + ) const testTexts = ["Hello world", "Another text"] const mockResponses = [ { @@ -671,7 +681,8 @@ describe("BedrockEmbedder", () => { describe("CI-EM-02: temporary failures", () => { const okResponse = { body: new TextEncoder().encode(JSON.stringify({ embedding: [0.1] })) } - const serverError = () => Object.assign(new Error("Service unavailable"), { $metadata: { httpStatusCode: 503 } }) + const serverError = () => + Object.assign(new Error("Service unavailable"), { $metadata: { httpStatusCode: 503 } }) const throttle = () => Object.assign(new Error("Slow down"), { name: "ThrottlingException" }) beforeEach(() => vitest.useFakeTimers()) diff --git a/src/services/code-index/embedders/bedrock.ts b/src/services/code-index/embedders/bedrock.ts index c2f8e683e3..10824172fa 100644 --- a/src/services/code-index/embedders/bedrock.ts +++ b/src/services/code-index/embedders/bedrock.ts @@ -29,10 +29,10 @@ export class BedrockEmbedder implements IEmbedder { * @param pacers Shared rate-limit waits; validation requests honor the same pause as indexing */ constructor( - private readonly region: string, - private readonly profile: string | undefined, - modelId: string | undefined, - private readonly pacers: ProviderPacerRegistry, + private readonly region: string, + private readonly profile: string | undefined, + modelId: string | undefined, + private readonly pacers: ProviderPacerRegistry, ) { if (!region) { throw new Error("Region is required for AWS Bedrock embedder") diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 2125992904..105a438d7d 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -334,5 +334,4 @@ export class OpenAICompatibleEmbedder implements IEmbedder { name: "openai-compatible", } } - } diff --git a/src/services/code-index/embedders/openrouter.ts b/src/services/code-index/embedders/openrouter.ts index e82a3cb737..f63af536f9 100644 --- a/src/services/code-index/embedders/openrouter.ts +++ b/src/services/code-index/embedders/openrouter.ts @@ -210,7 +210,6 @@ export class OpenRouterEmbedder implements IEmbedder { } } - /** * Validates the OpenRouter embedder configuration by testing API connectivity * @returns Promise resolving to validation result with success status and optional error message @@ -271,5 +270,4 @@ export class OpenRouterEmbedder implements IEmbedder { name: "openrouter", } } - } diff --git a/src/services/code-index/shared/validation-helpers.ts b/src/services/code-index/shared/validation-helpers.ts index 12614f8365..c1ce54ed72 100644 --- a/src/services/code-index/shared/validation-helpers.ts +++ b/src/services/code-index/shared/validation-helpers.ts @@ -213,7 +213,8 @@ export async function withValidationErrorHandling { beforeEach(() => { vitest.useFakeTimers() - mockQdrantClientInstance.getCollection.mockRejectedValue({ response: { status: 404 }, message: "Not found" }) + mockQdrantClientInstance.getCollection.mockRejectedValue({ + response: { status: 404 }, + message: "Not found", + }) mockQdrantClientInstance.createCollection.mockResolvedValue(true) vitest.spyOn(console, "error").mockImplementation(function () {}) }) @@ -704,7 +707,9 @@ describe("QdrantVectorStore", () => { it.each(["createCollection", "createPayloadIndex"] as const)( "recovers when %s fails twice with a server error", async (method) => { - mockQdrantClientInstance[method].mockRejectedValueOnce(serverError()).mockRejectedValueOnce(serverError()) + mockQdrantClientInstance[method] + .mockRejectedValueOnce(serverError()) + .mockRejectedValueOnce(serverError()) const pending = vectorStore.initialize() await vitest.advanceTimersByTimeAsync(1500) await expect(pending).resolves.toBe(true) From bbcd37507d03d1fd8749fabf93ecf29eeda51282 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 14:12:08 -0800 Subject: [PATCH 25/42] fix(code-index): address review findings and CI failures Start after a partial index stops the watcher session and rescans; segment-hash point ids; skip delete for new files; Qdrant recreate retried; required pacer and registry parameters; remove unused registry and orchestrator members; restore suppression entries; visual baselines from CI. --- .../__tests__/webviewMessageHandler.spec.ts | 54 +++++++ src/eslint-suppressions.json | 37 ++++- .../__tests__/file-failure-registry.spec.ts | 51 ++---- .../__tests__/file-index-writer.spec.ts | 37 +++-- .../__tests__/manager.reliability.spec.ts | 31 ++-- .../code-index/__tests__/manager.spec.ts | 16 +- .../code-index/__tests__/orchestrator.spec.ts | 38 ++++- .../__tests__/provider-pacer.spec.ts | 10 +- .../__tests__/service-factory.spec.ts | 153 ++++++++++-------- src/services/code-index/constants/index.ts | 1 - .../__tests__/embedder-factory.spec.ts | 5 - .../embedders/__tests__/ollama.spec.ts | 17 ++ .../embedders/__tests__/openrouter.spec.ts | 13 ++ .../code-index/embedders/embedder-factory.ts | 2 +- .../factories/bedrock-embedder-factory.ts | 4 +- .../code-index/embedders/openai-compatible.ts | 25 ++- .../code-index/embedders/openrouter.ts | 24 ++- .../code-index/file-failure-registry.ts | 53 +----- src/services/code-index/file-index-writer.ts | 14 +- .../code-index/interfaces/embedder-factory.ts | 2 +- .../interfaces/vector-store-factory.ts | 2 +- src/services/code-index/orchestrator.ts | 63 +++----- .../__tests__/file-preparation.spec.ts | 3 +- .../processors/__tests__/file-watcher.spec.ts | 10 +- .../processors/__tests__/parser.spec.ts | 5 +- .../__tests__/processor-factories.spec.ts | 13 +- .../processors/__tests__/scanner.spec.ts | 78 +++++---- .../code-index/processors/file-watcher.ts | 32 ++-- src/services/code-index/processors/scanner.ts | 24 +-- src/services/code-index/search-service.ts | 21 ++- src/services/code-index/service-factory.ts | 8 +- .../shared/__tests__/indexing-errors.spec.ts | 4 +- .../shared/__tests__/retry-policy.spec.ts | 45 +++--- .../code-index/shared/retry-policy.ts | 2 +- .../code-index/shared/validation-helpers.ts | 3 +- .../qdrant-client.rate-limit.spec.ts | 25 ++- .../__tests__/qdrant-client.spec.ts | 142 ++++++++++++++-- .../__tests__/vector-store-factory.spec.ts | 30 ++-- .../code-index/vector-store/qdrant-client.ts | 43 ++--- .../vector-store/vector-store-factory.ts | 2 +- .../chat/__tests__/CodeIndexPopover.test.tsx | 4 +- .../code-index-status-amber-badge.png | Bin 0 -> 869 bytes .../code-index-status-failed-files.png | Bin 0 -> 18118 bytes .../code-index-status-provider-wait.png | Bin 0 -> 15648 bytes 44 files changed, 741 insertions(+), 405 deletions(-) create mode 100644 webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-amber-badge.png create mode 100644 webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-failed-files.png create mode 100644 webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-provider-wait.png diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index 98c306a4f3..5091dc5676 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -773,6 +773,60 @@ describe("webviewMessageHandler - theme fixture probes", () => { }) }) +describe("CI-WV-01: code index start and status forwarding", () => { + const status = { + systemStatus: "Standby", + message: "partial", + processedItems: 1, + totalItems: 2, + currentItemUnit: "blocks", + failedFiles: [{ path: "a.ts", activity: "embed", reason: "down", state: "exhausted" }], + providerWaits: [{ service: "openai", resumeAt: 1, waitingFiles: ["b.ts"] }], + } + + function withManager(state: string) { + const manager = { + setWorkspaceEnabled: vi.fn().mockResolvedValue(undefined), + initialize: vi.fn().mockResolvedValue(undefined), + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + getCurrentStatus: vi.fn().mockReturnValue(status), + isFeatureEnabled: true, + isFeatureConfigured: true, + isInitialized: true, + state, + } + // The double provides only the scope lookup and webview members these handlers call. + const provider = { + ...mockClineProvider, + postMessageToWebview: vi.fn(), + getCurrentWorkspaceCodeIndexScope: () => ({ codeIndexManager: manager }), + } as unknown as ClineProvider + return { manager, provider } + } + + beforeEach(() => vi.clearAllMocks()) + + it.each(["Standby", "Error"])("starts indexing once from %s", async (state) => { + const { manager, provider } = withManager(state) + await webviewMessageHandler(provider, { type: "startIndexing" }) + expect(manager.startIndexing).toHaveBeenCalledTimes(1) + }) + + it("does not start again while indexing is active", async () => { + const { manager, provider } = withManager("Indexing") + await webviewMessageHandler(provider, { type: "startIndexing" }) + expect(manager.startIndexing).not.toHaveBeenCalled() + }) + + it("forwards failed files and provider waits after a stop", async () => { + const { manager, provider } = withManager("Indexing") + await webviewMessageHandler(provider, { type: "stopIndexing" }) + expect(manager.stopIndexing).toHaveBeenCalledTimes(1) + expect(provider.postMessageToWebview).toHaveBeenCalledWith({ type: "indexingStatusUpdate", values: status }) + }) +}) + import { changeLanguage, t } from "../../../i18n" vi.mock("vscode", () => { diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index e649abe268..61074bdfec 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1019,6 +1019,11 @@ "count": 1 } }, + "services/code-index/__tests__/manager.spec.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 84 + } + }, "services/code-index/__tests__/service-factory.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 43 @@ -1054,16 +1059,41 @@ "count": 4 } }, + "services/code-index/embedders/openai-compatible.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 1 + } + }, + "services/code-index/embedders/openrouter.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 2 + } + }, "services/code-index/interfaces/vector-store.ts": { "@typescript-eslint/no-explicit-any": { "count": 2 } }, + "services/code-index/orchestrator.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 2 + } + }, + "services/code-index/processors/__tests__/file-watcher.spec.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 25 + } + }, "services/code-index/processors/__tests__/parser.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 22 } }, + "services/code-index/processors/__tests__/scanner.spec.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 26 + } + }, "services/code-index/semble/__tests__/provider.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 12 @@ -1089,11 +1119,16 @@ "count": 3 } }, - "services/code-index/shared/validation-helpers.ts": { + "services/code-index/shared/__tests__/validation-helpers.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 4 } }, + "services/code-index/shared/validation-helpers.ts": { + "@typescript-eslint/no-explicit-any": { + "count": 5 + } + }, "services/code-index/vector-store/__tests__/qdrant-client.spec.ts": { "@typescript-eslint/no-explicit-any": { "count": 87 diff --git a/src/services/code-index/__tests__/file-failure-registry.spec.ts b/src/services/code-index/__tests__/file-failure-registry.spec.ts index 736ee3425b..224a14fa2c 100644 --- a/src/services/code-index/__tests__/file-failure-registry.spec.ts +++ b/src/services/code-index/__tests__/file-failure-registry.spec.ts @@ -21,73 +21,46 @@ describe("FileFailureRegistry", () => { expect(registry.getRetryBudget("a.ts", "two", "embed").retriesUsed).toBe(0) }) - it("retains exhausted budgets per revision and operation until explicit reset", () => { + it("keeps a displayed entry across a reset and a new revision until it is cleared", () => { const registry = new FileFailureRegistry() registry.renew("a.ts", "one") registry.record("a.ts", "one", "upsert", "server failed", 3, "exhausted", "server") - registry.renew("a.ts", "one") - registry.renew("b.ts", "new") - expect(registry.isExhausted("a.ts", "one", "upsert")).toBe(true) - expect(registry.isExhausted("a.ts", "one", "embed")).toBe(false) - expect(() => registry.assertRetryable("a.ts", "one")).toThrow("server failed") registry.resetAllBudgets() - expect(registry.isExhausted("a.ts", "one", "upsert")).toBe(false) - expect(registry.list()).toHaveLength(1) - registry.clear("a.ts", "one") - expect(registry.list()).toEqual([]) - }) - - it("distinguishes retained display entries from current revision failures", () => { - const registry = new FileFailureRegistry() - expect(registry.hasRevisionFailure("a.ts", "one")).toBe(false) - registry.renew("a.ts", "one") - registry.record("a.ts", "one", "embed", "old failure", 3, "exhausted", "server") - expect(registry.hasRevisionFailure("a.ts", "one")).toBe(true) registry.renew("a.ts", "two") - expect(registry.has("a.ts")).toBe(true) - expect(registry.hasRevisionFailure("a.ts", "one")).toBe(false) - expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) - registry.record("a.ts", "two", "upsert", "new failure", 1, "exhausted") - expect(registry.hasRevisionFailure("a.ts", "two")).toBe(true) - registry.resetAllBudgets() - expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) - expect(registry.has("a.ts")).toBe(true) - registry.record("a.ts", "two", "delete", "failed again", 1, "exhausted") + expect(registry.list()).toEqual([ + expect.objectContaining({ path: "a.ts", revision: "two", state: "exhausted" }), + ]) registry.clear("a.ts", "two") - expect(registry.hasRevisionFailure("a.ts", "two")).toBe(false) + expect(registry.list()).toEqual([]) }) - it("aborts only renewed path work and rejects stale records and clearing", () => { + it("rejects stale records and clearing after a path is renewed", () => { const registry = new FileFailureRegistry() - const old = registry.renew("a.ts", "old") - const other = registry.renew("b.ts", "one") + registry.renew("a.ts", "old") registry.record("a.ts", "old", "embed", "failed", 1, "retrying", "connection") registry.renew("a.ts", "new") - expect(old.aborted).toBe(true) - expect(other.aborted).toBe(false) + expect(registry.isCurrent("a.ts", "old")).toBe(false) registry.record("a.ts", "new", "parse", "new failure", 1, "exhausted") registry.record("a.ts", "old", "delete", "late failure", 3, "exhausted") registry.clear("a.ts", "old") expect(registry.list()[0]).toMatchObject({ revision: "new", activity: "parse", reason: "new failure" }) }) - it("emits every failed attempt and returns detached details", () => { + it("signals each change and returns detached details", () => { const registry = new FileFailureRegistry() - const records = vi.fn() const changed = vi.fn() - const subscription = registry.onDidRecord(records) - registry.onDidChange(changed) + const subscription = registry.onDidChange(changed) registry.renew("a.ts", "hash") registry.record("a.ts", "hash", "upsert", "busy", 1, "retrying", "server") registry.record("a.ts", "hash", "upsert", "busy", 2, "retrying", "server") - expect(records).toHaveBeenCalledTimes(2) expect(changed).toHaveBeenCalledTimes(2) registry.list()[0].reason = "mutated" expect(registry.list()[0].reason).toBe("busy") subscription.dispose() registry.clear("a.ts") - expect(records).toHaveBeenCalledTimes(2) + expect(changed).toHaveBeenCalledTimes(2) }) + it("CI-FR-03: a success clears the entry for that path only", () => { const registry = new FileFailureRegistry() registry.renew("a.ts", "one") diff --git a/src/services/code-index/__tests__/file-index-writer.spec.ts b/src/services/code-index/__tests__/file-index-writer.spec.ts index 17e4d510bf..5e2c071fb3 100644 --- a/src/services/code-index/__tests__/file-index-writer.spec.ts +++ b/src/services/code-index/__tests__/file-index-writer.spec.ts @@ -28,7 +28,7 @@ function setup() { upsertPoints: vi.fn(async () => {}), deletePointsByFilePath: vi.fn(async () => {}), } - const cache = { updateHash: vi.fn(), deleteHash: vi.fn() } + const cache = { getHash: vi.fn((): string | undefined => "old"), updateHash: vi.fn(), deleteHash: vi.fn() } const failures = new FileFailureRegistry() const pacers = new ProviderPacerRegistry() const run = vi.spyOn(pacers.get("openai:1"), "run") @@ -57,25 +57,36 @@ describe("FileIndexWriter", () => { expect(points[0].payload.filePath).toBe("a.ts") }) - it("uses the same point id for the same path and start line", async () => { + it("gives segments that share a start line distinct point ids", async () => { const { writer, store } = setup() - await writer.replace("/ws/a.ts", "h1", [block(3)]) - await writer.replace("/ws/a.ts", "h2", [{ ...block(3), content: "changed", segmentHash: "other" }]) + await writer.replace("/ws/a.ts", "h1", [block(3), { ...block(3), content: "tail", segmentHash: "other" }]) const ids = (store.upsertPoints.mock.calls as unknown as Array<[Array<{ id: string }>]>).map( ([points]) => points[0].id, ) - expect(ids[0]).toBe(ids[1]) + expect(new Set(ids).size).toBe(2) + }) + + it.each([ + ["a cached file", "old", 1], + ["a new file", undefined, 0], + ])("deletes old points only for %s", async (_name, cachedHash, deletes) => { + const { writer, store, cache } = setup() + cache.getHash.mockReturnValue(cachedHash) + await expect(writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(true) + expect(store.deletePointsByFilePath).toHaveBeenCalledTimes(deletes) }) it.each(["embed", "upsert"] as const)( "CI-WR-02: exhausted %s records the activity and keeps the hash", async (activity) => { - const context = setup() - if (activity === "embed") context.embedder.createEmbeddings.mockRejectedValue(serverError()) - else context.store.upsertPoints.mockRejectedValue(serverError()) - await expect(context.writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(false) - expect(context.cache.updateHash).not.toHaveBeenCalled() - expect(context.failures.get("a.ts")).toMatchObject({ activity, state: "exhausted", attempt: 3 }) + const context = setup() + if (activity === "embed") context.embedder.createEmbeddings.mockRejectedValue(serverError()) + else context.store.upsertPoints.mockRejectedValue(serverError()) + await expect(context.writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(false) + expect(context.cache.updateHash).not.toHaveBeenCalled() + expect(context.failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity, state: "exhausted", attempt: 3 }), + ]) }, ) @@ -84,7 +95,9 @@ describe("FileIndexWriter", () => { store.deletePointsByFilePath.mockRejectedValue(serverError()) await expect(writer.remove("/ws/a.ts")).resolves.toBe(false) expect(cache.deleteHash).not.toHaveBeenCalled() - expect(failures.get("a.ts")).toMatchObject({ activity: "delete", state: "exhausted" }) + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "a.ts", activity: "delete", state: "exhausted" }), + ]) }) it("CI-WR-03: each embed attempt goes through the pacer exactly once", async () => { diff --git a/src/services/code-index/__tests__/manager.reliability.spec.ts b/src/services/code-index/__tests__/manager.reliability.spec.ts index f253bd3bbe..01c66f4f74 100644 --- a/src/services/code-index/__tests__/manager.reliability.spec.ts +++ b/src/services/code-index/__tests__/manager.reliability.spec.ts @@ -187,20 +187,23 @@ describe("CodeIndexManager reliability", () => { { base: "Indexed", wait: false, failure: true, expected: "Standby" }, { base: "Indexed", wait: false, failure: false, expected: "Indexed" }, { base: "Standby", wait: false, failure: false, expected: "Standby" }, - ] as const)("$base with wait=$wait failure=$failure is $expected", async ({ base, wait, failure, expected }) => { - const { manager, stateManager } = makeManager() - stateManager.setSystemState(base, "base") - if (failure) withFailure(manager) - const release = wait ? await withWait(manager) : async () => {} - try { - const status = manager.getCurrentStatus() - expect(status.systemStatus).toBe(expected) - expect(status.failedFiles).toHaveLength(failure ? 1 : 0) - expect(status.providerWaits).toHaveLength(wait ? 1 : 0) - } finally { - await release() - } - }) + ] as const)( + "$base with wait=$wait failure=$failure is $expected", + async ({ base, wait, failure, expected }) => { + const { manager, stateManager } = makeManager() + stateManager.setSystemState(base, "base") + if (failure) withFailure(manager) + const release = wait ? await withWait(manager) : async () => {} + try { + const status = manager.getCurrentStatus() + expect(status.systemStatus).toBe(expected) + expect(status.failedFiles).toHaveLength(failure ? 1 : 0) + expect(status.providerWaits).toHaveLength(wait ? 1 : 0) + } finally { + await release() + } + }, + ) }) it("CI-MG-04: a CacheReadError from initialize() ends in Error and keeps existing data", async () => { diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 1263aaef14..6de8bfd1c9 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -353,7 +353,10 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { ;(manager as any)._cacheManager = mockCacheManager // Simulate an initialized manager by setting the required properties - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } + ;(manager as any)._orchestrator = { + stopWatcher: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + } ;(manager as any)._searchService = {} // Verify manager is considered initialized @@ -547,7 +550,11 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { }) // Mock orchestrator and search service to simulate initialized state - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn(), state: "Error" } + ;(manager as any)._orchestrator = { + stopWatcher: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + state: "Error", + } ;(manager as any)._searchService = {} ;(manager as any)._serviceFactory = {} }) @@ -679,7 +686,10 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { // Setup manager with service instances ;(manager as any)._configManager = mockConfigManager ;(manager as any)._serviceFactory = {} - ;(manager as any)._orchestrator = { stopWatcher: vi.fn(), stopIndexing: vi.fn() } + ;(manager as any)._orchestrator = { + stopWatcher: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + } ;(manager as any)._searchService = {} // Spy on console.error diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index c51663e2b7..1e53939f02 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -182,11 +182,13 @@ describe("CodeIndexOrchestrator", () => { try { const context = setup() const limited = Object.assign(new Error("limited"), { status: 429, retryAfterMs: 3_600_000 }) - context.scanner.scanDirectory.mockImplementation(async (_dir, _onError, _onIndexed, _onParsed, signal) => { - const pacer = context.pacers.get("openai:1") - await pacer.run("a.ts", signal, () => Promise.reject(limited)).catch(() => {}) - return emptyScan - }) + context.scanner.scanDirectory.mockImplementation( + async (_dir, _onError, _onIndexed, _onParsed, signal) => { + const pacer = context.pacers.get("openai:1") + await pacer.run("a.ts", signal, () => Promise.reject(limited)).catch(() => {}) + return emptyScan + }, + ) const start = context.orchestrator.startIndexing() await vi.advanceTimersByTimeAsync(0) expect(context.pacers.waitingPaths()).toEqual(["a.ts"]) @@ -219,7 +221,9 @@ describe("CodeIndexOrchestrator", () => { context.failures.record("late.ts", "h", "embed", "late", 1, "exhausted") await Promise.all([stop, start]) - const lateWrites = context.stateManager.setSystemState.mock.calls.slice(writesAtAbort).map(([state]) => state) + const lateWrites = context.stateManager.setSystemState.mock.calls + .slice(writesAtAbort) + .map(([state]) => state) expect(lateWrites).toEqual(["Standby"]) expect(context.stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() }) @@ -238,6 +242,28 @@ describe("CodeIndexOrchestrator", () => { expect(context.orchestrator.state).toBe("Indexed") }) + it("a start after a partial index stops the watcher session and rescans the failed file", async () => { + const context = setup() + context.scanner.scanDirectory + .mockImplementationOnce(async () => { + context.failures.renew("a.ts", "h") + context.failures.record("a.ts", "h", "embed", "server down", 3, "exhausted") + return emptyScan + }) + .mockImplementationOnce(async () => { + context.failures.clear("a.ts", "h") + return emptyScan + }) + await context.orchestrator.startIndexing() + expect(context.orchestrator.state).toBe("Standby") + + await context.orchestrator.startIndexing() + expect(context.scanner.scanDirectory).toHaveBeenCalledTimes(2) + expect(context.fileWatcher.dispose).toHaveBeenCalled() + expect(context.failures.list()).toEqual([]) + expect(context.orchestrator.state).toBe("Indexed") + }) + it("a start while stopping waits for the stop, then starts", async () => { const context = setup() context.scanner.scanDirectory.mockImplementationOnce(waitForAbort) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index a9a629b1df..512b15cba3 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -109,9 +109,9 @@ describe("ProviderPacer (CI-PP-02)", () => { it("rethrows non-rate-limit errors unchanged", async () => { const failure = Object.assign(new Error("server"), { status: 500 }) - await expect(new ProviderPacer("openai:1").run(undefined, undefined, () => Promise.reject(failure))).rejects.toBe( - failure, - ) + await expect( + new ProviderPacer("openai:1").run(undefined, undefined, () => Promise.reject(failure)), + ).rejects.toBe(failure) }) it("keeps waits longer than the 32-bit timer limit", async () => { @@ -134,7 +134,9 @@ describe("ProviderPacerRegistry", () => { const registry = new ProviderPacerRegistry() const changed = vi.fn() registry.onDidChange(changed) - const paused = settle(registry.get("openai:1").run("a.ts", undefined, vi.fn().mockRejectedValueOnce(rateLimit(5000)))) + const paused = settle( + registry.get("openai:1").run("a.ts", undefined, vi.fn().mockRejectedValueOnce(rateLimit(5000))), + ) await vi.advanceTimersByTimeAsync(0) const same = settle(registry.get("openai:1").run("a.ts", undefined, vi.fn().mockResolvedValue("ok"))) const other = vi.fn().mockResolvedValue("other") diff --git a/src/services/code-index/__tests__/service-factory.spec.ts b/src/services/code-index/__tests__/service-factory.spec.ts index bc1d619af2..e01945cd8c 100644 --- a/src/services/code-index/__tests__/service-factory.spec.ts +++ b/src/services/code-index/__tests__/service-factory.spec.ts @@ -12,6 +12,8 @@ import { VercelAiGatewayEmbedder } from "../embedders/vercel-ai-gateway" import { BedrockEmbedder } from "../embedders/bedrock" import { OpenRouterEmbedder } from "../embedders/openrouter" import type { CodeIndexConfig } from "../interfaces/config" +import { FileFailureRegistry } from "../file-failure-registry" +import { ProviderPacerRegistry } from "../provider-pacer" import { clearAllMocks } from "../../../test-utils/reset" @@ -58,6 +60,8 @@ const mockGetDefaultModelId = getDefaultModelId as MockedFunction describe("CodeIndexServiceFactory", () => { + const failures = new FileFailureRegistry() + const pacers = new ProviderPacerRegistry() let factory: CodeIndexServiceFactory let embedderFactory: EmbedderFactory let vectorStoreFactory: VectorStoreFactory @@ -75,7 +79,7 @@ describe("CodeIndexServiceFactory", () => { mockCacheManager = {} - factory = new CodeIndexServiceFactory(mockConfigManager, "/test/workspace", mockCacheManager) + factory = new CodeIndexServiceFactory(mockConfigManager, "/test/workspace", mockCacheManager, failures, pacers) }) describe("createEmbedder", () => { @@ -91,8 +95,8 @@ describe("CodeIndexServiceFactory", () => { modelId: "second-model", }) - const first = embedderFactory.create(mockConfigManager.getConfig()) - const second = embedderFactory.create(mockConfigManager.getConfig()) + const first = embedderFactory.create(mockConfigManager.getConfig(), pacers) + const second = embedderFactory.create(mockConfigManager.getConfig(), pacers) expect(first).not.toBe(second) expect(MockedOpenAiEmbedder).toHaveBeenNthCalledWith(1, { @@ -118,7 +122,9 @@ describe("CodeIndexServiceFactory", () => { it.each(requiredSettings)("rejects missing options for %s (%s.%s)", (provider, _options, _setting, error) => { mockConfigManager.getConfig.mockReturnValue({ embedderProvider: provider }) - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow(`serviceFactory.${error}`) + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( + `serviceFactory.${error}`, + ) }) it.each(requiredSettings)("rejects empty %s setting %s.%s", (provider, options, setting, error) => { @@ -127,13 +133,15 @@ describe("CodeIndexServiceFactory", () => { [options]: { baseUrl: "https://example.com", apiKey: "test-key", [setting]: "" }, }) - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow(`serviceFactory.${error}`) + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( + `serviceFactory.${error}`, + ) }) it.each(["constructor", "toString", "__proto__"])("rejects prototype key %s as a provider", (provider) => { mockConfigManager.getConfig.mockReturnValue({ embedderProvider: provider }) - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.invalidEmbedderType", ) }) @@ -152,7 +160,7 @@ describe("CodeIndexServiceFactory", () => { }) mockConfigManager.getConfig.mockReturnValue(config) - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) expect(embedder).toBeInstanceOf(OpenAiEmbedder) expect(MockedOpenAiEmbedder).toHaveBeenCalledWith({ @@ -176,7 +184,7 @@ describe("CodeIndexServiceFactory", () => { }) mockConfigManager.getConfig.mockReturnValue(config) - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) expect(embedder).toBeInstanceOf(CodeIndexOllamaEmbedder) expect(MockedCodeIndexOllamaEmbedder).toHaveBeenCalledWith({ ...ollamaOptions, ollamaModelId: "new-model" }) @@ -196,7 +204,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedOpenAiEmbedder).toHaveBeenCalledWith({ @@ -218,7 +226,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedCodeIndexOllamaEmbedder).toHaveBeenCalledWith({ @@ -239,7 +247,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedOpenAiEmbedder).toHaveBeenCalledWith({ @@ -260,7 +268,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedCodeIndexOllamaEmbedder).toHaveBeenCalledWith({ @@ -281,7 +289,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.openAiConfigMissing", ) }) @@ -298,7 +306,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.ollamaConfigMissing", ) }) @@ -317,7 +325,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedOpenAICompatibleEmbedder).toHaveBeenCalledWith( @@ -340,7 +348,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedOpenAICompatibleEmbedder).toHaveBeenCalledWith( @@ -363,7 +371,9 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow("validation.baseUrlRequired") + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( + "validation.baseUrlRequired", + ) }) it.each([undefined, ""])("rejects missing or empty OpenAI Compatible API key: %s", (apiKey) => { @@ -379,7 +389,9 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow("validation.apiKeyRequired") + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( + "validation.apiKeyRequired", + ) }) it("should throw error when OpenAI Compatible options are missing", () => { @@ -392,7 +404,9 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow("validation.baseUrlRequired") + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( + "validation.baseUrlRequired", + ) }) it("should create GeminiEmbedder with default model when no modelId specified", () => { @@ -406,7 +420,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedGeminiEmbedder).toHaveBeenCalledWith("test-gemini-api-key", undefined) @@ -424,7 +438,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedGeminiEmbedder).toHaveBeenCalledWith("test-gemini-api-key", "gemini-embedding-001") @@ -443,7 +457,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert - factory passes the original modelId; GeminiEmbedder migrates it internally expect(MockedGeminiEmbedder).toHaveBeenCalledWith("test-gemini-api-key", "text-embedding-004") @@ -460,7 +474,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.geminiConfigMissing", ) }) @@ -474,7 +488,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.geminiConfigMissing", ) }) @@ -492,7 +506,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedMistralEmbedder).toHaveBeenCalledWith("test-mistral-key", testModelId) @@ -510,7 +524,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.mistralConfigMissing", ) }) @@ -527,7 +541,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — modelId omitted so the embedder selects its documented default model. expect(MockedMistralEmbedder).toHaveBeenCalledWith("test-mistral-key", undefined) @@ -546,7 +560,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedVercelAiGatewayEmbedder).toHaveBeenCalledWith("test-vercel-key", testModelId) @@ -564,7 +578,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.vercelAiGatewayConfigMissing", ) }) @@ -581,7 +595,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — modelId omitted so the embedder selects its documented default model. expect(MockedVercelAiGatewayEmbedder).toHaveBeenCalledWith("test-vercel-key", undefined) @@ -601,10 +615,10 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert - expect(MockedBedrockEmbedder).toHaveBeenCalledWith("eu-west-1", "test-profile", testModelId) + expect(MockedBedrockEmbedder).toHaveBeenCalledWith("eu-west-1", "test-profile", testModelId, pacers) }) it("should pass undefined profile to Bedrock embedder when no profile is configured", () => { @@ -620,10 +634,10 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — profile omitted so the embedder falls back to the default credential chain. - expect(MockedBedrockEmbedder).toHaveBeenCalledWith("us-east-1", undefined, testModelId) + expect(MockedBedrockEmbedder).toHaveBeenCalledWith("us-east-1", undefined, testModelId, pacers) }) it("should throw error when Bedrock region is missing", () => { @@ -639,7 +653,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.bedrockConfigMissing", ) }) @@ -657,10 +671,10 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — modelId omitted so the embedder selects its documented default model. - expect(MockedBedrockEmbedder).toHaveBeenCalledWith("us-east-1", "test-profile", undefined) + expect(MockedBedrockEmbedder).toHaveBeenCalledWith("us-east-1", "test-profile", undefined, pacers) }) it("should pass API key, model ID and specific provider to OpenRouter embedder", () => { @@ -677,7 +691,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert expect(MockedOpenRouterEmbedder).toHaveBeenCalledWith( @@ -701,7 +715,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — specificProvider omitted so the embedder routes to its default provider. expect(MockedOpenRouterEmbedder).toHaveBeenCalledWith( @@ -724,7 +738,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.openRouterConfigMissing", ) }) @@ -742,7 +756,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig) // Act - embedderFactory.create(mockConfigManager.getConfig()) + embedderFactory.create(mockConfigManager.getConfig(), pacers) // Assert — modelId omitted so the embedder selects its documented default model. expect(MockedOpenRouterEmbedder).toHaveBeenCalledWith("test-openrouter-key", undefined, undefined, "openai") @@ -757,7 +771,7 @@ describe("CodeIndexServiceFactory", () => { mockConfigManager.getConfig.mockReturnValue(testConfig as any) // Act & Assert - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.invalidEmbedderType", ) }) @@ -768,7 +782,7 @@ describe("CodeIndexServiceFactory", () => { } mockConfigManager.getConfig.mockReturnValue(testConfig as any) - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "Semble provider handles its own embedding. Do not call createEmbedder() for semble", ) }) @@ -793,7 +807,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(3072) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai", testModelId) @@ -802,6 +816,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 3072, "test-key", + pacers, ) }) @@ -818,7 +833,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(768) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("ollama", testModelId) @@ -827,6 +842,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 768, "test-key", + pacers, ) }) @@ -843,7 +859,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(3072) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai-compatible", testModelId) @@ -852,6 +868,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 3072, "test-key", + pacers, ) }) @@ -875,7 +892,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(modelDimension) // This should be used // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai-compatible", testModelId) @@ -884,6 +901,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", modelDimension, // Should use model's built-in dimension, not manual "test-key", + pacers, ) }) @@ -906,7 +924,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(undefined) // Model has no built-in dimension // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai-compatible", testModelId) @@ -915,6 +933,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", manualDimension, // Should use manual dimension as fallback "test-key", + pacers, ) }) @@ -935,7 +954,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(768) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai-compatible", testModelId) @@ -944,6 +963,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 768, "test-key", + pacers, ) }) @@ -965,7 +985,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(undefined) // Act & Assert - expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace")).toThrow( + expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers)).toThrow( "serviceFactory.vectorDimensionNotDeterminedOpenAiCompatible", ) }) @@ -987,7 +1007,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(undefined) // Act & Assert - expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace")).toThrow( + expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers)).toThrow( "serviceFactory.vectorDimensionNotDeterminedOpenAiCompatible", ) }) @@ -1004,7 +1024,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(3072) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("gemini", "gemini-embedding-001") @@ -1013,6 +1033,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 3072, "test-key", + pacers, ) }) @@ -1028,7 +1049,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(3072) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetDefaultModelId).toHaveBeenCalledWith("gemini") @@ -1038,6 +1059,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 3072, "test-key", + pacers, ) }) @@ -1053,7 +1075,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(1536) // Act - vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace") + vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers) // Assert expect(mockGetModelDimension).toHaveBeenCalledWith("openai", "default-model") @@ -1062,6 +1084,7 @@ describe("CodeIndexServiceFactory", () => { "http://localhost:6333", 1536, "test-key", + pacers, ) }) @@ -1077,7 +1100,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(undefined) // Act & Assert - expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace")).toThrow( + expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers)).toThrow( "serviceFactory.vectorDimensionNotDetermined", ) }) @@ -1094,7 +1117,7 @@ describe("CodeIndexServiceFactory", () => { mockGetModelDimension.mockReturnValue(1536) // Act & Assert - expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace")).toThrow( + expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers)).toThrow( "serviceFactory.qdrantUrlMissing", ) }) @@ -1105,7 +1128,7 @@ describe("CodeIndexServiceFactory", () => { } mockConfigManager.getConfig.mockReturnValue(testConfig as any) - expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace")).toThrow( + expect(() => vectorStoreFactory.create(mockConfigManager.getConfig(), "/test/workspace", pacers)).toThrow( "Semble provider handles its own vector storage. Do not call createVectorStore() for semble", ) }) @@ -1136,7 +1159,7 @@ describe("CodeIndexServiceFactory", () => { mockEmbedderInstance.validateConfiguration.mockResolvedValue({ valid: true }) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1163,7 +1186,7 @@ describe("CodeIndexServiceFactory", () => { }) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1189,7 +1212,7 @@ describe("CodeIndexServiceFactory", () => { mockEmbedderInstance.validateConfiguration.mockResolvedValue({ valid: true }) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1214,7 +1237,7 @@ describe("CodeIndexServiceFactory", () => { mockEmbedderInstance.validateConfiguration.mockResolvedValue({ valid: true }) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1237,7 +1260,7 @@ describe("CodeIndexServiceFactory", () => { mockEmbedderInstance.validateConfiguration.mockResolvedValue({ valid: true }) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1262,7 +1285,7 @@ describe("CodeIndexServiceFactory", () => { mockEmbedderInstance.validateConfiguration.mockRejectedValue(networkError) // Act - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) const result = await factory.validateEmbedder(embedder) // Assert @@ -1287,7 +1310,7 @@ describe("CodeIndexServiceFactory", () => { // Act & Assert // This should throw when trying to create the embedder await expect(async () => { - const embedder = embedderFactory.create(mockConfigManager.getConfig()) + const embedder = embedderFactory.create(mockConfigManager.getConfig(), pacers) await factory.validateEmbedder(embedder) }).rejects.toThrow("serviceFactory.openAiConfigMissing") }) @@ -1302,7 +1325,7 @@ describe("CodeIndexServiceFactory", () => { // Act & Assert // This should throw when trying to create the embedder - expect(() => embedderFactory.create(mockConfigManager.getConfig())).toThrow( + expect(() => embedderFactory.create(mockConfigManager.getConfig(), pacers)).toThrow( "serviceFactory.invalidEmbedderType", ) }) diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 388bf27eae..921ee8bc45 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -21,7 +21,6 @@ export const FILE_RETRY_ATTEMPTS = 3 export const FILE_RETRY_BASE_DELAY_MS = 500 export const PACER_MAX_BACKOFF_MS = 300_000 export const PARSING_CONCURRENCY = 10 -export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting /**OpenAI Embedder */ export const MAX_BATCH_TOKENS = 100000 diff --git a/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts index a807b8743a..7cd4bbe076 100644 --- a/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts +++ b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts @@ -18,9 +18,4 @@ describe("EmbedderFactory (CI-EM-03)", () => { new EmbedderFactory().create(config, pacers) expect(BedrockEmbedder).toHaveBeenLastCalledWith("us-east-1", "dev", "amazon.titan-embed-text-v2:0", pacers) }) - - it("still creates a Bedrock embedder when no registry is supplied", () => { - new EmbedderFactory().create(config) - expect(vi.mocked(BedrockEmbedder).mock.lastCall?.[3]).toBeInstanceOf(ProviderPacerRegistry) - }) }) diff --git a/src/services/code-index/embedders/__tests__/ollama.spec.ts b/src/services/code-index/embedders/__tests__/ollama.spec.ts index 40f745b70a..3decb72413 100644 --- a/src/services/code-index/embedders/__tests__/ollama.spec.ts +++ b/src/services/code-index/embedders/__tests__/ollama.spec.ts @@ -326,4 +326,21 @@ describe("CodeIndexOllamaEmbedder", () => { expect(result.error).toBe("Network timeout") }) }) + + describe("CI-EM-01: single attempt with classified errors", () => { + it.each([ + [503, undefined, "server", undefined], + [429, "7", "rate-limit", 7000], + ])("sends one request on HTTP %s and classifies it", async (status, retryAfter, classification, delay) => { + mockFetch.mockResolvedValueOnce( + new Response("busy", { + status, + headers: retryAfter === undefined ? {} : { "retry-after": retryAfter }, + }), + ) + const error = await embedder.createEmbeddings(["text"]).catch((caught: unknown) => caught) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: delay }) + }) + }) }) diff --git a/src/services/code-index/embedders/__tests__/openrouter.spec.ts b/src/services/code-index/embedders/__tests__/openrouter.spec.ts index 088334ad57..14ddfa6005 100644 --- a/src/services/code-index/embedders/__tests__/openrouter.spec.ts +++ b/src/services/code-index/embedders/__tests__/openrouter.spec.ts @@ -379,6 +379,19 @@ describe("OpenRouterEmbedder", () => { }) }) + describe("CI-EM-01: single attempt with classified errors", () => { + it.each([ + [503, {}, "server", undefined], + [429, { "retry-after": "7" }, "rate-limit", 7000], + ])("sends one request on HTTP %s and classifies it", async (status, headers, classification, delay) => { + mockEmbeddingsCreate.mockRejectedValue(Object.assign(new Error("failed"), { status, headers })) + const embedder = new OpenRouterEmbedder(mockApiKey) + const error = await embedder.createEmbeddings(["text"]).catch((caught: unknown) => caught) + expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1) + expect(error).toMatchObject({ classification, statusCode: status, retryAfterMs: delay }) + }) + }) + describe("integration with shared models", () => { it("should work with defined OpenRouter models", () => { const openRouterModels = [ diff --git a/src/services/code-index/embedders/embedder-factory.ts b/src/services/code-index/embedders/embedder-factory.ts index 3dddac4ab6..af6812aa59 100644 --- a/src/services/code-index/embedders/embedder-factory.ts +++ b/src/services/code-index/embedders/embedder-factory.ts @@ -32,7 +32,7 @@ export class EmbedderFactory implements IEmbedderFactory { [providerIdentifiers.openrouter]: new OpenRouterEmbedderFactory(), } - public create(config: CodeIndexConfig, pacers?: ProviderPacerRegistry): IEmbedder { + public create(config: CodeIndexConfig, pacers: ProviderPacerRegistry): IEmbedder { const provider = config.embedderProvider // Semble is a self-contained search binary, not a vector embedder. diff --git a/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts b/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts index 4de2de4b16..5992fe9a46 100644 --- a/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts +++ b/src/services/code-index/embedders/factories/bedrock-embedder-factory.ts @@ -1,11 +1,11 @@ import type { CodeIndexConfig } from "../../interfaces/config" import type { IEmbedderFactory } from "../../interfaces/embedder-factory" -import { ProviderPacerRegistry } from "../../provider-pacer" +import type { ProviderPacerRegistry } from "../../provider-pacer" import { BedrockEmbedder } from "../bedrock" import { requireSetting } from "./require-setting" export class BedrockEmbedderFactory implements IEmbedderFactory { - create({ bedrockOptions, modelId }: CodeIndexConfig, pacers = new ProviderPacerRegistry()): BedrockEmbedder { + create({ bedrockOptions, modelId }: CodeIndexConfig, pacers: ProviderPacerRegistry): BedrockEmbedder { return new BedrockEmbedder( requireSetting(bedrockOptions?.region, "embeddings:serviceFactory.bedrockConfigMissing"), bedrockOptions?.profile, diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index 105a438d7d..b0be6ef8ab 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -256,14 +256,29 @@ export class OpenAICompatibleEmbedder implements IEmbedder { encoding_format: "base64", })) as OpenAIEmbeddingResponse) - const embeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding !== "string") return item.embedding - const buffer = Buffer.from(item.embedding, "base64") - return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)) + // Convert base64 embeddings to float32 arrays + const processedEmbeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding === "string") { + const buffer = Buffer.from(item.embedding, "base64") + + // Create Float32Array view over the buffer + const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + + return { + ...item, + embedding: Array.from(float32Array), + } + } + return item }) + // Replace the original data with processed embeddings + response.data = processedEmbeddings + + const embeddings = response.data.map((item) => item.embedding as number[]) + return { - embeddings, + embeddings: embeddings, usage: { promptTokens: response.usage?.prompt_tokens || 0, totalTokens: response.usage?.total_tokens || 0, diff --git a/src/services/code-index/embedders/openrouter.ts b/src/services/code-index/embedders/openrouter.ts index f63af536f9..b0d484ea8c 100644 --- a/src/services/code-index/embedders/openrouter.ts +++ b/src/services/code-index/embedders/openrouter.ts @@ -182,16 +182,32 @@ export class OpenRouterEmbedder implements IEmbedder { } } + // The SDK types the response as float arrays, but base64 encoding returns strings, so the shapes do not overlap. const response = (await this.embeddingsClient.embeddings.create( requestParams, )) as unknown as OpenRouterEmbeddingResponse - const embeddings = response.data.map((item: EmbeddingItem) => { - if (typeof item.embedding !== "string") return item.embedding - const buffer = Buffer.from(item.embedding, "base64") - return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)) + // Convert base64 embeddings to float32 arrays + const processedEmbeddings = response.data.map((item: EmbeddingItem) => { + if (typeof item.embedding === "string") { + const buffer = Buffer.from(item.embedding, "base64") + + // Create Float32Array view over the buffer + const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + + return { + ...item, + embedding: Array.from(float32Array), + } + } + return item }) + // Replace the original data with processed embeddings + response.data = processedEmbeddings + + const embeddings = response.data.map((item) => item.embedding as number[]) + return { embeddings, usage: { diff --git a/src/services/code-index/file-failure-registry.ts b/src/services/code-index/file-failure-registry.ts index 3f02746a34..3fc66785d6 100644 --- a/src/services/code-index/file-failure-registry.ts +++ b/src/services/code-index/file-failure-registry.ts @@ -30,7 +30,6 @@ export class FileFailureRegistry { private readonly entries = new Map() private readonly revisions = new Map() private readonly changes = new Set<() => void>() - private readonly records = new Set<(failure: FailedFile) => void>() readonly onDidChange: Event = (listener, thisArgument, disposables) => { const callback = () => listener.call(thisArgument, undefined) @@ -44,45 +43,7 @@ export class FileFailureRegistry { return disposable } - readonly onDidRecord: Event = (listener, thisArgument, disposables) => { - const callback = (failure: FailedFile) => listener.call(thisArgument, { ...failure }) - this.records.add(callback) - const disposable: Disposable = { - dispose: () => { - this.records.delete(callback) - }, - } - disposables?.push(disposable) - return disposable - } - - getRevision(path: string): string | undefined { - return this.revisions.get(path)?.revision - } - - get(path: string): FailedFile | undefined { - const failure = this.entries.get(path) - return failure && { ...failure } - } - - has(path: string): boolean { - return this.entries.has(path) - } - - hasRevisionFailure(path: string, revision: string): boolean { - const current = this.revisions.get(path) - return current?.revision === revision && current.failures.size > 0 - } - - assertRetryable(path: string, revision: string): void { - const current = this.revisions.get(path) - if (current?.revision !== revision) return - for (const failure of current.failures.values()) { - if (failure.state === "exhausted") throw Object.assign(new Error(failure.reason), failure) - } - } - - renew(path: string, revision: string): AbortSignal { + renew(path: string, revision: string): void { let current = this.revisions.get(path) if (!current || current.revision !== revision) { current?.controller.abort() @@ -94,7 +55,6 @@ export class FileFailureRegistry { this.changed() } } - return current.controller.signal } isCurrent(path: string, revision: string): boolean { @@ -102,12 +62,6 @@ export class FileFailureRegistry { return current?.revision === revision && !current.controller.signal.aborted } - getFailure(path: string, revision: string, activity: FileFailureActivity): FailedFile | undefined { - const current = this.revisions.get(path) - const failure = current?.revision === revision ? current.failures.get(activity) : undefined - return failure && { ...failure } - } - getRetryBudget(path: string, revision: string, activity: FileFailureActivity): RetryBudget { const current = this.revisions.get(path) if (current?.revision !== revision) return { retriesUsed: 0 } @@ -119,10 +73,6 @@ export class FileFailureRegistry { return budget } - isExhausted(path: string, revision: string, activity: FileFailureActivity): boolean { - return this.getFailure(path, revision, activity)?.state === "exhausted" - } - record( path: string, revision: string, @@ -137,7 +87,6 @@ export class FileFailureRegistry { const failure = { path, revision, activity, reason, attempt, state, classification, ...delay } this.revisions.get(path)!.failures.set(activity, failure) this.entries.set(path, failure) - for (const listener of this.records) listener(failure) this.changed() } diff --git a/src/services/code-index/file-index-writer.ts b/src/services/code-index/file-index-writer.ts index c2353b1745..eb3cd22cd8 100644 --- a/src/services/code-index/file-index-writer.ts +++ b/src/services/code-index/file-index-writer.ts @@ -12,7 +12,7 @@ export class FileIndexWriter { constructor( private readonly embedder: IEmbedder, private readonly store: IVectorStore, - private readonly cache: Pick, + private readonly cache: Pick, private readonly failures: FileFailureRegistry, private readonly pacers: ProviderPacerRegistry, private readonly workspacePath: string, @@ -25,9 +25,11 @@ export class FileIndexWriter { this.failures.renew(relativePath, hash) const selected = blocks.filter((block) => block.content.trim()) try { - await this.attempt(relativePath, hash, "delete", this.store.rateLimitScope, signal, () => - this.store.deletePointsByFilePath(filePath), - ) + if (this.cache.getHash(filePath) !== undefined) { + await this.attempt(relativePath, hash, "delete", this.store.rateLimitScope, signal, () => + this.store.deletePointsByFilePath(filePath), + ) + } for (let offset = 0; offset < selected.length; offset += this.batchSize) { const batch = selected.slice(offset, offset + this.batchSize) const { embeddings } = await this.attempt( @@ -99,11 +101,11 @@ export class FileIndexWriter { }) } - // Same id as the merge-base watcher so scanner and watcher writes of one block replace each other. + // Segment hash, not start line: segments split from one long line share a start line. private toPoint(block: CodeBlock, vector: number[]): PointStruct { const absolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) return { - id: uuidv5(`${absolutePath}:${block.start_line}`, QDRANT_CODE_BLOCK_NAMESPACE), + id: uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE), vector, payload: { filePath: generateRelativeFilePath(absolutePath, this.workspacePath), diff --git a/src/services/code-index/interfaces/embedder-factory.ts b/src/services/code-index/interfaces/embedder-factory.ts index 36aeca9501..96fbf84bf6 100644 --- a/src/services/code-index/interfaces/embedder-factory.ts +++ b/src/services/code-index/interfaces/embedder-factory.ts @@ -4,5 +4,5 @@ import type { ProviderPacerRegistry } from "../provider-pacer" /** Creates a vector embedder, validating the provider's required configuration. */ export interface IEmbedderFactory { - create(config: CodeIndexConfig, pacers?: ProviderPacerRegistry): IEmbedder + create(config: CodeIndexConfig, pacers: ProviderPacerRegistry): IEmbedder } diff --git a/src/services/code-index/interfaces/vector-store-factory.ts b/src/services/code-index/interfaces/vector-store-factory.ts index 57fde591f6..033a970682 100644 --- a/src/services/code-index/interfaces/vector-store-factory.ts +++ b/src/services/code-index/interfaces/vector-store-factory.ts @@ -3,5 +3,5 @@ import type { IVectorStore } from "./index" import type { ProviderPacerRegistry } from "../provider-pacer" export interface IVectorStoreFactory { - create(config: CodeIndexConfig, workspacePath: string, pacers?: ProviderPacerRegistry): IVectorStore + create(config: CodeIndexConfig, workspacePath: string, pacers: ProviderPacerRegistry): IVectorStore } diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 585364157b..0151fedb51 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -6,8 +6,8 @@ import { IFileWatcher, IVectorStore, BatchProcessingSummary } from "./interfaces import { DirectoryScanner } from "./processors" import { CacheManager } from "./cache-manager" import { CodeIndexScanExecutor } from "./code-index-scan-executor" -import { FileFailureRegistry } from "./file-failure-registry" -import { ProviderPacerRegistry } from "./provider-pacer" +import type { FileFailureRegistry } from "./file-failure-registry" +import type { ProviderPacerRegistry } from "./provider-pacer" import { formatProviderWaitMessage } from "./shared/provider-wait-message" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" @@ -21,7 +21,6 @@ import { t } from "../../i18n" */ export class CodeIndexOrchestrator { private _fileWatcherSubscriptions: vscode.Disposable[] = [] - private _isProcessing: boolean = false private _abortController: AbortController | null = null private scanPromise?: Promise private stopPromise?: Promise @@ -38,8 +37,8 @@ export class CodeIndexOrchestrator { private readonly vectorStore: IVectorStore, scanner: DirectoryScanner, private readonly fileWatcher: IFileWatcher, - private readonly failures: FileFailureRegistry = new FileFailureRegistry(), - private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + private readonly failures: FileFailureRegistry, + private readonly pacers: ProviderPacerRegistry, ) { this.scanExecutor = new CodeIndexScanExecutor(workspacePath, scanner, vectorStore, stateManager) } @@ -116,7 +115,8 @@ export class CodeIndexOrchestrator { /** * Starts one indexing session (scan, then watcher). A repeated call while a scan runs returns the running - * scan; a call while stopping waits for the stop to finish first. + * scan; a call while stopping waits for the stop to finish first; a call during the watcher phase stops + * that session and rescans, so an explicit retry reaches failed files. */ public async startIndexing(): Promise { if (this.stopPromise) await this.stopPromise @@ -135,11 +135,10 @@ export class CodeIndexOrchestrator { } if (this._abortController && !this._abortController.signal.aborted) { - console.warn("[CodeIndexOrchestrator] Start rejected: indexing session already active.") - return + await this.stopIndexing() + if (this.scanPromise) return this.scanPromise } - this._isProcessing = true this.scanDone = false this._abortController = new AbortController() const signal = this._abortController.signal @@ -219,8 +218,6 @@ export class CodeIndexOrchestrator { }), ) this.disposeWatcher() - } finally { - this._isProcessing = false } } @@ -278,34 +275,28 @@ export class CodeIndexOrchestrator { * and resetting the cache file. */ public async clearIndexData(): Promise { - this._isProcessing = true + await this.stopIndexing() try { - await this.stopIndexing() - - try { - if (this.configManager.isFeatureConfigured) { - await this.vectorStore.deleteCollection() - } else { - console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") - } - } catch (error: any) { - console.error("[CodeIndexOrchestrator] Failed to clear vector collection:", error) - TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - location: "clearIndexData", - }) - this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) + if (this.configManager.isFeatureConfigured) { + await this.vectorStore.deleteCollection() + } else { + console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") } + } catch (error: any) { + console.error("[CodeIndexOrchestrator] Failed to clear vector collection:", error) + TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + location: "clearIndexData", + }) + this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) + } - await this.cacheManager.clearCacheFile() + await this.cacheManager.clearCacheFile() - if (this.stateManager.state !== "Error") { - this.stateManager.setSystemState("Standby", "Index data cleared successfully.") - } - } finally { - this._isProcessing = false + if (this.stateManager.state !== "Error") { + this.stateManager.setSystemState("Standby", "Index data cleared successfully.") } } @@ -313,10 +304,6 @@ export class CodeIndexOrchestrator { return this.scanPromise !== undefined } - public get isStopping(): boolean { - return this.stopPromise !== undefined - } - /** * Gets the current state of the indexing system. */ diff --git a/src/services/code-index/processors/__tests__/file-preparation.spec.ts b/src/services/code-index/processors/__tests__/file-preparation.spec.ts index 96c585bb05..6ba76c3194 100644 --- a/src/services/code-index/processors/__tests__/file-preparation.spec.ts +++ b/src/services/code-index/processors/__tests__/file-preparation.spec.ts @@ -273,7 +273,8 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo ["parser load", "parse"], ] as const)("CI-FP-02: a %s error is a local error with activity %s", async (stage, activity) => { const dependencies = setup() - const error = stage === "parser load" ? new ParserLoadError(".ts", new Error("wasm")) : new Error(`${stage} failed`) + const error = + stage === "parser load" ? new ParserLoadError(".ts", new Error("wasm")) : new Error(`${stage} failed`) const fail = () => { throw error } diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index f2968fb81b..c05bf2dcc6 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -5,6 +5,7 @@ import * as vscode from "vscode" import { FileWatcher } from "../file-watcher" import { FilePreparation } from "../file-preparation" import { FileFailureRegistry } from "../../file-failure-registry" +import { ProviderPacerRegistry } from "../../provider-pacer" import { clearAllMocks } from "../../../../test-utils/reset" @@ -175,6 +176,10 @@ describe("FileWatcher", () => { mockEmbedder, mockVectorStore, mockIgnoreInstance, + undefined, + undefined, + new FileFailureRegistry(), + new ProviderPacerRegistry(), ) }) @@ -376,6 +381,7 @@ describe("FileWatcher", () => { undefined, undefined, failures, + new ProviderPacerRegistry(), ) return failures } @@ -409,7 +415,9 @@ describe("FileWatcher", () => { error: new Error("bad"), }) await runChange() - expect(failures.list()).toEqual([expect.objectContaining({ path: "src/file.ts", activity, reason: "bad" })]) + expect(failures.list()).toEqual([ + expect.objectContaining({ path: "src/file.ts", activity, reason: "bad" }), + ]) expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() }, ) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 89be4e7495..864025d732 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -166,7 +166,10 @@ describe("CodeParser", () => { }) it.each([ - ["a rejected load", () => vi.mocked(loadRequiredLanguageParsers).mockRejectedValueOnce(new Error("Load failed"))], + [ + "a rejected load", + () => vi.mocked(loadRequiredLanguageParsers).mockRejectedValueOnce(new Error("Load failed")), + ], ["no parser for the extension", () => vi.mocked(loadRequiredLanguageParsers).mockResolvedValueOnce({})], ])("CI-PA-01: %s throws ParserLoadError and the next call reloads", async (_label, failLoad) => { failLoad() diff --git a/src/services/code-index/processors/__tests__/processor-factories.spec.ts b/src/services/code-index/processors/__tests__/processor-factories.spec.ts index a3aaf211b9..776dd7e483 100644 --- a/src/services/code-index/processors/__tests__/processor-factories.spec.ts +++ b/src/services/code-index/processors/__tests__/processor-factories.spec.ts @@ -38,16 +38,17 @@ describe("processor factories", () => { function dependencies() { const context = makeExtensionContext() + const pacers = new ProviderPacerRegistry() return { context, workspacePath: "/workspace", cacheManager: new CacheManager(context, "/workspace"), embedder: new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), - vectorStore: new QdrantVectorStore("/workspace", "http://localhost:6333", 512), + vectorStore: new QdrantVectorStore("/workspace", "http://localhost:6333", 512, undefined, pacers), parser: codeParser, ignoreInstance: ignore(), failureRegistry: new FileFailureRegistry(), - pacers: new ProviderPacerRegistry(), + pacers, } } @@ -189,7 +190,13 @@ describe("processor factories", () => { Object.defineProperty(configManager, "isFeatureConfigured", { value: false }) const createEmbedder = vi.spyOn(EmbedderFactory.prototype, "create") const createVectorStore = vi.spyOn(VectorStoreFactory.prototype, "create") - const factory = new CodeIndexServiceFactory(configManager, options.workspacePath, options.cacheManager) + const factory = new CodeIndexServiceFactory( + configManager, + options.workspacePath, + options.cacheManager, + options.failureRegistry, + options.pacers, + ) expect(() => factory.createServices(options.context, options.cacheManager, options.ignoreInstance)).toThrow( "serviceFactory.codeIndexingNotConfigured", diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 6275c031d6..e901322c9b 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -3,6 +3,7 @@ import { DirectoryScanner } from "../scanner" import { ParserLoadError } from "../parser" import { FileFailureRegistry } from "../../file-failure-registry" +import { ProviderPacerRegistry } from "../../provider-pacer" import { stat } from "fs/promises" import * as vscode from "vscode" @@ -115,6 +116,9 @@ describe("DirectoryScanner", () => { mockCodeParser, mockCacheManager, mockIgnoreInstance, + undefined, + new FileFailureRegistry(), + new ProviderPacerRegistry(), ) // Mock default implementations - create proper Stats object @@ -181,6 +185,9 @@ describe("DirectoryScanner", () => { mockCodeParser, mockCacheManager, mockIgnoreInstance, + undefined, + new FileFailureRegistry(), + new ProviderPacerRegistry(), ) const { listFiles } = await import("../../../glob/list-files") @@ -231,6 +238,8 @@ describe("DirectoryScanner", () => { mockCacheManager, mockIgnoreInstance, 1, + new FileFailureRegistry(), + new ProviderPacerRegistry(), ) const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false]) @@ -302,6 +311,9 @@ describe("DirectoryScanner", () => { mockCodeParser, mockCacheManager, mockIgnoreInstance, + undefined, + new FileFailureRegistry(), + new ProviderPacerRegistry(), ) const { listFiles } = await import("../../../glob/list-files") @@ -488,6 +500,7 @@ describe("DirectoryScanner", () => { mockIgnoreInstance, undefined, failures, + new ProviderPacerRegistry(), ) const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/gone.js"], false]) @@ -515,6 +528,7 @@ describe("DirectoryScanner", () => { mockIgnoreInstance, undefined, failures, + new ProviderPacerRegistry(), ) const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/denied.js"], false]) @@ -529,38 +543,42 @@ describe("DirectoryScanner", () => { it.each([ ["parser load", new ParserLoadError(".js", new Error("wasm"))], ["parse", new Error("syntax")], - ])("CI-SC-02: a %s failure is recorded, the hash is not stored and other files continue", async (_label, error) => { - const failures = new FileFailureRegistry() - const parseScanner = new DirectoryScanner( - mockEmbedder, - mockVectorStore, - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - undefined, - failures, - ) - const block = { - file_path: "test/file2.js", - content: "ok", - start_line: 1, - end_line: 1, - identifier: null, - type: "function", - fileHash: "h", - segmentHash: "s", - } - mockCodeParser.parseFile.mockImplementation(async (filePath: string) => { - if (filePath === "test/file1.js") throw error - return [block] - }) + ])( + "CI-SC-02: a %s failure is recorded, the hash is not stored and other files continue", + async (_label, error) => { + const failures = new FileFailureRegistry() + const parseScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + const block = { + file_path: "test/file2.js", + content: "ok", + start_line: 1, + end_line: 1, + identifier: null, + type: "function", + fileHash: "h", + segmentHash: "s", + } + mockCodeParser.parseFile.mockImplementation(async (filePath: string) => { + if (filePath === "test/file1.js") throw error + return [block] + }) - await parseScanner.scanDirectory("/test") + await parseScanner.scanDirectory("/test") - expect(failures.list()).toEqual([expect.objectContaining({ path: "test/file1.js", activity: "parse" })]) - expect(mockCacheManager.updateHash).not.toHaveBeenCalledWith("test/file1.js", expect.anything()) - expect(mockCacheManager.updateHash).toHaveBeenCalledWith("test/file2.js", expect.any(String)) - }) + expect(failures.list()).toEqual([expect.objectContaining({ path: "test/file1.js", activity: "parse" })]) + expect(mockCacheManager.updateHash).not.toHaveBeenCalledWith("test/file1.js", expect.anything()) + expect(mockCacheManager.updateHash).toHaveBeenCalledWith("test/file2.js", expect.any(String)) + }, + ) it("should not process deleted files when signal is aborted", async () => { const { listFiles } = await import("../../../glob/list-files") diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index ad96dd824f..254085ab6a 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -8,8 +8,8 @@ import { codeParser } from "./parser" import { FilePreparation } from "./file-preparation" import { CacheManager } from "../cache-manager" import { Package } from "../../../shared/package" -import { FileFailureRegistry, UNKNOWN_REVISION } from "../file-failure-registry" -import { ProviderPacerRegistry } from "../provider-pacer" +import { type FileFailureRegistry, UNKNOWN_REVISION } from "../file-failure-registry" +import type { ProviderPacerRegistry } from "../provider-pacer" import { FileIndexWriter } from "../file-index-writer" import { generateRelativeFilePath } from "../shared/get-relative-path" @@ -64,13 +64,13 @@ export class FileWatcher implements IFileWatcher { private workspacePath: string, _context: vscode.ExtensionContext, private readonly cacheManager: CacheManager, - private embedder?: IEmbedder, - private vectorStore?: IVectorStore, - ignoreInstance?: Ignore, - ignoreController?: RooIgnoreController, - batchSegmentThreshold?: number, - private readonly failures: FileFailureRegistry = new FileFailureRegistry(), - private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + private embedder: IEmbedder | undefined, + private vectorStore: IVectorStore | undefined, + ignoreInstance: Ignore | undefined, + ignoreController: RooIgnoreController | undefined, + batchSegmentThreshold: number | undefined, + private readonly failures: FileFailureRegistry, + private readonly pacers: ProviderPacerRegistry, ) { this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) if (ignoreInstance) { @@ -112,8 +112,8 @@ export class FileWatcher implements IFileWatcher { } /** - * Initializes the file watcher; `signal` is the indexing session's abort signal. - */ + * Initializes the file watcher; `signal` is the indexing session's abort signal. + */ async initialize(signal?: AbortSignal): Promise { this.signal = signal // Create file watcher @@ -231,7 +231,11 @@ export class FileWatcher implements IFileWatcher { const batchResults: FileProcessingResult[] = [] const totalFilesInBatch = eventsToProcess.size let processedCountInBatch = 0 - this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: totalFilesInBatch, currentFile: undefined }) + this._onBatchProgressUpdate.fire({ + processedInBatch: 0, + totalInBatch: totalFilesInBatch, + currentFile: undefined, + }) const events = [...eventsToProcess.values()] for (let offset = 0; offset < events.length; offset += this.FILE_PROCESSING_CONCURRENCY_LIMIT) { @@ -241,7 +245,9 @@ export class FileWatcher implements IFileWatcher { chunk.map(async (event) => { const filePath = event.uri.fsPath try { - return event.type === "delete" ? await this.removeFile(filePath) : await this.indexFile(filePath) + return event.type === "delete" + ? await this.removeFile(filePath) + : await this.indexFile(filePath) } catch (error) { return { path: filePath, status: "error", error: error as Error } satisfies FileProcessingResult } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 9e45a003f8..c6adc50342 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -7,7 +7,7 @@ import { generateRelativeFilePath } from "../shared/get-relative-path" import { getWorkspacePathForContext } from "../../../utils/path" import { scannerExtensions } from "../shared/supported-extensions" import * as vscode from "vscode" -import { ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" +import { CodeBlock, ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" import { createHash } from "crypto" import pLimit from "p-limit" import { CacheManager } from "../cache-manager" @@ -24,8 +24,8 @@ import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" -import { FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" -import { ProviderPacerRegistry } from "../provider-pacer" +import { type FileFailureRegistry, UNKNOWN_REVISION, type FileFailureActivity } from "../file-failure-registry" +import type { ProviderPacerRegistry } from "../provider-pacer" import { FileIndexWriter } from "../file-index-writer" import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" @@ -38,9 +38,9 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly codeParser: ICodeParser, private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, - batchSegmentThreshold?: number, - private readonly failures: FileFailureRegistry = new FileFailureRegistry(), - private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + batchSegmentThreshold: number | undefined, + private readonly failures: FileFailureRegistry, + private readonly pacers: ProviderPacerRegistry, ) { // Get the configurable batch size from VSCode settings, fallback to default // If not provided in constructor, try to get from VSCode settings @@ -119,7 +119,12 @@ export class DirectoryScanner implements IDirectoryScanner { const writeLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) const writes: Promise[] = [] - const recordLocalFailure = (filePath: string, revision: string, activity: FileFailureActivity, error: unknown) => { + const recordLocalFailure = ( + filePath: string, + revision: string, + activity: FileFailureActivity, + error: unknown, + ) => { if (signal?.aborted) return const relativePath = generateRelativeFilePath(filePath, scanWorkspace) const reason = error instanceof Error ? error.message : String(error) @@ -157,7 +162,7 @@ export class DirectoryScanner implements IDirectoryScanner { return } - let blocks + let blocks: CodeBlock[] try { blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash }) } catch (error) { @@ -226,7 +231,8 @@ export class DirectoryScanner implements IDirectoryScanner { error instanceof Error ? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`) : new Error( - t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) + ` (Workspace: ${scanWorkspace})`, + t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) + + ` (Workspace: ${scanWorkspace})`, ), ) } diff --git a/src/services/code-index/search-service.ts b/src/services/code-index/search-service.ts index 2840f1cd97..30339a81d9 100644 --- a/src/services/code-index/search-service.ts +++ b/src/services/code-index/search-service.ts @@ -1,11 +1,11 @@ import * as path from "path" +import { VectorStoreSearchResult } from "./interfaces" +import { IEmbedder } from "./interfaces/embedder" +import { IVectorStore } from "./interfaces/vector-store" +import { CodeIndexConfigManager } from "./config-manager" +import { CodeIndexStateManager } from "./state-manager" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" -import type { VectorStoreSearchResult } from "./interfaces/vector-store" -import type { IEmbedder } from "./interfaces/embedder" -import type { IVectorStore } from "./interfaces/vector-store" -import type { CodeIndexConfigManager } from "./config-manager" -import type { CodeIndexStateManager } from "./state-manager" import type { FileFailureRegistry } from "./file-failure-registry" import type { ProviderPacerRegistry } from "./provider-pacer" import { retryWithPolicy } from "./shared/retry-policy" @@ -35,6 +35,13 @@ export class CodeIndexSearchService { private readonly providerPacers: Pick, ) {} + /** + * Searches the code index for relevant content. + * @param query The search query + * @param directoryPrefix Optional directory path to filter results by + * @returns Search results with incomplete-index details + * @throws Error if the service is not properly configured or ready + */ public async searchIndex(query: string, directoryPrefix?: string): Promise { if (!this.configManager.isFeatureEnabled || !this.configManager.isFeatureConfigured) { throw new Error("Code index feature is disabled or not configured.") @@ -67,11 +74,13 @@ export class CodeIndexSearchService { throw new Error("Failed to generate embedding for query.") } + // Handle directory prefix let normalizedPrefix: string | undefined = undefined if (directoryPrefix) { normalizedPrefix = path.normalize(directoryPrefix) } + // Perform search const results = await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) const failedPaths = [...new Set(this.failures.list().map((failure) => failure.path))] const waitingPaths = this.providerPacers.waitingPaths() @@ -84,6 +93,8 @@ export class CodeIndexSearchService { } } catch (error) { console.error("[CodeIndexSearchService] Error during search:", error) + + // Capture telemetry for the error TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { error: (error as Error).message, stack: (error as Error).stack, diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index b54d47add6..00e5e8ce2e 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -14,8 +14,8 @@ import { FileWatcherFactory } from "./processors/file-watcher-factory" import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" -import { FileFailureRegistry } from "./file-failure-registry" -import { ProviderPacerRegistry } from "./provider-pacer" +import type { FileFailureRegistry } from "./file-failure-registry" +import type { ProviderPacerRegistry } from "./provider-pacer" /** * Factory class responsible for creating and configuring code indexing service dependencies. @@ -34,8 +34,8 @@ export class CodeIndexServiceFactory { private readonly configManager: CodeIndexConfigManager, private readonly workspacePath: string, private readonly cacheManager: CacheManager, - private readonly failureRegistry: FileFailureRegistry = new FileFailureRegistry(), - private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + private readonly failureRegistry: FileFailureRegistry, + private readonly pacers: ProviderPacerRegistry, ) {} /** diff --git a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts index 1706100a3c..7183c883d0 100644 --- a/src/services/code-index/shared/__tests__/indexing-errors.spec.ts +++ b/src/services/code-index/shared/__tests__/indexing-errors.spec.ts @@ -176,7 +176,7 @@ describe("parseRetryAfterMs", () => { }) it.each(["2", "invalid", undefined])( - "T33 discovers header delay once across formatting and retry wrappers: %s", + "discovers header delay once across formatting and retry wrappers: %s", (value) => { const get = vi.fn((name: string) => (name === "retry-after" ? value : undefined)) const original = { status: 429, headers: { get } } @@ -192,7 +192,7 @@ it.each(["2", "invalid", undefined])( }, ) -it("T33 explicit delay wins without reading headers", () => { +it("explicit delay wins without reading headers", () => { const get = vi.fn() const error = new VectorStoreError("limited", { status: 429, headers: { get } }, { retryAfterMs: 7000 }) expect(error.retryAfterMs).toBe(7000) diff --git a/src/services/code-index/shared/__tests__/retry-policy.spec.ts b/src/services/code-index/shared/__tests__/retry-policy.spec.ts index 03377a6e33..a206d5d898 100644 --- a/src/services/code-index/shared/__tests__/retry-policy.spec.ts +++ b/src/services/code-index/shared/__tests__/retry-policy.spec.ts @@ -61,22 +61,27 @@ it.each([undefined, 100, 2000, 60_000, 120_000, NaN, -1])( }, ) -it.each([503, { code: "ECONNRESET" }])("CI-RP-01/02: at most 3 requests, waits 500 ms then 1 s: %s", async (failure) => { - const cause = typeof failure === "number" ? { status: failure } : failure - const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", cause)) - const failures: RetryFailure[] = [] - const pending = retryWithPolicy(request, { onFailure: (failure) => failures.push(failure) }).catch((error) => error) - for (const [index, delay] of [500, 1000].entries()) { - await vi.advanceTimersByTimeAsync(0) - expect(request).toHaveBeenCalledTimes(index + 1) - expect(failures[index]).toMatchObject({ delayMs: delay, delaySource: "default", willRetry: true }) - await vi.advanceTimersByTimeAsync(delay - 1) - expect(request).toHaveBeenCalledTimes(index + 1) - await vi.advanceTimersByTimeAsync(1) - } - expect(await pending).toMatchObject({ attempts: 3 }) - expect(request).toHaveBeenCalledTimes(3) -}) +it.each([503, { code: "ECONNRESET" }])( + "CI-RP-01/02: at most 3 requests, waits 500 ms then 1 s: %s", + async (failure) => { + const cause = typeof failure === "number" ? { status: failure } : failure + const request = vi.fn().mockRejectedValue(new EmbeddingError("failed", cause)) + const failures: RetryFailure[] = [] + const pending = retryWithPolicy(request, { onFailure: (failure) => failures.push(failure) }).catch( + (error) => error, + ) + for (const [index, delay] of [500, 1000].entries()) { + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(index + 1) + expect(failures[index]).toMatchObject({ delayMs: delay, delaySource: "default", willRetry: true }) + await vi.advanceTimersByTimeAsync(delay - 1) + expect(request).toHaveBeenCalledTimes(index + 1) + await vi.advanceTimersByTimeAsync(1) + } + expect(await pending).toMatchObject({ attempts: 3 }) + expect(request).toHaveBeenCalledTimes(3) + }, +) it("CI-RP-02: abort ends a wait at once without another request", async () => { const controller = new AbortController() @@ -122,7 +127,7 @@ it("shares two retries across successful batches without charging their first ca expect(request).toHaveBeenCalledTimes(6) }) -it("T33 wraps a frozen error without changing its metadata", async () => { +it("wraps a frozen error without changing its metadata", async () => { const original = Object.freeze(new EmbeddingError("unavailable", { status: 503 }, { retryAfterMs: 2000 })) const properties = Object.getOwnPropertyDescriptors(original) const request = vi.fn().mockRejectedValue(original) @@ -143,7 +148,7 @@ it("T33 wraps a frozen error without changing its metadata", async () => { expect(request).toHaveBeenCalledTimes(3) }) -it("T33 keeps independent metadata when one error is reused across operations", async () => { +it("keeps independent metadata when one error is reused across operations", async () => { const original = Object.assign(new Error("unavailable"), { status: 503 }) const properties = Object.getOwnPropertyDescriptors(original) const request = vi.fn().mockRejectedValue(original) @@ -169,7 +174,7 @@ it.each([ new DOMException("cancelled", "AbortError"), new APIUserAbortError(), Object.freeze(Object.assign(new Error("cancelled"), { name: "APIUserAbortError" })), -])("T33 preserves live-signal cancellation identity: %s", async (original) => { +])("preserves live-signal cancellation identity: %s", async (original) => { const properties = Object.getOwnPropertyDescriptors(original) const request = vi.fn().mockRejectedValue(original) const onFailure = vi.fn() @@ -183,7 +188,7 @@ it.each([ expect(request).toHaveBeenCalledOnce() }) -it("T33 rethrows signal.reason rather than an operation rejection after abort", async () => { +it("rethrows signal.reason rather than an operation rejection after abort", async () => { const controller = new AbortController() const reason = Object.freeze(new DOMException("stopped", "AbortError")) const rejection = new Error("operation rejected") diff --git a/src/services/code-index/shared/retry-policy.ts b/src/services/code-index/shared/retry-policy.ts index 490ef34ff6..bf9b048dac 100644 --- a/src/services/code-index/shared/retry-policy.ts +++ b/src/services/code-index/shared/retry-policy.ts @@ -14,7 +14,7 @@ export interface RetryFailure { classification: IndexingErrorClassification willRetry: boolean delayMs: number - delaySource: "default" | "provider" + delaySource: "default" } export interface RetryPolicyOptions { diff --git a/src/services/code-index/shared/validation-helpers.ts b/src/services/code-index/shared/validation-helpers.ts index c1ce54ed72..57a82c0238 100644 --- a/src/services/code-index/shared/validation-helpers.ts +++ b/src/services/code-index/shared/validation-helpers.ts @@ -1,7 +1,6 @@ import { t } from "../../../i18n" import { serializeError } from "serialize-error" import { EmbeddingError, isQuotaError } from "./indexing-errors" -import { FILE_RETRY_ATTEMPTS } from "../constants" /** * Sanitizes error messages by removing sensitive information like file paths and URLs @@ -215,7 +214,7 @@ export function toEmbeddingError(error: unknown): EmbeddingError { if (error instanceof EmbeddingError) return error if (isQuotaError(error)) return new EmbeddingError(t("embeddings:quotaExceeded"), error, { classification: "permanent" }) - return new EmbeddingError(formatEmbeddingError(error, FILE_RETRY_ATTEMPTS).message, error) + return new EmbeddingError(formatEmbeddingError(error, 1).message, error) } export function formatEmbeddingError(error: any, maxRetries: number): Error { diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts index c5636dcf42..0ab5c7a939 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.rate-limit.spec.ts @@ -2,6 +2,9 @@ import path from "node:path" import { QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" import { QdrantVectorStore } from "../qdrant-client" import { VectorStoreError } from "../../shared/indexing-errors" +import { ProviderPacerRegistry } from "../../provider-pacer" + +const pacers = new ProviderPacerRegistry() vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) @@ -42,7 +45,13 @@ describe("Qdrant Retry-After (CI-QD-02)", () => { ["120", 120_000], ])("reads the whole %s-second header, not its first character", async (header, expected) => { respondToUpsert(header) - const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const store = new QdrantVectorStore( + path.resolve("/workspace"), + "http://example.test:6333", + 2, + undefined, + pacers, + ) const error = await store.upsertPoints([point]).catch((caught: unknown) => caught) expect(error).toBeInstanceOf(VectorStoreError) expect(error).toMatchObject({ classification: "rate-limit", statusCode: 429, retryAfterMs: expected }) @@ -51,16 +60,22 @@ describe("Qdrant Retry-After (CI-QD-02)", () => { it("classifies a 429 without a header as a rate limit with no delay", async () => { respondToUpsert(undefined) - const store = new QdrantVectorStore(path.resolve("/workspace"), "http://example.test:6333", 2) + const store = new QdrantVectorStore( + path.resolve("/workspace"), + "http://example.test:6333", + 2, + undefined, + pacers, + ) const error = await store.upsertPoints([point]).catch((caught: unknown) => caught) expect(error).toMatchObject({ classification: "rate-limit" }) expect((error as VectorStoreError).retryAfterMs).toBeUndefined() }) it("scopes Qdrant by normalized URL, independently of workspace and key", () => { - const first = new QdrantVectorStore("/first", "http://EXAMPLE.test:6333/", 2, "first-key") - const same = new QdrantVectorStore("/second", "example.test:6333", 2, "second-key") - const other = new QdrantVectorStore("/first", "http://example.test:6334", 2) + const first = new QdrantVectorStore("/first", "http://EXAMPLE.test:6333/", 2, "first-key", pacers) + const same = new QdrantVectorStore("/second", "example.test:6333", 2, "second-key", pacers) + const other = new QdrantVectorStore("/first", "http://example.test:6334", 2, undefined, pacers) expect(first.rateLimitScope).toBe(same.rateLimitScope) expect(first.rateLimitScope).not.toBe(other.rateLimitScope) }) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index fb1f70e69b..bc1c23afc0 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -2,6 +2,7 @@ import { QdrantClient } from "@qdrant/js-client-rest" import { createHash } from "crypto" import { QdrantVectorStore } from "../qdrant-client" +import { ProviderPacerRegistry } from "../../provider-pacer" import { getWorkspacePath } from "../../../../utils/path" import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE } from "../../constants" @@ -42,6 +43,8 @@ const mockQdrantClientInstance = { delete: vitest.fn(), } +const pacers = new ProviderPacerRegistry() + const mockCreateHashInstance = { update: vitest.fn().mockReturnThis(), digest: vitest.fn(), @@ -72,7 +75,7 @@ describe("QdrantVectorStore", () => { // Mock getWorkspacePath ;(getWorkspacePath as any).mockReturnValue(mockWorkspacePath) - vectorStore = new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, mockVectorSize, mockApiKey) + vectorStore = new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, mockVectorSize, mockApiKey, pacers) }) it("should correctly initialize QdrantClient and collectionName in constructor", () => { @@ -94,7 +97,7 @@ describe("QdrantVectorStore", () => { expect((vectorStore as any).vectorSize).toBe(mockVectorSize) }) it("should handle constructor with default URL when none provided", () => { - new QdrantVectorStore(mockWorkspacePath, undefined as any, mockVectorSize) + new QdrantVectorStore(mockWorkspacePath, undefined as any, mockVectorSize, undefined, pacers) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -108,7 +111,7 @@ describe("QdrantVectorStore", () => { }) it("should handle constructor without API key", () => { - new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, mockVectorSize) + new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, mockVectorSize, undefined, pacers) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "mock-qdrant", @@ -128,6 +131,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "https://qdrant.ashbyfam.com", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "qdrant.ashbyfam.com", @@ -143,7 +148,13 @@ describe("QdrantVectorStore", () => { }) it("should use explicit port for HTTPS URLs with explicit port", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "https://example.com:9000", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "https://example.com:9000", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "example.com", https: true, @@ -162,6 +173,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "https://example.com/api/v1?key=value", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "example.com", @@ -179,7 +192,13 @@ describe("QdrantVectorStore", () => { describe("HTTP URL handling", () => { it("should use explicit port 80 for HTTP URLs without port", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "http://example.com", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "http://example.com", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "example.com", https: false, @@ -194,7 +213,13 @@ describe("QdrantVectorStore", () => { }) it("should use explicit port for HTTP URLs with explicit port", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "http://localhost:8080", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "http://localhost:8080", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -213,6 +238,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://example.com/api/v1?key=value", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "example.com", @@ -230,7 +257,13 @@ describe("QdrantVectorStore", () => { describe("Hostname handling", () => { it("should convert hostname to http with port 80", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "qdrant.example.com", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "qdrant.example.com", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "qdrant.example.com", https: false, @@ -244,7 +277,13 @@ describe("QdrantVectorStore", () => { }) it("should handle hostname:port format with explicit port", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "localhost:6333", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "localhost:6333", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -258,7 +297,13 @@ describe("QdrantVectorStore", () => { }) it("should handle explicit HTTP URLs correctly", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "http://localhost:9000", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "http://localhost:9000", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -274,7 +319,13 @@ describe("QdrantVectorStore", () => { describe("IP address handling", () => { it("should convert IP address to http with port 80", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "192.168.1.100", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "192.168.1.100", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "192.168.1.100", https: false, @@ -288,7 +339,13 @@ describe("QdrantVectorStore", () => { }) it("should handle IP:port format with explicit port", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "192.168.1.100:6333", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "192.168.1.100:6333", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "192.168.1.100", https: false, @@ -304,7 +361,13 @@ describe("QdrantVectorStore", () => { describe("Edge cases", () => { it("should handle undefined URL with host-based config", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, undefined as any, mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + undefined as any, + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -318,7 +381,7 @@ describe("QdrantVectorStore", () => { }) it("should handle empty string URL with host-based config", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "", mockVectorSize) + const vectorStore = new QdrantVectorStore(mockWorkspacePath, "", mockVectorSize, undefined, pacers) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -332,7 +395,7 @@ describe("QdrantVectorStore", () => { }) it("should handle whitespace-only URL with host-based config", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, " ", mockVectorSize) + const vectorStore = new QdrantVectorStore(mockWorkspacePath, " ", mockVectorSize, undefined, pacers) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", https: false, @@ -348,7 +411,13 @@ describe("QdrantVectorStore", () => { describe("Invalid URL fallback", () => { it("should treat invalid URLs as hostnames with port 80", () => { - const vectorStore = new QdrantVectorStore(mockWorkspacePath, "invalid-url-format", mockVectorSize) + const vectorStore = new QdrantVectorStore( + mockWorkspacePath, + "invalid-url-format", + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "invalid-url-format", https: false, @@ -369,6 +438,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/some/path", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -388,6 +459,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -407,6 +480,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "https://qdrant.ashbyfam.com/api", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "qdrant.ashbyfam.com", @@ -426,6 +501,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/api/", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -445,6 +522,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/api///", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -464,6 +543,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/api/v1/qdrant", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -480,7 +561,13 @@ describe("QdrantVectorStore", () => { it("should handle complex URL with multiple segments, multiple trailing slashes, query params, and fragment", () => { const complexUrl = "https://example.com/ollama/api/v1///?key=value#pos" - const vectorStoreComplex = new QdrantVectorStore(mockWorkspacePath, complexUrl, mockVectorSize) + const vectorStoreComplex = new QdrantVectorStore( + mockWorkspacePath, + complexUrl, + mockVectorSize, + undefined, + pacers, + ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "example.com", https: true, @@ -499,6 +586,8 @@ describe("QdrantVectorStore", () => { mockWorkspacePath, "http://localhost:6333/api/path?key=value#fragment", mockVectorSize, + undefined, + pacers, ) expect(QdrantClient).toHaveBeenLastCalledWith({ host: "localhost", @@ -732,6 +821,19 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) }) + it("retries the collection recreate after a dimension mismatch", async () => { + mockQdrantClientInstance.getCollection + .mockResolvedValueOnce({ config: { params: { vectors: { size: mockVectorSize + 1 } } } }) + .mockRejectedValueOnce(new Error("Not found")) + mockQdrantClientInstance.deleteCollection.mockResolvedValue(true) + mockQdrantClientInstance.createCollection.mockRejectedValueOnce(serverError()) + vitest.spyOn(console, "warn").mockImplementation(function () {}) + const pending = vectorStore.initialize() + await vitest.advanceTimersByTimeAsync(1000) + await expect(pending).resolves.toBe(true) + expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(2) + }) + it("treats an existing payload index as success", async () => { mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(new Error("index already exists")) await expect(vectorStore.initialize()).resolves.toBe(true) @@ -914,7 +1016,13 @@ describe("QdrantVectorStore", () => { const newVectorSize = 768 // Create a new vector store with the new dimension - const newVectorStore = new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, newVectorSize, mockApiKey) + const newVectorStore = new QdrantVectorStore( + mockWorkspacePath, + mockQdrantUrl, + newVectorSize, + mockApiKey, + pacers, + ) mockQdrantClientInstance.getCollection .mockResolvedValueOnce({ diff --git a/src/services/code-index/vector-store/__tests__/vector-store-factory.spec.ts b/src/services/code-index/vector-store/__tests__/vector-store-factory.spec.ts index c8d4058602..4e68d062e2 100644 --- a/src/services/code-index/vector-store/__tests__/vector-store-factory.spec.ts +++ b/src/services/code-index/vector-store/__tests__/vector-store-factory.spec.ts @@ -2,6 +2,7 @@ import { providerIdentifiers } from "@roo-code/types/provider-identifiers" import { getDefaultModelId, getModelDimension } from "../../../../shared/embeddingModels" import { clearAllMocks } from "../../../../test-utils/reset" import type { CodeIndexConfig } from "../../interfaces/config" +import { ProviderPacerRegistry } from "../../provider-pacer" import { QdrantVectorStore } from "../qdrant-client" import { VectorStoreFactory } from "../vector-store-factory" @@ -12,6 +13,7 @@ vi.mock("../../../../shared/embeddingModels", () => ({ })) describe("VectorStoreFactory", () => { + const pacers = new ProviderPacerRegistry() let factory: VectorStoreFactory let config: CodeIndexConfig @@ -32,45 +34,53 @@ describe("VectorStoreFactory", () => { it.each([undefined, 0])("uses the manual dimension when the built-in dimension is %s", (dimension) => { vi.mocked(getModelDimension).mockReturnValue(dimension) - expect(factory.create(config, "/workspace")).toBeInstanceOf(QdrantVectorStore) - expect(QdrantVectorStore).toHaveBeenCalledWith("/workspace", config.qdrantUrl, 512, undefined) + expect(factory.create(config, "/workspace", pacers)).toBeInstanceOf(QdrantVectorStore) + expect(QdrantVectorStore).toHaveBeenCalledWith("/workspace", config.qdrantUrl, 512, undefined, pacers) }) it("does not override an invalid negative built-in dimension with the manual dimension", () => { vi.mocked(getModelDimension).mockReturnValue(-1) - expect(() => factory.create(config, "/workspace")).toThrow("serviceFactory.vectorDimensionNotDetermined") + expect(() => factory.create(config, "/workspace", pacers)).toThrow( + "serviceFactory.vectorDimensionNotDetermined", + ) expect(QdrantVectorStore).not.toHaveBeenCalled() }) it.each([undefined, 0, -1])("rejects an unavailable or invalid manual dimension: %s", (dimension) => { config = { ...config, modelDimension: dimension } - expect(() => factory.create(config, "/workspace")).toThrow("serviceFactory.vectorDimensionNotDetermined") + expect(() => factory.create(config, "/workspace", pacers)).toThrow( + "serviceFactory.vectorDimensionNotDetermined", + ) expect(QdrantVectorStore).not.toHaveBeenCalled() }) it("rejects Semble before resolving dimensions or creating a store", () => { config = { ...config, embedderProvider: "semble" } - expect(() => factory.create(config, "/workspace")).toThrow("Semble provider handles its own vector storage") + expect(() => factory.create(config, "/workspace", pacers)).toThrow( + "Semble provider handles its own vector storage", + ) expect(getModelDimension).not.toHaveBeenCalled() expect(getDefaultModelId).not.toHaveBeenCalled() expect(QdrantVectorStore).not.toHaveBeenCalled() }) it("uses current configuration and workspace on every call without caching stores", () => { - const first = factory.create(config, "/first") - const second = factory.create({ ...config, modelDimension: 1024 }, "/second") + const first = factory.create(config, "/first", pacers) + const second = factory.create({ ...config, modelDimension: 1024 }, "/second", pacers) expect(first).not.toBe(second) - expect(QdrantVectorStore).toHaveBeenNthCalledWith(1, "/first", config.qdrantUrl, 512, undefined) - expect(QdrantVectorStore).toHaveBeenNthCalledWith(2, "/second", config.qdrantUrl, 1024, undefined) + expect(QdrantVectorStore).toHaveBeenNthCalledWith(1, "/first", config.qdrantUrl, 512, undefined, pacers) + expect(QdrantVectorStore).toHaveBeenNthCalledWith(2, "/second", config.qdrantUrl, 1024, undefined, pacers) }) it("reports the dimension error before a missing Qdrant URL", () => { config = { ...config, modelDimension: undefined, qdrantUrl: undefined } - expect(() => factory.create(config, "/workspace")).toThrow("serviceFactory.vectorDimensionNotDetermined") + expect(() => factory.create(config, "/workspace", pacers)).toThrow( + "serviceFactory.vectorDimensionNotDetermined", + ) }) }) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index af5476522f..283c685fb6 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -7,7 +7,7 @@ import { Payload, VectorStoreSearchResult } from "../interfaces" import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" import { t } from "../../../i18n" import { VectorStoreError } from "../shared/indexing-errors" -import { ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" +import { type ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" import { retryWithPolicy } from "../shared/retry-policy" // The patched client reports Retry-After in whole seconds on retry_after (patches/@qdrant__js-client-rest@1.20.0.patch). @@ -15,7 +15,7 @@ function toVectorStoreError(error: unknown): VectorStoreError { if (error instanceof VectorStoreError) return error const message = error instanceof Error ? error.message : String(error) if (error instanceof QdrantClientResourceExhaustedError) { - const seconds = Number((error as QdrantClientResourceExhaustedError).retry_after) + const seconds = Number(error.retry_after) return new VectorStoreError(message, error, { classification: "rate-limit", statusCode: 429, @@ -48,8 +48,8 @@ export class QdrantVectorStore implements IVectorStore { workspacePath: string, url: string, vectorSize: number, - apiKey?: string, - private readonly pacers: ProviderPacerRegistry = new ProviderPacerRegistry(), + apiKey: string | undefined, + private readonly pacers: ProviderPacerRegistry, ) { // Parse the URL to determine the appropriate QdrantClient configuration const parsedUrl = this.parseQdrantUrl(url) @@ -228,7 +228,7 @@ export class QdrantVectorStore implements IVectorStore { created = false // Exists and correct } else { // Exists but wrong vector size, recreate with enhanced error handling - created = await this._recreateCollectionWithNewDimension(existingVectorSize) + created = await this._recreateCollectionWithNewDimension(existingVectorSize, signal) } } @@ -260,7 +260,10 @@ export class QdrantVectorStore implements IVectorStore { * @param existingVectorSize The current vector size of the existing collection * @returns Promise resolving to boolean indicating if a new collection was created */ - private async _recreateCollectionWithNewDimension(existingVectorSize: number): Promise { + private async _recreateCollectionWithNewDimension( + existingVectorSize: number, + signal?: AbortSignal, + ): Promise { console.warn( `[QdrantVectorStore] Collection ${this.collectionName} exists with vector size ${existingVectorSize}, but expected ${this.vectorSize}. Recreating collection.`, ) @@ -289,18 +292,22 @@ export class QdrantVectorStore implements IVectorStore { `[QdrantVectorStore] Creating new collection ${this.collectionName} with vector size ${this.vectorSize}...`, ) recreationAttempted = true - await this.client.createCollection(this.collectionName, { - vectors: { - size: this.vectorSize, - distance: this.DISTANCE_METRIC, - on_disk: true, - }, - hnsw_config: { - m: 64, - ef_construct: 512, - on_disk: true, - }, - }) + await this.create( + () => + this.client.createCollection(this.collectionName, { + vectors: { + size: this.vectorSize, + distance: this.DISTANCE_METRIC, + on_disk: true, + }, + hnsw_config: { + m: 64, + ef_construct: 512, + on_disk: true, + }, + }), + signal, + ) console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`) return true } catch (recreationError) { diff --git a/src/services/code-index/vector-store/vector-store-factory.ts b/src/services/code-index/vector-store/vector-store-factory.ts index 2c8ad82bbb..e3afe84d16 100644 --- a/src/services/code-index/vector-store/vector-store-factory.ts +++ b/src/services/code-index/vector-store/vector-store-factory.ts @@ -7,7 +7,7 @@ import { QdrantVectorStore } from "./qdrant-client" import type { ProviderPacerRegistry } from "../provider-pacer" export class VectorStoreFactory implements IVectorStoreFactory { - public create(config: CodeIndexConfig, workspacePath: string, pacers?: ProviderPacerRegistry): IVectorStore { + public create(config: CodeIndexConfig, workspacePath: string, pacers: ProviderPacerRegistry): IVectorStore { if (config.embedderProvider === "semble") { throw new Error( "Semble provider handles its own vector storage. Do not call createVectorStore() for semble — use SembleProvider instead.", diff --git a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx index 53240cf487..5fac9ee3ac 100644 --- a/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx +++ b/webview-ui/src/components/chat/__tests__/CodeIndexPopover.test.tsx @@ -431,8 +431,8 @@ describe("CodeIndexPopover indexing reliability", () => { state: index % 2 === 0 ? "retrying" : "exhausted", })) renderStatusPopover({ ...failedStatus("retrying"), failedFiles, failedFileSummary: undefined }) - const details = screen.getByText("settings:codeIndex.failedFileDetails (6)").closest("details")! - fireEvent.click(within(details).getByText("settings:codeIndex.failedFileDetails (6)")) + const details = screen.getByText("settings:codeIndex.failedFileDetails (5)").closest("details")! + fireEvent.click(within(details).getByText("settings:codeIndex.failedFileDetails (5)")) for (const failure of failedFiles) { const entry = within(details).getByText(failure.path).closest("li")! expect( diff --git a/webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-amber-badge.png b/webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-amber-badge.png new file mode 100644 index 0000000000000000000000000000000000000000..dafbc590a8613dc1d5dc0933e92411e03e2c8add GIT binary patch literal 869 zcmeAS@N?(olHy`uVBq!ia0y~yV0-~&n{zM$Ne^8=Q3eKPDNh&2kcv5P?%H}s2TC0H zxY+!vitnOjVXX0bDpy=LaHw8Ub#hkQwh|4Npc4}m zTPJB8n`P2DX}g`vg(U5z-{V@@|JfVG6!{zdvz*gD`$x`gGX@6RHeLn?(L=lprv&mC z0@@0g8;n|{8Kx-SVOYTPkc}ag!<<3Gc?Y9|ByJ@&&(66t-wi*Xeg2vLlV6{=U)(OQ zbah!O1H-$B$B!RB%iH7kzvAN((TWSx!~X>S{&@J^@1z3^40k@J9nX!tcH>5bh0L_q zReS5o%FIknr$%V}->0^*Ad!J#-A=CUxwlWI7|Hef`}zH=y#D{|(wC;Z3v$f{YD=(ba7y5Gg;SxrM9nWm;EMLC-Vn#`N`tkq`k7Z9| zc1Nw{64!7!ot8LS+3SI z2^N~x?}~Qr*}i@IrcIaLmH}O0y{>I?k-?|J2isUrpFR!LsR`71Sb%f;Z&QYb|BwDm z{62Bw#GN~LM(C8?X0o)O$Xd7mJ_Ey^$2G@mi;9GJ*nmb!_a3`=F>v|io&SR7=clEq z32>Y^^6AHq8z5yicixumj?j7c{{88KAfwu3i^__M2n`W4Gcz7wK;_9-uIH0rI8a+y zcT6|<_``?wHFq}Yv`uDsp!FJYCK6&*D7@oRQ zy?}0=sCau@ukxn5anaGt8r&fvxCVE3ceez0cMU;;yA$9y@6A;GshU53 z=KcDqN>%FS-gEl&>E3(owbt%XB?YPX@ObcV-n@A){ZU-y&6~H~;4cX*1o))*W;^Z8 z8rE-WU=u1Kfm;@k(evmC$6MAf_R4Fy6v_REi*zxXAwcl_rEl=o1QN zje!`3C0Q{Fsy7MNk3t9tcx9*nB~fXb9EuO3(j@PFPgnOmyV^!}V&7A4K0a^hd9~HC z*Qp9!W}L2$wLUJ``_}dxF6~%b_>;hjihL3F${${KU^?kZ)O-mQ(rqUj}btltoaW#6lwA{Yh{{P1I1JexZ=%fr|y9;9^3F zumlVigRhZ_OTH69K_q3P13!WNfBXr}*9cD!kM~3Z?z^KY`rhY##maef>ZRM;k#%Nc zrOG|ezJ>4cIPQLbN3JjHXKY%nwU{EKhKwc>J5JU0CN^eFuAOw3$Vqf)m;zsnc=0nb8->PFq8UZp3< z!EqIct+&|b#;pH$A}gAAzS^+oJw8{mTp^L5C)B%l89WX~o85u5w6v*9h}$NC@R+_2 z`_zJH#Kc)AYi$b^I^49haOCH+C8`JBBZ-VX-Q7%j?ad}bv0}kU4u#(L7j7Fq+>Hl; z2-vUp_qw%92!1^6B32KJj(#~Pt72ngbEo%`-E&7rN1N+$ z*o`2VfRd6DeZQycsXVE<`;d?jK9_^u{{Ckjnv;{YRu|*ZkBW-a*M$78XFP7}9@L?Z zoD>uRx~xBDD?fYv9nbLSTkZ6DoUhP9S5ILxeg1c;n5lpcl|%8ToHI;2L%>7DviMJm zd8UXMFOIOU_x^N2kz9)ZSA}GjUvqPFGc$4D1-)*(|D6v+;&IYxR_fZxR905CUkoSE zYbN()3H$kuCbOnbjV>=Y+OAm66jD-Ap;5Q?+%M?+Ww4xfKA!B3r42+8>NMJDAtOg8 z>HDTs*>_xHK|K7soN%&Fpi|fLxmGu^>yN-~u$uGSj*+?;7UmUais`;x_p%4`83A+f zF9ekw2bxMTQ*Kw;$pvjgVyi!rmCM-3NHU5*nhsptdO9D@F_b<}F-s_f%4)sFc%a>= z^Lkzf<~?2jl;6#gIn;+g9t83z0kI)N)^5w}=mVR5-TRzq`+Q?A`` zFkK+q^>nimLoUT4JH@u0Ebs-b3`JsB4{=@J_udqY6sLQSr&SWsLlA;CHpS`hxC6p; zsJHQD?5h7egvsVCzvug(-yosHYWbb_z6QloiatL-p2{!@KkVgY3VM;XHFL7Dvl9*w zprHJusr7kuy~5s?^5+ymk)at^I7sQwMQZ$8^ER4TxYAF86D**h@@$_2#OV@5R*jU& zKiuo?Vj1S0qc+$X5`J~HwX;J#!N`w~k4|4!Y@2Gr0x2?hoq}MMt;~kOU@2Y6Voq0@ zPQja7tG7gVvennu7gRu#zy;H4dyAT^afxw|jFA$7o;>TquiQdzHD4ZjLT|14t)Yy_ z_He%9*RNlD#+|IjjmGsx>yFq8Z?Ja2?gZl;-UEx5(`J#S@aB$k4!3Y|!{?Up(z^E} za=$YXt0)uW!g)eq$UKRY`x}P5zR|0Ti|H=Mh#rg{r=MTn;`n?DS@da@@wJAwHH9A2 z#n(X|!5Dzp&EJI@rfCcqjZ^oA9nK5e_`rV1uRjopZ@yzjsd;_4AmzmREiW&Ra6qnc z2!CCL=^SIF(RKz(geHfFzroc~%O}OHr;7Rf6h7`SkLB6gZ88Fid(B@&KJa5PZ@? z(W!F>M-PYK;6pNG)0;MY+lE!jG%C&ttlqs%qJ*mv?PK*20~bvHnRz^0f<9npXLqNk zrL*Ne_=A+eaI>cmJ@quYqgqT$GL6$3%dy>gU*&bthS$y0$Ylv-%w>Wo2i1nA?(_0= zi^|puF+uhBoPzAjr>GLMGvCg;KjCsNT5mPOPltV^KVtj{qxz4MmIFu z6IY;sJzs9H)~X>T-943CPkG*=Fg;Ea_=0^?<*4+>SHj65-{{KlX)|{Y9(H)76>D8_ zx?4G0q4p=I=aq#({$gpqEOfPBf|V2IW71CEY1hlTNPYS5yvAmdSxxu=b4_7J6sVkl z&0N8zx_Jn5KPRM}(c+D(J%a{P|5jnOZ)~-?JrAAgh>4kR-_0C=AokmMPB7xDGuuFu zqq7mTQBCe(;*?RO(U7+3kjF>Du8L_-SsrQ8){Vvw20X?Axm^eJxe-myOSPu2@jG11 zCkgW>o_adFu?w2)n2Z=_T^y>7arLy)PlV7pOP33O+_Jc$u}>Mj42<4SjoB|HLPe&&&0&U1>sMB z%6C7e`XlitQaP=!_NG(|<#1UHzG|R6T5Oh@cDo^{r2tKWtW{L^DNt$!pLI} zwM>}VFuY99qs5wgXarC^OOcGk zYjE5d28oW0yQ=en&h%l;ZG+dTs;tZ!Y(1$cf}!(^oso*lpuoU2*A?4$KHQG1f2i5D zr#9sRMsYv_xa>g|2ID8z7I}R!;59edtqrp_B&Mdm4t}Ock)Qnw5-uABCFRaxRhKK< zb6T8#;z8tx+mkgAF3DFGRN>*xP2)HB{`7@g6r1@5t{yGbm1$OuJAmkf`)|KK-TU#l zX(EH4+itaaaq-hKJkcBR%c`#D?yuL^*FIa3+-BqHp~EXJELQX+}>2vGV z<%BRX21aknYx?5xx<1(1-EB1Z<&P|MXK?pK(i#n%L`z z)s$H=jVrQIugUK7g`6X7FUSI&4+r!RdDs-c&Z?~!Dm^ZDmn2P7I#%qv1VN%Vuc1qs zE7Nr1BqQ^G6O2q;h^Uys*N3ap%T`b?4bhWDmci#jXX*vk3-^Y%Q+EIy05_xbqN4Jb zTUZLu4~JD#Q&aUOW^i+*>aucja`b%0a#{qyQt)nlk_dO*9-bh-cJ<= zwA(*&f7%#PXspHQjV^$Uto6=o1JO42!eb{vQ_p?#-LfQz?k$JEe|;CAHLPx!oy#Mbu~xwfQyXKq?AlR)=)d%Dm?#3BZh-Aik_9rS#*9j7=w(IWHx z4m?%()cH97>131o-=gg6l@f}8-vD~$ykwVt`}WNhy`V~Ergws48$f`8_S|`|G!c_) zgTYFxyghRDzN7DwEp}^su~cSOQCMPxc?dM`8Y&MKxfHZnSi%1I&NN*8>oY}UKAZbw ziN(O`$Ff}c8FeLBvKFv~TlxkD21ITcYUpsynnOr1qpmV0Li+Umeplh#L8DyAL*{Z7TB4E(}t#_U?I- zk(pj6E$Qh5bWyq4!6fLts-KKH~@f;{YIytG1tz`5OvmB`xS<_I8BwQy%7|dL}oZInWB6Li~E`7 z(#=PIeB6R+dz^|CrHce6F~|J5JgI2=t|yH;m_AVrMwn`}4{!A>gHa7#f`+qUrr-1E z0hpc_EmS)GaTHB zi?AP(<>NC;<=UmTjkMe=^%sH9RT0=m*_wy68s(pq3qJnsb=(wjC*u&Ta=)+=fNyIM zt0Sd=SsjG5GW*ZrKL;}PaAagHgi={aow z>TFb@On<$vL5@^(bYTMM(7)7~F{yp&#@Qqa@c{DQdUNRVcynBwS@4wK13UIL#{{D- z%>$ZIixYa7*o&PRSHe#=Iml{!gog7MhTMxxoY9Yb&YAXETn^539{Mt8sSUS%Y(mLr;lJKvpe4WP5LlOMe+|H#J(&6(G_b);pa&bB?wCXoPB*zc+K@b;Z( zu;HH)s%H<0kA}JLKDRpmO0unT(#Zu6Ew^i& zqAZ5w94(frw)(aWbtyrpN}G7!6yU8=5n))0ITdSrI&C~pul-mgmBf<%xx}1IjJ0O4 zxRY%=HwWh*qR+5&m~>UYmAOIhR$R(&5D3*PROu}O%$OXgj5IcD-m|Xaobe*Pe{cG1 z>VbJ0-5R(=;H2Xs=`m?FTl}pr;N`8KY8;h3$E7wdmXw~ubYQ%)>j0tKnKK6)v{RwL zB8GTnIx?enJ&EI}$?ozMYs9;OC8sy?#IoXqWl|(riwvb79f}RjEzMwrP1{U{&{j>N z$a|-Teg_b+=u=H3CfU^4ax<1=n(mcswIk9HMrg^2ww{BsDnU4gmO3yZcpC(cK3J8x z(!7Dw=5rpc{Kkg7l%mK~tvYwTHHRvjLqi%WmDKd^Jkh$(+SZ z0^_FX*g0lAhc&yE#z@HRgkS3Hw^SA4TjiXkx>KL{oGpjl0DV!q#(qN1OI@M)s8d6V0*$wm?Mf7(Z4ICK zZI?NYMC$J<+fGFHSUu9CUv?=rR2Z>Xev;=!1TK$_<#dj~SNEy~un-#jFmKMwVvjNW zg#Mlr$uuP0(7*`Z5S`Vj{&TEj5C{7KPTahM!EUk|rG@~#0FJ7ag|iTVv);1T{lWXF z8jR{ya0?@NbE85^*Va6{Ih6K0Cw%^qUn>RAJ$ z5fzqYdhPB4S~Kmd5dDd^epN|XrwMg!Y>je!O{Qy@Rn@gV{cVIRI~rX}JNlXORjf4; zkA*f>K_^uDn#Vu;^!3`SH!c=5=Q>*b)aQLv6E)ctyAHIMTkFSQ1X6=MG_E=v|2;Yh z%MHp+Ob!R8boYB|UbW_0CPn-BZT>}OoAGH0w{W5h@}Q9C>2OX6&uEA9SDOB!AxW=Z zdNs;ELBQ`iy0P&BYLCeDj{suvFP zbI12M`Z+#5;TuxozU9hCj!re?$+{nucu2C$?MqO z4zD%YXB`XGYLkAU|7&*Pg5bSys`VtOfQ!51R508zbmv&H;+3fRctKTE7S4;C#Qxj> zU1gHaeo9a>?rzByuBpk!_6+yKVJG&UM@ltKzBRt|BsZh&LeWONz$3xQ;qo8%G8o9E zrwENDMn8AhhU&(sx^1_W+l)rwF;m?uzXR?bNcvY-LEG_W9Ihy-s&-K@E9Lo6sKRF^ zcO^qwsL11%f%v{&V^+7t`6_IFl+ulTS(nn~oGv)Wlb3v^U9sx>>uQCd{h6=V0VBzN z_#0F!jLb`G_3DOgBjw=o&05#o#MM?+H=3^gPD^dvpG#QJPKO-(GEGl$Pxp`aRZ5}| z>P}aO+N!+1XJ%3{W4kd|Sn$^E|FPVdmS5S%%F;|zi9FC1YG^c<8P(<7Xg&Nqq(aWxvZ z4pT-tr^9}vS70Y~?cs4X{`s-+Ii-|1TxP9(Kg?|ZVYE5jAdOqg04)G5qz->)sk3g9 zwOpv?X#7uS+-Ayd9S&vME|{S9W^Mc(!GC8xj5>`GQBkhXShH~j)bb!kH9M>|+F}7- z@2x@P@I1wTQHuB#60L;^c{q(@NXhJb0C8s6<8`ILtXF5dI0JZxeUJWGx zO`6nnR{Fg4-zUS-3vm>`NQX^R)Hq&8eLgtietgH{ z7tGpT8gO&6roBSIc^Q7n)lbCd!9`9@e(d~bov{qd=V8AVeXi#*aKmaEYBMPNhpgQF zz~PCI^WNmnND`2V_`MW0hIelV+B;a)+w9z-0-`fea8bIPPwql%h6=y_s1DJC)VsLF z5q8iLO>||COK3jY8A!BTfMtG}Ji}Vv?c1>9y}vzL8!PiJIjXa4xiKGKf)r-d(<;<2 zV){j$*qb>Awdv{9V8X%1=6Jf^0qFiBGmN8bhb_K*JNXNQi`#QnQ>yY5eoiZ$(%nt` zX={rrq+j1KYh(X3R;CJb^UX8-Bog}7p%-~HIjATby29hPo8Ev`d$hSyzc4xI`n<@R zBF!3JIKGi=Qxes?C`jWlWgVA$TX4zgu~Mp4Q^QJlV5vn%o&y|Yvel? zHlw*MPTW*x9(Tc42bk*&+*w*BaI908N&SUpJ8}1ZzpDM}k{UrRhVfhK8>L-DDxWYF z@5p|+Cp0n+w}8dgW0f-E`mZOd2rDf9`vdHS~4+z$V8%UQhUHaBPLAAh{v6e%pP z-)^=Iea`8$*2`>WsQF?XU1E1AdE(XiM1D?;T&$D8^+aSD^Ke`wdsDdMJHgxbwV#}h=^z;5i{mGwjkvnWPQV_u(0D|vex52e(``iFOpnH zXy>y&HKYC zX-BGOo}3sAhpg7!MRklZRmJc{m0hN}=iNG|iF1JESiX7CdoN!z-4EMmusb-_V`;%| z9U(bj;CTbccvrKtII15_s?3D)G1sdy8g2gIxZ4_fe+XWM8K=H{>FO2u2O-Z|M^jxy zWzQaX13$vZQdAd~Iv#lrnGeGp@hD-Z*yRa`ys$wP0qzVntL@7ZttYPSdbdT zq+p3XryWu(^?{;N3qL8ccTUM?!6vhjuO98*)%6#wpSvBP^5gQe|~ zGPHVcHhT#TD{ncQO2%Y`Z>An5;C9dnGp~XdoeQDOl4eM8#R=U9wh}GkF(0_Irw@#82&pK5U!dzS~eE9 z`3;p+#CP~iS}U*8)U2SuER~=2Hs3OnDvl|SN080vd~xEFXZ~O1~pAsv>kVE!6>da`~*j3pp*~>=eE_DJly{dOHP+u)z322;aKgrP`_|_�)x{YmN+_B|1jeS2YP zMQyJM(H(?nnaE>3nvav7b^Xx6+ZZ!!ebCw9w4uk3?Nsx4rW_MPcr!jieR=;QL;Y*{ z?k*RPz^E`S6J@rS_*tXY%7xC&h>U`m%nQ+10ajnZ2GKYt(rbT>!m#MJ-7{=?Q@Y1y zeVKNpvng$qFg&ri2OtsD$fdAV=rrvvj*NsC+`mJ>N(ZWZ+kyMt-D8?#QT%?3nPpqG zq_d&ooC`fvV;g-4ZGDs9$6`MXQJX(~U8h4btF5F%MPj!o8@h1ka8Woy6bwHv2IRDc zRucR3p(LX4xk>7Yq9Bnq^d;qSFj2qarV*xE zsy~Ay(sB#!28(dRe$i7Lfexb(Uy850Hu6XJCJ><{iGgCK!uvzq!4eFbN*^C)rLg2@ z*({%%B~ZfL98Vn+GRwG2$XWjIYg|xaDIPVYy?7sX|^Qzr6G!6O8eRc}R?%oW%T$DeP+Jq7z;-X%1g<9;^LgL8DH9EoP+e+Ui@aS z5dBeJR(`SB%2?$zWHTo8lu`R&gWgd$WN3TqFbz%^d0px*K3lu~ zmp|R1_~g9aH;Rh%3K$h?m>k!etM!o*!qX{JzlZ7tMSZ zJjm3(xp(uL;SgyaCpc&;hsEMUodnVyF`x5CbUO_JZ?WL$!vcveK^NOxS{8~yc1!u( zoGxV(3On6qmNuzq-|zkag4PQD`ZJNuh^O9B@1!-Wn48lbDQZ|grDNApko^*4E_eW{ zlT;<6bEM`nHK#9+5 zF8K;t|!SCZ^9;yI{PkL&E_J3+-`u_pp|7TIq z{UZW+cbWi}2Xb_`yc{VgL-VEM28jIx*x6?$ChUPw3FN6QlVBu50EE4c7FpYxcshag zN=c43g7W|bD5KsGQ0qq$@S?F+sF!Jc`NHxSD6%>QG70S`j^65^ib__&v7YVgD^M#@ zNzNt667+HfiV}gq&7!f0hzL#*PyzwNbDk~o9oPrl?#?zrwbA1A*Y$W=y{pxDAj&zp8ibA=+xVRX2qHX~{TT@em z&t=2od9?==8s#AX$?ex#F=H523r7LR47UXCadvi=!ft+`IiW1ITXAV#k8HpQu=c$jfoa%LWaynfC9~NLtde}bPU+e&3EF|;+DCuJ< z>;rS06KVp2f^)wRnA^Wz6c!euVXFGRR-nmBIJjejg9f3YmTeDVPO3|3*i5>`N;wcP zKxq|l_|u!nsPn~n-RE|VF4-5eNA*c=cx0pusJN+GNkA){2z&W=e+jHAkPr~SFB4!S zYDjgx7{Nd$c;1VC_aUZxdwU!BJ_0W91eJSUO%uGwGer|XlkoBJ;RWig#7G&`dZ!P6 zmIygsQQ*4*?#qcaXg}-Rzok5^OqQ5K$Q@O52y*hh_=&V0#H`J=`C`la3MA8`-JC&H#BIeh8qtK=^o zCt$ELh+sztr{*`5`_fjGRaohgeN7|=%NbAtI85vm+ zGVwYv=(*H7Z1r2&{}W}gnx%1byB*KqFYCAhm#m}=Jdflmj@8>>>R?tMs{B^!0+HIXWo12^6-QB@^`Qi=ae_p5EFDO`88h{@t zp8`Mo4;iqJM_xu|czj$B$mwAHR_nd$;J<$xK)e8~11xKo=jU+9$kdHn1JP|QPaV1H z;CsNRm^TZ268HN>S*@q5N7ZY4Wbnc zF-I(zEb#pN9PA|;AdZ$zDqTgfhcIXyzz}@9c3Vy#=k+T1e9YI&^}DCqK85uUJ1xVX6&KiVeK|oTOi)qx9ni)>u=PVPgt4Z zW65uS)$#-8!#nK?ZBCAO_9DeB1=+_HOxgNDN*EYqqF%^w5$*L++orWbJ8ZCE9pUSG z>#gkf^^p?C4-XD_>>;*Y8gyG6M+ey1*_&)vf_4Tm^qMF!lXG8#OQve!m zuv>QVa($l)OV*a~-5JJo`AS)Kg>fw|cDPX0Wj;aly1M9;1~(!VK7D2-ALzUB)-F68 zp{M6-M*Y?X%q&2@zjnK*tQ=l9>95f9)~_>@FT96AB4`FyEMOSC2!n!b1U4IjI?=xL z&l&cU+28mqW{YEL)gADDWBnW}ZLfKKO$ww*$w=vrRWBN2qW^H?;qHK3ecExW=jP@n zkZD09E1AC0Q*DKTZo%hhU7Qi&7kSP!Q(zj^Ms)y&L;yO(Qt#zSffeFln|(qc zPVLjBNc&@^VxKDZH=6q z92lB@*nez&073G*zdu2M~5_a~uOaekedDMks<-6%@s;L`K~GyYYz2Fw>)!=JQZ;d0T%2@#Eg2UwGIHy{>lmM|HN*TNZZ9vl zz6w-n)dk^OhQue&9jX9poumfqc18{7!FTw#Q^$UK7xUv7FLP6 zY_>OziKFa4wKA)Fx-uNf2QOP&FL*_Hy8MrjJU$N`nnQ1%BqVaPY3?Xkc0T-4S&rv@ zcMU+lDncUMxst}xWR42hBrV1RNGs1Hl%8ac`@6eX=;+2vOQ4`jaR#2lPLE5omHwft za9IsVmL=8bQR-unjCy%ZKUcnY$L4@DO4Vv8Q!5b_ZYV_<@z0gFS_lJhjW!SI#o%$C zHxo)E|78-3R*JLl6)=_zGI!qjr*82w_831Gw@?HY?}~euYI<4iV$V$}*J#8CO}EqR zfPAZD*qfq>RHDho;&o$#hZU&MlUp>EKgs-%06){=<)#h278@54Y2wuvh6xheAcyoP z+^%94^*iAFG0>kh-}^kf5rNCTSA2yv@&kDBcBDOMJdtbzqi%n}5PxAN4{-G1fqY6q zRMJ2s?fuY?5&}>!J2wSnJ+=5Z(d>*8>^v1+=Qm{^-8ovr*oUU9LgCZ!YLu#U8v>=W zKkWsZ$c+sQK~j#2(QXiX0Nl(fA#RJdS*}%w;?r7b=9-_~4)B>_d%nGDS7Ob+^eRBC z&dTZ<5gr=2+3C{f+e2|072%{|Z!y17adD)wnTzLq!HLHG$WfPLdmUuEQo3MMnmmxR zbC#ezsef+kMPAxVB0mjMM(qJYBICC~oqCI3Z*8FsB^WsB;)}}<@Je^4R9P4-Ak|nU zdM%__r%`b-uiH+Vw~*FS^Um1~;;<<59zrseDzqDH>T*DJ4PzP-w0zu^x)lA2L}FRX zd7EDHo^>rDly6s}0>{2NM#5?e-?drSs!Dv*|EIs&zu?&KhQw zDkkS?D#Bot2ArYG$}lKz0yk(I4S1e8b#4y7slkcE`0nkEQt zXq1x1DpY9+au^tRSz+H=NhFBo9o&b*4CUB`NgAme`WfS^hY?|LE(l0GO4p1_Sg*)B zEzoyog(=8z^C3-WO=YF)|jn-@8jD8t%hDZMN-Ik#uC+7E22waiDds# zRyN43yk9cIop3BSXV9vh7`kTjWi(r&gWgS5kFxxM5HjKW(0j(!Mu+Jl8?jiOQG|XQ z2DCzc3sz)1?$6gpsV#3&a6XGgq< zu=I47Nq2xo2xRI#3*6(eh^l`xH&1AMZ{!)T*)g%i_NF6sAI?^plUA_Fgs6csu9dgE zS|+}n?MZwuCAbtW`!fpaZgMgvxYD!-=0ws`+<50sgsvww4<`uC{ZG;Hu zS2=V}lF`CQVmtazjId9lC;+DN>y)XQUmDilCIGM+?ah1%NBYAbHWmXPB zGgrORTCQ|UMtMa;k!evA!GK;{;aT5vKuhCM=DpkzV=&&_(xS;Mi8j4bbZoqQOna~- z9qz8iB3oksZ+gGXD!7uUo`*I}kFUn3fY}>=PF00A54lny^v-7>rC%D&873&;$qa|F zT5Ra+n)CG(#Ah?LZ0-Ft*Fy;0DM)?U%3awt`SCFOT?yJ;ZcyGxVmW|%}OHTJO7g6s1!NYS@y4&u=xu3WR{#c zyL!}q>c;r7F(2e*zftGbD+{)@{GnCDHQL>iop!$VuFzzQU=(XIBpZqtrQ>=xgXf95 zOE)0Lb>6%bIc#?*O6hR31Wlz}=*y$dsJL9=x#CsA`E*M5afWjIdk!K z@oZ@FlJk-ooghB>duF%CauFNsnQv)g1g#A3Cv74VF(w+3Y zO_lEo-K-(C9^d?uQ4)XN6s`=z-X;>>ne|IZV=IJX40G2 z>B2o0ODRj_{`+kzp7cy6j!g;TKZYymUH)d9WaM2;QY;QVInHz$ zTE!$^>wSVObt+Tn<@JwqGnL7SD$K1#BUpHd?<7K%@2((LTH2$&XZpVjgZ%cfS^N#B zp;4GIr?na5xA`(Q8yDIt8yZ>kW8twKEn7a+t)!iv{dGQx^|dpC+}RywZD% z?L|U85w1Ny5n=h?83!ApGyeQIYlp;Q*^rwDYb+*MVI-!HGLJ7}R7o_7GmAInBBd3= zP2Jaz*YQ^5!(4>k%ew>;SZ@5jA@G4Tdmfzs7bkK+-~G>ROTCdydC<3QJFaD2ir_04 zjNB(O9c5*QV#tE=nVA4$-2m{0<(xM1^XE^1Sk2AN^+8n)ngKvvbpJe*14sZ+&#S)_ zgL>ezvTeCmpB?E zbz7m^Qo9q>>lyIL|CRMxsxyyi2X(h`*umpkYjgA7<75B8fX=aD`A*Gk#g3n5$e2|R z=~(+h`HtN%$hNahnSlMP{Ek6*s^|^s38RVmD*deVqvZzKj43yao9pW_>r*Q*e%ino zWrCL!G?4gjr|!bZ;hpgzJ=2B;udz+)L9 z-|Wx8pvUS!KtROL3Oydx9b`~iA%m`(KuS>?bwSHA?^yJJT2LAfChnN6rFvvBHKaL$ z<~YDQiiwIUdG|&|;Tg15zsvr+8={2p5|5|Zz7pa8khHxrCU7b@;4M}^L2)$+EYbEo=vw-fLh z59+e>?)iF-J3`^y>)9nIJgaMw-sjV?Fe&Qb%3^zTbwJbga7NM+9*GD{@k1D zF+eree*D^)hXcoOo8?yR)&R)D`0ha(EB_CRCt5|PGkL^|YpV+b|x&DuLYUNcZAF1Bd{o=(OvF}b?&jHpyz zg_V?1AJ#Z9F71)5VK8i3i9xENsGRcQxwFv=XC<;4*fhu`BPQc2)yw8VO9Uh2;+nkH zhG!!Rt#&RKQU9YBsIGuvdW0~L@bohytf)gi%REh;8dmtj&2e~aE&LUL#kL@i>V;a->?;m~SLkB3sgY7F;1JZ46w1+wqf2du4 zkX0%&3cFjWp>=(GxR3A5tKRA;dnZB@g^?C{5$4+TyS;kVv+)aUEmO3lj%|~jo0K77 z+urn9=C$XpG4e1oG-Y;_^1~zeD;fQGk_=A^iJ6l{B_nWtQJb%OoeBo1JEa0rSmZnOQU?YB!79Mj?RE?nVdB@IQMlBYnYl_+ z&tpgUwotN#v6-?NT=hs$R7 zJHgQmw{>8+F$QHRV*P)%4YdG11TY*zZYR2qj;)tez-79=FplSc8q)zSsGXow2oN?Tf)*B_?HIKj@SfU% zQt+%xAbx(|zUI0FOhm3csF9_*ns?@P>>ChYcGK(uryuN1wzGQ3Lnqso9MNFA@~S=P zzTThnl1UYK4m^(=78KHXeyJ0@WA6ILhTD&)?vI0mb2lx^d`_ZXswUDIqg*x`*901a zIzSU->V4KV=qs*W)qs~p_fjP+BP^^~w-y}*BE~%67VU7J*o%iuKy{B_=hc?>o05{U zM4oRvOE?-e_y=j=o-*K-j_v*tp`pblB+LP?;aGdHH^?JK#>P{em^kQ)$C$^6jmlbj zech3u@#2Oy<@Wko0_t2d_mu9#cj))!WIzdFuF;#4VzYOX2oLU?2zTfJg)+Fkw(Xdd z$WEK$B}f9OMptT}3;WLCqvXuYOcH}ut@HkLd!vXEUZpk6G`qteaRw!V!clKlOxqq~ zw3AE*lEF7`pk-gr3s|ZfaO4|>M>{CQBv>hCFyxBlw8eY|-B_OPEBG}ZUVt$AN+Y(Y z*+bPpR15_54NAmkdF-YD6`@y!JLTfH7Ls>^4$i0B8^g9b6V~H`(py13j#XP+;maNOs+&Ym+dlxe&@e%=FXq9eQXWw zc*Mkt^{HZX8s#+h=4b&%QikQqi6Wm!l}||=GXa@?3sMs^bw8dNUaTQLG^25!8E9=8 zB(5{S%ATH{{)n)JBw<`z&Kqf~mI%lUwlM1W5^jiKhCfaQgm4RtX5>Ma;HI#RVAlh7 z9khX6lvQ<#;)a4A!axSpMJ|cjTM#vBy?dKQa~XbtciQqz@dM#jq(m+4Tdce@qUupj z<9@{W2aq-av)UJFRA+m64FM;4s}-|=c-f7>{F`^|a82GlGMbQ|>a+zt;7=bgfp1S_ z-+w*UqQHaS@O@;YS94`H?BKGVmjiNB7`ag-I6DD@sOFV=jUfAfIU?czkb37!8zUmR& z6cUD!g#(hz@-}=XHeCk9=KjdULjQm`3;OD>fNTt`x-ZMtRV^(oJD6ab69cp|2V9eO zYOD3Gig!o`1) zz$yTG>5rEi_yJc0BI;`@s^D2^){nWDRBal|!bn)O7%cX-od(Ao2mtVt!S5!?eGk~! z>4*qKP0fYZYy#L#V(+tG_m_VMB5-?BC}k5#;z3X8O0^**X-k1CE4Xe@B+dQAKJ{z zK+pEyQ|pV#>ADNp9~l`>8Mtvnvmn%fPz(;P@E>}mz&6~s-eU&g`TzljMEJRVSXI#C2$yn z*J;N>PmdV;%N-mxwbKm%zu?r2ChZ2R^m3H~_&A@24}K3734wJ2B4DtwxpY;K9$9iTAZ!Nv(SsmYY^Sj`^X0LLi+u5##} zR<$sKkRQvO#y_M=wW0->33fmP8;E%ThiynoNeKh`wVal9=Wja^sS3Z-t{f5)u~0iW zS>#n7!Ta+cz3cv8#$a27yP^jpGVw6ew7;gccK~|L^X#`s*(DGsfY_F34%<)tta{DK z!{ZLv@g!z_4Sk@_4EoQlcX)xm196)N51`Rl{{+z>tQ${0I)lIFr>Z4`jL!SH(JqPx zzC48D?vr}KzW)$n#%LcB9;`?IX8C2t$LGIJCUsax#K}sN;_KS+9Yj+6-{08Xb5BPx z1QWGj!u*v!peRzz_I3G&#}QJQB~^j00TS4EZ{J_*u%EnuLug)JI%OS;!9Sp1(+cF9 Y(B{H#(dh}5;FCAf5(?s#q6Yr|7xs8sMF0Q* literal 0 HcmV?d00001 diff --git a/webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-provider-wait.png b/webview-ui/src/components/chat/__tests__/__screenshots__/code-index-status-provider-wait.png new file mode 100644 index 0000000000000000000000000000000000000000..3202a68a26d332b90448bc02a5385d608952beb9 GIT binary patch literal 15648 zcmcJ$bx>VFwC;(!1_>S{NN{&Oz`;F8a1HM6Zoz{D0>Rx~gF7U+1_&;}Lm)Uz=f1f$ zf7~~3>fKjU6{=GEoI1Vt?(VhL_pM(?t18Q2qLHA%z`$V2$x5liz`*u_KS~rhaHaI` zP9_Wt4vd_X#9Obd<1gO&Z#AEXvBzOJ8Dm1wSvMezpSMCt3O`1_NmF~8`^Y@Ut?);V zMbEEvj*y9Kmit58_)S1oQoC6J;el_>Y@f6f2sy{)oJu2tx{%McaHT+v%T2;F_-hK)oSbSyQ?E5 zMPcuYot_?XJUm}t-|a30s-5f-G|syj1vdA?IqT(m3%0N45MA9L$IA_0zI++%9LwOc z6n%bh)lgrkF$x@r!mX~Z4#82!tK;P`NTyTr^Y>r(`}19+M0s=jaK6GMzR7mcVKQ69 zrpf1gqc=M{`xC2yZi|C~k5Bv5=V1O{J3E(uCdH08o!#8XxUEzxw5rrKG)!I_wz)`W zpZ#dE6&LS`Bjw|DSoJzx707aSa`L*_j5MiuZ((7--t{;TjaQGAAoj%n_6-%)=89wY z_wn(5Ic)5Nr^kCfyQT92+0Ro$JPk2DFwa3{VOqx3u^-y<+dKPD&d-sa}! zrm^S^4GuoeGAt~>BV!F@B_$_^&LofvMXdaquXu?z*di(_T1r6ok&s1qU}#7y#r5oG zXNTume5K7|_2clXz>QM`szN%`avrp%cE8*DIn~jWHyXKl{3w{1RhktGU%mfO=5oFN zR@>OvSXbBEJDN=2eX|ww_he0_(pW$LC}piiG*o z|Jq<=^^;M+G8oq~$Gc&qETSnhYulC803-FRFNMXU}^I?Gcqz-5oS->gN_J{yt(s~#maZ^Lbw?@qW(T$$=v@v39UKV+R7it z3x^Qb7&=fMTNZ7*I`N~vS=p|-n0^@peDPW=y@V@I3{-xt8i#gTE3OYBv3TG{~ki!Un8TG z8&vC>v=5IhOj$8krN=j*F0Cm(@&eH$9BsIi^z94ebb8}Su(w_gYiVgkb(-2^4x0_f z556EYZH)@kx-(;2%F}`B)tjU4c#WsAfH&#NF;9XRL&(x+GdWYFV4@At6nnhXKp`H0 zOH#^`d5sH)Jg!!NI6c|&(+9npRjX%rs{_M`nid}?UVv~n36s)dQ$Zet)Epr~AZ1!Lh~ahR1Sfq%V~T*crb0$CyTbPc)pzZA z5ojkNsqCKy2BdUaajeou(|JL>CnqGNL!shHVjO4SKpY^;&5pY5b4f@@kU@awKn9B^ zS{3R@h^h-lEbb@IM)^O_h5sdUs__svH8*#ib=@DIoJh}GFH~muU6WDmRQJ!%Yd(|r zNaZk&SVdxyr!l|d1R2r$&t7;~SX4Q?-_3bRwokd^dY6&k@9>U}j@GKLcZU@sK9}sm z*r~&q9}bZ`MeKhx$|oZtA;nXO=%xon6Mv5U^yyPeON;Q|>ubl=*1D-KINRN^)O^9I z$>;Y$ykzXg;u*=Vd*6c5qJ%;-r^_|-EymNp5-*>jHR$m0zuo=x%J-N~hkpUS@fWt(gYj5XT3Q;&6Imc^$G14F>F%ur1qI!ob$6GaLAzpZ z-o=!PDvsh%^v{)P1U~#84n@ZPP(Ib}ekACANG9q>H0)(V61UeEj;>u}$YZfZCgfq~ z<8ucRqvE&cpM~XBKJ@HXQ@OW)_DQ?4Kguemswv9wIjjbO`DNMxIdh><4v)<9>?e7$ z7!r$ez~!_I2*tA{DhV+$l135;F!|n>2WmvMtgNhOAf9gx#!wZ%3b-+`u{n&wr7?a{ zTvRk)u6Y#4_w)Snav-B+J%H33B%-;oF`JVgO*#I5m#^0Y#hf;Jz{VS&oW$O%l`rbk zcHN&~VP##I$D}v%+6)Kxy0^PaB_c1gmhJ!d-K~QfzR%&@n?mp3gD;czR@+?LTfSW% z&MSzic)NT{=a?}xY;~f#^8xus)e;{$m>I0QiZeA;)ebmsfcDmK|@p3^$e?PWZ<@l>A^Cj=g zjx3X8R^7&flAM5#*$6(AjK()^=L96M3u)ou(tdsqAX(ZAtE_47YLRxzc6kzJ6xeQl z+s_G%XN{X8EIjWx>*}ZFZvzu}e(rQ>?u8mh!XqyyH##vfq+$E>>un;HAVm$pFUZ57p4!ElG9B|B z-45qawI6Tyb6PbDgR5Bfqnqtl`d$qlK<{&_D7<%)9J_9j5>)2!6;r2?FiAi`1HJxT zNr(Z{$gfN668ifb)T}01+j?$Mi-x|K&|y%T#n!PPi;?Jh3Q)WVGagHMLnNlE393UE z^g*4$WayKVBt_tZbGo+IVSl5zx%(((M|}e1)$h^`|6GmJpPi)AG?u~wmqfy52XHBH zDh8rv{le3sN&R(Z!|YNV_8W}&m~w+V!4lyCvFOtkQzaa4bB{-Mre$6Y(#FTfL(5B4 zGSY%0(Q&0Pov9$05K^5W#_w4Ie?XeKA-oO|g5*f9K=4%sp*2Yu)A$ERQzUcUM%@8c z2nO=851jMRP@=SOlc3cdr$Mtm)yUn_SC}9OJs#C`8@_!Th8$VkgA%Mr`ASGgNXjk| zMv}ypKo`bRH8H?mjlc%b1!=a}q;&ooc7a`B5?F6yd zoie8B+8;MDKgKd45mtehn8HVrkyv?=xqTzBog}~VloHh%kwEn%`<(OqP@>o3!2a?_ zqje~EBB>N9yU`Y;UyJ9s2b!(h_676tq5Jvq45$tc<>|tU?>`x^uwSa6nZ`lKM@G`w zO(3BB;jk=NOJgcauuk#J1Oy1}V@rK;;J3JlCT(p|TP+YY=3<@MDWdv-z=gE(&RoBVUKOn{k zJ(;GE7*hAT_DP`?pq2;Msi~{0J8%li5+eldnf8UXF*b%^h02O$VEjNkIr_xWv;~vQ z9;Rot&h!bC{ME6&Y(=nram)4j!n3op*JtYq@Rt~uNN?3Yi`B|cX>dGI9TAbXI%7Ge29RIBxtG%yyaq8Vyc{*^==^5Zu@$&YK5NsP+-#mvhlFj^TXK$sRUl zn$^Ipi1XlLu25%xD*7El#$N7DD<~v1{(^(wq-_K0Br|pS;i9bckL;Mg-~q2}G`xNk zDZZ`&>mN6BR|!dHnE8H@L-RQ=HqJ`k;{3|WO3Q3-sFkIg8Q)9HU0fwU^^HV| zg`g5cMFq*eBCWMdX{~L9712!Qs>qUaBdi~yecUAVlN!uT=Z97~!K4NoerhXM%;pab zF29dL&pXp04o!u1#Dy2-=EfK3=L+eBVR{ith{W^WefvLVAy?~;c1*B`XThRKNIcZI zr{{M{JYr1Y;D34nfzrhJ9C?V3_t!Gadh4z55%WXoY%R>&2NZEr&`v6Kr&U$Lh|3QI zOinNU;Jl~A5Mv;p=wDzplQ?43ci5k93N=Y4w4El1>yA(qnRa|<_YS#3*6453;qJRwqM)i4rOR&%u$JKFVM zahL5bCbT2)0^djU7-r$s@Y2~Xh_I!*pYC$Ay(-iD&DkI-ykF%%HJnPsbj!G5_RppI z@N-(xF>{V3q`TzhvEOi9jZ(f*^byd`gnKXi5Ke#d@Mm(oB)$YL4(A79B#joGh@B*n zL;AV%@zD|bv%<5URNxNn`QFCL7#C;ALZh{Yj$)J;<Eb4VR@F` zK}a`bRvcDaaZNb4iK(ZeM^Rm10*T!hp2uo`?4}{wN|X3pv{J*%Bgei=47Bie!X9!Vhk6lcSHcxZKC9oi0#mH*_vy7u2 z)VBpQL`MW^wGQpaqn0-N_3hiNRH%iywe2QSVRdPgeeA0!B_X8c( ztt);K7>7j0nzG_w^X_MVDD8Wp!2{qH2uAu-d$F;Ws*Rl^2Fy;fZE&H?F0^{hcq-aY-)9c;kal!H3Xce z@+-!tK2nlA28!*+VMxKgPR@y6EU|F7d9Ssd~Q`uFmmdyWpEUl_g zkQ@5_RkDE#`-HU)vy@AC?y%3fg7@cOch~1xH!b@k|M2FM%6Znbt>ZYO^)bPM=c5X` z=FqMxv(B2Hg_WVE=#igaI-e`cHG<+A6h;TXXNXOVOpfBY!_UFa9)Gb{2xI7~6*&4d zdEu(>L{QJ-f~8tC(b{eiDJ|em6E?>U>N%gQL!}JPT_8pr92^D*2VVs~)IkfT*R=!~ z7)tY%^e{a$PL}El99}J-th9{Vo31Z@OiiU#99)Q>k&dVG;3>Q8RQ= z>pSi1YhSQwQ_+6EIa>Q*>k@PIbqiexO}<5MlDR#R+0KA@T$#aJx~v1;#CO0mo)Zqiqs!cOk9}G1AuS?i!_=MK$1jvMueE*kU+=t6 zCiGBZmU=(B-bTe_qPN&LwwGhaR7v{i1F<358-U_o&)M%W@|m!B&T2((vNrtFaGRp_ zd#Wk#Qod*r32h%Pko$D(_s<4S)^#X&w5~TbDw>E5P~C{-x4#)xV#WPQSd=-P##k0^ zZ$MN&=zaUc-zRP^4o4lnEJTBC9iFC9&}(Y|^|5Abtx?mob-WV%I7-9ePro2d-?r|0 zRCB`YCG8ovIirdCKZ%0y5c^|$=EJ7OplgG}uN^uq; zleN|!DOt>cxws3A^>dZtRQy-sPy7 zM`$Lm{eLuDzPPGGlRth0L*g{#YN-65{Otecz1f-&f+k6sK}U0w`|hWggoL0iVbC35 zfCG_L8XzMMk65VETb!ABtw5+AGr*tw@dd=@_4~2XXc9&VQ+)_At>%#Y$->XY7GKiq zyLgX+9<)%AXwE|END3ueXBG z`>G+52`Qzw91s`{kEv)yjeY0aS@RxW$iHk_XTCGOTMAmB(~{UFJU`7>fi`NP&K_yS zEwxbm9<9d?S=FQ~tST(uHN?F+t;dG8W(I_F+L7L1kuDe<(*d>taI$}n`;#V_%ab#AYcu@_${i2a26JwH{+s_{fkmpK9q0avp2 z*Awd`I}Ple>gXsCp#!N4eSGFFwFwFV&CulZ!<&}6y(K|*^Hs`8bzeEW7OYg+Z*N*0 z`0dvPjo&)us@&6+U+%OXTrp`wi||WV{UWe1FIlTBgXuT~{jSpZl_Y3WpOK0FzQQ1g z#GB7Pt+uL7DXm4UD40c4tS*n!9=Sx#R?C@?;#%^JOaJ}Bb9FUCXqCSE=%pBwo40tIpU0xl96B*uX1)auNr!B7rTW-;}|{v(xfz>M(HX% zo#OfUI_J^uZ??$lMK`nzoQ>kMLP4J!uW~%yKRXj`#Kg&LDZQmpl(Jn!0DhE+Zb`pN|B;Z^07x%}eta7h$&`J3{4dtHa@j6#@7 z76}RCsJPydoSU`PUDxa2V496BO6wj0`^rP=T`YRc14n8_ zq@WKf$?9`EzhlbLDg-t!eKjR?=VZ2rzmgLsBh50h z3KsQDpRnCCiimOwbF$L1a0r^pgg7~SKI+-Mb@&A@jAeerXxymN&PC0}SR|F=yE~;?+C}98&YMgH^Pz?ryi`4Ock9%;r`xrjE}q&Y z!%{;wXlUFVn@0s21YzQpHbM)8pDW(oX%*Y||Lyc&vpu38Z_j_zSfPFYzD*-6J^r>6 zC*}-TkNWlIa>^581>gGn;foJID-_MWGt2isy#RD{Y0Wocyyc8dfoSy&SIWyhH4B3M z1tEUI21T366mPo6*+v$2Y>_#tM+tdtnv|jQtntbeDt@hps0ProH}S1^HE`%?F7@bd zZDdlL2RN5X$?Xw1AC6+kl$vWj-D)R2-tLx|?~fn3Qc@W5s|pDS^89`}a|v9VU%q9a z@5pLjHafwY!*a1p?>g zf5ybQ|BB1d*IYGvh$nyZv@N{7?-kBugvPk(d59Lx}DdU-;%s@nD z7;+T&g?dN1*N>Qmk?})Zod4agl20spbd&r-LJ4F7z@_c}iYSKbeo4$jHR3#c{PJfI z)$Z2)kJydwuG(P3C981NBCQI<(a{Bbt3lPBl-he+-|$p{;)2`ZYoK5QA^!n$7S1Sx)@?||*I&7x=HgwkmF zot=`bd%;NA^RJQ|(5zqU^z|fL`SHWlIsmj7KCu}Q@{y+&WM;O{6@7cdqwom#!wh9* zB=2Zvb8#4twXriZp+*lR>an=)>Nm0iU+L$k0}~Sz%EI`45S55XH>#h_ibERHS*Cww z_s*6?-ZVou#KY%5X}|yaVh4+ho1RJ#y1iHP{oU?NS9gM$uA9CM)YPy1M}rFePN~c9 z)Acg9TjUo+E%Xv=Z$?x|E1Mk2`8^)Q1I5l#sz>FbV^%5n+*C)x zdcs$5T4EBku^)|z&EYr(+>Z))bk8(>)wbf>V#tS)CU1oKU;Q->c(`jTd*KP#h%N*<6~l^EunO` zXq~)F#91$lgSt-2t=0u}3`e2n1Zt4*$9+)n(B7J}%vpZ5;;-YlPBYs5(4>>&dHRWG z|4GhxcdEFi#_Zx0g>^Pr2p0XfkKLc9V9oYo>)GPFR~;TFcY)8>fymettFaHcA>YqEtrl@omlQe)1-eo0E-$e?=^m5TTY+pE)P;6IW|8_LhMyN8D& zClH;u?@i^=MEg8r{3!D8UtNl-%_++R(Sgji(Rq7F^YN8;rHnzVlSCp(Os8*-NpGlM z<1<;}i_U=E!_pL+n{GA-CCI6laolH`0#Ne;5T_Qf931{~a&pClz>(lV=lXJYv{)l4 zId9Vxy~1mjmq8+rQdX8uOj_hFheGwDD<=BCi8cL~R_=d?jbivXtyQHvUt?s10~ryC z%{VJNTdLbeXZxC%lXG@>cn8=a7T^}qKR!OHd=Yft>=KO!YsiU*ByxfaF=nRDpHCoR)vA_RDq@toi zRKlX$c=L3B23UrZ{e91;huglszEmb{(Tfq9c1M64@tO`q{p*5iW0DKv=YRD$(S3a| zU!m>n{2Me4_a}35l9OprsCFEBfUw|<2oLuMT|>*s?6{bil?F={7nh5Ai}Bo#piLv< zcf&5#>~Zn~&`-kZe~%W~*x8+aZwRlZ?bU@B`$R z1?O4|yYTL0p`E?Gkmo6LW4o}+j%?)(z-a`9g;&?sfc6?O2oX>gv4b*GcLL5TDj0$` zJ2UgI_4Fso7lNtRY!^N93y?LEl9C4c`c5;;%kQ4`UW>Zypg<~`Z01AI6KWFS8@+yh z^o34q!)+zh-}Ft(qhemNeSZA6 zVB*dL5C5AqUs!%168RjLI+UdC@ zqhn*Axh$27M!$A;0t!tI%-OsWkK>A+Am|z&u61~V$-|gDK0XG$y4{_f){YLb%PGm5 z(r(EB;Bjd(rY0xP0^scL>Dmfl-azNOQm3A%;E=t3T$nU zc6N5a_qKkB`rkF$E-DvHgFRKHQwIUU+TARk$A@tlY62$*Y$njY?zCI3XVR_~Ken^8 zqwv3+#w6txr~YS3&5Co|&X+gYuQY>BIz$BcXC50pu$GqltN>L5t>#ZhmM8>zr)XkAg~iFmm6u0LOG`*d z$Pa%%x3JKxXbM3oBmCxk7$~4?dls#W-Poyi+LAPUoFtwuuBMiBO5{U7SglBs6&oEL zESe_ZeRVhwxQlENU#;h_bE@Pxv#YDCU=vD4nVFgG>g58f#fRgMXTd9bw+XgJcHx|4 z%iCTgOA|s6qt6F$PX~VcU$4=tWQ+R$7#~-l0ANG~XxLAw4ybIlNkU1vt&Cqw^6~LK z7&KZ<19rmmboC8b`E>ted!NOTLvp6(nOLZ48|o26N2S|&8|k%X_G*LWBnW`n-h0Ge zm*-=31Nb;N=YT-j-`{_}*-ik&$tK*6)F02vx(ogx8m^Nfk^C^G#X8-BK9>wUDLCpU zbLO^htzH%!F4iQ3j(=h^+cX3#qe;ELN-fWzqucG*M=)`jNGeqftm&MsCP z_yXqOA{Rg_7dr(BqKDhtH02Ya*rEq#ph5U}of&{??m1H^*B^nI$?t3mc8aw7`48I~ zLUSP~xT3CF~;F7Ttat3VW?&$YSJ{cJqQnpgH0_ddZ zePw57=V>eeMgX1o510ka1j=j=EzVSbB=#f#1wa%WX;ApY3Lyrm0+4RDE6r7JqTO4J z4ARp#0Z+Sibi{P(a(S4Lq{Q+x5}WH?LotgK*Doz-G{n5886S9)T%-(djPs zbh90f{_?crO!W2%`?X=6skE|kJTm~E+9e|}B;Jg`n!&91;WRfju`o011G)yp6J!0J z3BQ<8Sw$E%U6PKD5-KPDvqlpfx z;k>_;o>jsZ1b}wM=zoI-$&b<;@eSL5b7av%?uTx-7Z#;d_0m^k=Hcq+3`wN&%zR7g#gn;J zl;Qjl&;n-SS2d-geYs4gro6|&9v&VC!eH01%owBfhS&G^IQM4c=TjRz$RpF)y{!ZQ zAw&p|OvLAHqA#gy<@DB;DG?k|kLn}>d4%FF;**CXWKtnip-{h@vV(Gk%REis% z8tXv^2n_wOEgy58L_3|i0Q)rwE-t!}S4*yS5H zh|FFD<|(zJwZL;Bdmv-rWrckYnK8j2(ZWWGT2(XB^eWn;y}bx;^kRn3_+~3!5$&IVWKV-_>%|xjE-5M5`~o2{ zKFMc}!ZoENSc5zM4PlwGRm_Jlc_dOC@E4OAIFWJ1O?7o3Vj&z`tbQ@cp5jM(ll`57biLNMt*u@pi@Qiv5B^ev=5T zN|#D9R?mdQVk}$)@z=CiD7BL<(PX{8frTnxDa(UHdJmK{EWRf?A~<47rcFs(Rh80} z>#N36w^xc%Q_pVXBST|y;?4Ej!gb0!N|o0AcHvzK!1{fpRminCKmDy}!gvVbHoiIE z1cyNAv;6f-^|-m1)sU6*@oy`O+(^p)<5hGepeKm8@G{UFyV*5TVHheJB;L3PvFGEV zu;r#Yg@&yDNxu!HCt|J@GS9^+N01JcVV+0m+5vmP*t{2^NA9s=gZ!xGr6xjbJwU$L zfNUy`B>NlDLl)sr^(b{?OB9#CRc^W=C#hn!-X81Blwjh3QsvTH8DRl?^dc?SjUX>| zhJza+4@2sOmW_mPUaS!|9_kA^MR?r^y~K$0%U&w9(%J`dI2$QLkKOCP3;|HfkfU|KVKtf}>a$-AK_T{&aNzScAq1VYx3Y<_Vv z!aoRG6Hydfzi5J4MeZ;zb!`~-1m1j(pgp`M#t+tMoe9APkBi@cRK-R#0C zDSIi(5=BSl+WL;5irTapYLQzwqd|4e|6aF1mE+RaAPWncI1wY|gyZb*#(AmydiJ@D zPLP8c#Va1_2hD4GWl};JxD#y_hBdyV53!B415iwiu!k;?d$HzGj%p01G3-!ERvpbL zf5ej@^B!Zl#gJS^ErDFN5xiZJ_q6`vU(`aq;0|*g7dB}=1SK#l!W7{0()9T`G48gb z_4wz(QrWlqr$?s4Qx-=h=U?UiAZpj4tzeL3=BAoKjwE3W6;{J+&Do^`>qdCsrI8;l zFIyNAm2#E@N$ZYItK{8>d?rEmO{!O`qQ$g|7%oYe$#^b7h#yL#e`y)r;3!B)np9y3 z6qLJkaWkuRPf~_(dz~9IFq`9#`=@rXuGBJ-{1w;9MH$b1EC-|KP$ zSr{^r%FyVQUQmzJ)??hG>qcylzsJ%k-eN785p~sb2^|iCF64Y1?HslD>gBwvb~eIj zhELNsAs0vAi0YM{&an1GEd6^jqnd}w1We5r*>uMejoGYz&e|H*Ic9eneD#3EFExCoHOE&{cqfPVw zjmG$oV_E}WSPMYmR$QDQS zU-_v%xwda=O|^5w!omWn5zgbYGm-=_woh+lf!gdzptNqAsrRP z4wm|B#T39u3Z5H#R1EicUW+FeqK@RTUC{sPRhM6tJI)e?194z%=_9dUuBSbgxxGV^ zVliPqbTo|Cu>_bl&~RaY5=8F~fOvDBnVbY@tbaTAMU{!u`P%BLo1O${pFDlb-~y+K zY87`KN+J+b?~B!e;DUlD>MI>ERRh&*%!d!gzJ2*~o%>i1m$NFhWHRw&sPp&0r3qia zPpu)CE#g=Wd|53r@$XpEE2?50tB^jx1$QV)01fTH>A3~<2GSsuQ)@KQ&CN|JC8$>w zGQzUAs)6gddImi?KK|x3g!{HA;swM3-E%ddAj-X3x9OisH-LLC$^QSc!&b>xcoT@Z zhDL_Itu|-ixR7o(e+T{Zs1CE0_jzkkh2S^Ef)RzFmYvktSe_AOVYAJndF60CND;9>qvSLxZcl{D+Ny;|YcSyBTI^ zZ_j`{zShpavdhe2MG>V<^54ferlS9D9Fo%aj{}gxeaP|opVCeLt7DI08P9_$i;Xhx zJMz5UKjW9pg}e>2*OKojT|}hXnJ8{;Z7nT57>Xk$uOdyO^-oMFza)DmZG{SX(QrqGAM0Jm5ew%C7>01PBwLq`_0R(Uf7?a?`W=Dae=9 z%|g8dK7aJ<3qyU$s(&*X_{3#BQwWMs`@L@&;D8kYZ2)L00UPZf?@^|M+WnH;Y!Ss` z$54>ab#yzL1D=F#kW68mC)*1gzW|G})uZZitsfN?wN?%f5AWah%fTY%+=-^owpy$v zpV;5o0pBYCW;r-03Va;k?Tr9ZJ^S-#pY0(`{iPO1qlbAb?tcs~7Vt2$K$L|-p}>{_ za$ipx8i_kkNIE}InN6NH6L8Fw`lArI9e!8y+V@vS$ZG?`Kx^#m?!MF2?fNqz1gw~U zhfMzS$R#j Date: Fri, 13 Nov 2026 15:27:30 -0800 Subject: [PATCH 26/42] fix(code-index): align tests with single-attempt writer flow and keep Qdrant init errors --- knip.json | 1 + .../__tests__/cache-manager.spec.ts | 5 +- .../code-index/__tests__/manager.spec.ts | 14 ++++-- .../embedders/__tests__/bedrock.spec.ts | 11 ++++- .../__tests__/openai-compatible.spec.ts | 4 +- .../embedders/__tests__/openai.spec.ts | 48 ++++++++----------- .../processors/__tests__/file-watcher.spec.ts | 2 + .../processors/__tests__/scanner.spec.ts | 2 + .../__tests__/qdrant-client.spec.ts | 13 ++++- .../code-index/vector-store/qdrant-client.ts | 17 ++++--- 10 files changed, 73 insertions(+), 44 deletions(-) diff --git a/knip.json b/knip.json index 1f4956faa6..0b097bc728 100644 --- a/knip.json +++ b/knip.json @@ -32,6 +32,7 @@ ], "ignoreDependencies": [ "@roo-code/config-typescript", + "async-mutex", "@types/node-cache", "@types/vscode", "@vscode/ripgrep", diff --git a/src/services/code-index/__tests__/cache-manager.spec.ts b/src/services/code-index/__tests__/cache-manager.spec.ts index 852241c0c3..603a027784 100644 --- a/src/services/code-index/__tests__/cache-manager.spec.ts +++ b/src/services/code-index/__tests__/cache-manager.spec.ts @@ -99,7 +99,10 @@ describe("CacheManager", () => { it.each([ ["missing (ENOENT)", () => Promise.reject(Object.assign(new Error("gone"), { code: "ENOENT" }))], - ["missing (FileNotFound)", () => Promise.reject(Object.assign(new Error("gone"), { code: "FileNotFound" }))], + [ + "missing (FileNotFound)", + () => Promise.reject(Object.assign(new Error("gone"), { code: "FileNotFound" })), + ], ["invalid JSON", () => Promise.resolve(Buffer.from("{not json"))], ["a non-object", () => Promise.resolve(Buffer.from("[1, 2]"))], ["non-string hashes", () => Promise.resolve(Buffer.from('{"a.ts": 3}'))], diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 6de8bfd1c9..382163610c 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -3,6 +3,7 @@ import { makeExtensionContext } from "../../../test-utils/vscode" import { CodeIndexManager } from "../manager" import { CodeIndexManagerRegistry } from "../code-index-manager-registry" import { CodeIndexServiceFactory } from "../service-factory" +import { CacheManager } from "../cache-manager" import { CodeIndexOrchestrator } from "../orchestrator" import { CodeIndexSearchService } from "../search-service" import type { MockedClass } from "vitest" @@ -243,11 +244,13 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { }) manager["startIndexingInBackground"]() - await Promise.resolve() - expect(consoleErrorSpy).toHaveBeenCalledWith( - "[CodeIndexManager] Background indexing failed:", - indexingError, + // The start-promise join (R13) adds microtask hops before the catch runs. + await vi.waitFor(() => + expect(consoleErrorSpy).toHaveBeenCalledWith( + "[CodeIndexManager] Background indexing failed:", + indexingError, + ), ) consoleErrorSpy.mockRestore() }) @@ -640,6 +643,9 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { }), } + // The vscode mock has no fs; a missing cache is the only empty-cache case under D8. + vi.spyOn(CacheManager.prototype, "initialize").mockResolvedValue(undefined) + // Enable workspace indexing before re-initialization await manager.setWorkspaceEnabled(true) diff --git a/src/services/code-index/embedders/__tests__/bedrock.spec.ts b/src/services/code-index/embedders/__tests__/bedrock.spec.ts index 4ffdd647cb..97ff7f62e5 100644 --- a/src/services/code-index/embedders/__tests__/bedrock.spec.ts +++ b/src/services/code-index/embedders/__tests__/bedrock.spec.ts @@ -685,7 +685,16 @@ describe("BedrockEmbedder", () => { Object.assign(new Error("Service unavailable"), { $metadata: { httpStatusCode: 503 } }) const throttle = () => Object.assign(new Error("Slow down"), { name: "ThrottlingException" }) - beforeEach(() => vitest.useFakeTimers()) + beforeEach(() => { + vitest.useFakeTimers() + // A shared registry would carry the throttle case's pause into the next case. + embedder = new BedrockEmbedder( + "us-east-1", + "test-profile", + "amazon.titan-embed-text-v2:0", + new ProviderPacerRegistry(), + ) + }) afterEach(() => vitest.useRealTimers()) it.each([ diff --git a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts index 08380d0e80..f163c876b7 100644 --- a/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai-compatible.spec.ts @@ -938,7 +938,9 @@ describe("OpenAICompatibleEmbedder", () => { Object.assign(rateLimitError, { status: 429 }) mockEmbeddingsCreate.mockRejectedValue(rateLimitError) - await expect(embedder.validateConfiguration()).rejects.toBe(rateLimitError) + const result = await embedder.validateConfiguration() + + expect(result).toEqual({ valid: false, error: "embeddings:validation.serviceUnavailable" }) }) it("should fail validation with generic error", async () => { diff --git a/src/services/code-index/embedders/__tests__/openai.spec.ts b/src/services/code-index/embedders/__tests__/openai.spec.ts index 5918ce83c3..c0b7f7e0cb 100644 --- a/src/services/code-index/embedders/__tests__/openai.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai.spec.ts @@ -103,13 +103,10 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith( - { - input: testTexts, - model: testModelId, - }, - undefined, - ) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ + input: testTexts, + model: testModelId, + }) expect(result).toEqual({ embeddings: [[0.1, 0.2, 0.3]], usage: { promptTokens: 10, totalTokens: 15 }, @@ -126,13 +123,10 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith( - { - input: testTexts, - model: testModelId, - }, - undefined, - ) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ + input: testTexts, + model: testModelId, + }) expect(result).toEqual({ embeddings: [ [0.1, 0.2, 0.3], @@ -153,13 +147,10 @@ describe("OpenAiEmbedder", () => { await embedder.createEmbeddings(testTexts, customModel) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith( - { - input: testTexts, - model: customModel, - }, - undefined, - ) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ + input: testTexts, + model: customModel, + }) }) it("should handle missing usage data gracefully", async () => { @@ -215,13 +206,10 @@ describe("OpenAiEmbedder", () => { expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) // Verify only normal texts were processed - expect(mockEmbeddingsCreate).toHaveBeenCalledWith( - { - input: [normalText, "another normal"], - model: testModelId, - }, - undefined, - ) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ + input: [normalText, "another normal"], + model: testModelId, + }) expect(result.embeddings).toHaveLength(2) }) @@ -509,7 +497,9 @@ describe("OpenAiEmbedder", () => { Object.assign(rateLimitError, { status: 429 }) mockEmbeddingsCreate.mockRejectedValue(rateLimitError) - await expect(embedder.validateConfiguration()).rejects.toBe(rateLimitError) + const result = await embedder.validateConfiguration() + + expect(result).toEqual({ valid: false, error: "embeddings:validation.serviceUnavailable" }) }) it("should fail validation with connection error", async () => { diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index c05bf2dcc6..8cbe54faef 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -156,10 +156,12 @@ describe("FileWatcher", () => { } mockEmbedder = { + rateLimitScope: "openai:1", createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), } mockVectorStore = { + rateLimitScope: "qdrant:1", upsertPoints: vi.fn().mockResolvedValue(undefined), deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index e901322c9b..c462f5bee1 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -82,10 +82,12 @@ describe("DirectoryScanner", () => { beforeEach(async () => { mockEmbedder = { + rateLimitScope: "openai:1", createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), embedderInfo: { name: "mock-embedder", dimensions: 384 }, } mockVectorStore = { + rateLimitScope: "qdrant:1", upsertPoints: vi.fn().mockResolvedValue(undefined), deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index bc1c23afc0..4f50eb5e84 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -784,6 +784,7 @@ describe("QdrantVectorStore", () => { beforeEach(() => { vitest.useFakeTimers() + mockQdrantClientInstance.createPayloadIndex.mockResolvedValue(undefined) mockQdrantClientInstance.getCollection.mockRejectedValue({ response: { status: 404 }, message: "Not found", @@ -791,7 +792,11 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.createCollection.mockResolvedValue(true) vitest.spyOn(console, "error").mockImplementation(function () {}) }) - afterEach(() => vitest.useRealTimers()) + afterEach(() => { + vitest.useRealTimers() + // clearAllMocks keeps implementations; a persistent rejection here would leak into later tests. + mockQdrantClientInstance.createPayloadIndex.mockReset() + }) it.each(["createCollection", "createPayloadIndex"] as const)( "recovers when %s fails twice with a server error", @@ -1354,7 +1359,11 @@ describe("QdrantVectorStore", () => { mockQdrantClientInstance.upsert.mockRejectedValue(upsertError) vitest.spyOn(console, "error").mockImplementation(function () {}) - await expect(vectorStore.upsertPoints(mockPoints)).rejects.toThrow(upsertError) + // The wrapper adds the classification the writer's retry policy reads; message and cause are unchanged. + await expect(vectorStore.upsertPoints(mockPoints)).rejects.toMatchObject({ + message: "Upsert failed", + cause: upsertError, + }) expect(mockQdrantClientInstance.upsert).toHaveBeenCalledTimes(1) expect(console.error).toHaveBeenCalledWith("Failed to upsert points:", upsertError) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 283c685fb6..25ee35e560 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -6,7 +6,7 @@ import { IVectorStore } from "../interfaces/vector-store" import { Payload, VectorStoreSearchResult } from "../interfaces" import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" import { t } from "../../../i18n" -import { VectorStoreError } from "../shared/indexing-errors" +import { IndexingRetryError, VectorStoreError } from "../shared/indexing-errors" import { type ProviderPacerRegistry, makeRateLimitScope } from "../provider-pacer" import { retryWithPolicy } from "../shared/retry-policy" @@ -111,11 +111,16 @@ export class QdrantVectorStore implements IVectorStore { } /** Runs one creation request with bounded server/connection retries and the shared rate-limit wait. */ - private create(request: () => Promise, signal?: AbortSignal): Promise { - return retryWithPolicy(() => this.pacers.get(this.rateLimitScope).run(undefined, signal, request), { - signal, - budget: { retriesUsed: 0 }, - }) + private async create(request: () => Promise, signal?: AbortSignal): Promise { + try { + return await retryWithPolicy(() => this.pacers.get(this.rateLimitScope).run(undefined, signal, request), { + signal, + budget: { retriesUsed: 0 }, + }) + } catch (error) { + // The original error keeps initialize()'s existing connection and dimension-mismatch messages. + throw error instanceof IndexingRetryError ? error.cause : error + } } /** From 532a8fb73b15cb2c26e817a3f25ce7e34d0e8f25 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 16:36:02 -0800 Subject: [PATCH 27/42] fix(code-index): resolve lint, knip and file-watcher test failures --- knip.json | 2 -- src/services/code-index/__tests__/manager.spec.ts | 6 +++--- .../code-index/embedders/__tests__/embedder-factory.spec.ts | 4 +++- .../code-index/processors/__tests__/file-watcher.spec.ts | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/knip.json b/knip.json index 0b097bc728..8535bcc70c 100644 --- a/knip.json +++ b/knip.json @@ -22,7 +22,6 @@ "!**/__tests__/**!", "!vitest.setup.ts!", "!utils/vitest-verbosity.ts!", - "!utils/TaskSemaphore.ts!", "!utils/WorkspacePathResolver.ts!", "!test-utils/**!", "!utils/versionToken.ts!", @@ -32,7 +31,6 @@ ], "ignoreDependencies": [ "@roo-code/config-typescript", - "async-mutex", "@types/node-cache", "@types/vscode", "@vscode/ripgrep", diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 382163610c..eb30afce76 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -826,15 +826,15 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { }) describe("stopIndexing", () => { - it("should delegate to orchestrator.stopIndexing()", () => { + it("should delegate to orchestrator.stopIndexing()", async () => { const mockOrchestrator = { - stopIndexing: vi.fn(), + stopIndexing: vi.fn().mockResolvedValue(undefined), stopWatcher: vi.fn(), state: "Indexing", } ;(manager as any)._orchestrator = mockOrchestrator - manager.stopIndexing() + await manager.stopIndexing() expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() }) diff --git a/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts index 7cd4bbe076..b1e43ff292 100644 --- a/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts +++ b/src/services/code-index/embedders/__tests__/embedder-factory.spec.ts @@ -1,3 +1,5 @@ +import { providerIdentifiers } from "@roo-code/types/provider-identifiers" + import { EmbedderFactory } from "../embedder-factory" import { BedrockEmbedder } from "../bedrock" import { ProviderPacerRegistry } from "../../provider-pacer" @@ -8,7 +10,7 @@ vi.mock("../bedrock", () => ({ BedrockEmbedder: vi.fn() })) describe("EmbedderFactory (CI-EM-03)", () => { const config: CodeIndexConfig = { isConfigured: true, - embedderProvider: "bedrock", + embedderProvider: providerIdentifiers.bedrock, modelId: "amazon.titan-embed-text-v2:0", bedrockOptions: { region: "us-east-1", profile: "dev" }, } diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 8cbe54faef..7e7542da0b 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -36,6 +36,7 @@ vi.mock("../parser", () => ({ content: "test content", start_line: 1, end_line: 1, + segmentHash: `segment-${filePath}`, }, ]), }, From 7ad9a7e6d9619735985085d836c79a44a510093c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 17:46:54 -0800 Subject: [PATCH 28/42] fix(code-index): prune stale lint suppressions and make failure-path tests platform-neutral --- src/eslint-suppressions.json | 6 +++--- .../code-index/processors/__tests__/file-watcher.spec.ts | 3 ++- .../code-index/processors/__tests__/scanner.spec.ts | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 61074bdfec..94035beff0 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -1086,7 +1086,7 @@ }, "services/code-index/processors/__tests__/parser.spec.ts": { "@typescript-eslint/no-explicit-any": { - "count": 22 + "count": 19 } }, "services/code-index/processors/__tests__/scanner.spec.ts": { @@ -1131,12 +1131,12 @@ }, "services/code-index/vector-store/__tests__/qdrant-client.spec.ts": { "@typescript-eslint/no-explicit-any": { - "count": 87 + "count": 85 } }, "services/code-index/vector-store/qdrant-client.ts": { "@typescript-eslint/no-explicit-any": { - "count": 4 + "count": 3 } }, "services/glob/__tests__/gitignore-integration.spec.ts": { diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 7e7542da0b..9bd04a4b32 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -1,5 +1,6 @@ // npx vitest services/code-index/processors/__tests__/file-watcher.spec.ts +import nodePath from "path" import * as vscode from "vscode" import { FileWatcher } from "../file-watcher" @@ -419,7 +420,7 @@ describe("FileWatcher", () => { }) await runChange() expect(failures.list()).toEqual([ - expect.objectContaining({ path: "src/file.ts", activity, reason: "bad" }), + expect.objectContaining({ path: nodePath.normalize("src/file.ts"), activity, reason: "bad" }), ]) expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() }, diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index c462f5bee1..9b1e5e8caa 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -5,6 +5,7 @@ import { ParserLoadError } from "../parser" import { FileFailureRegistry } from "../../file-failure-registry" import { ProviderPacerRegistry } from "../../provider-pacer" import { stat } from "fs/promises" +import nodePath from "path" import * as vscode from "vscode" // Mock TelemetryService @@ -576,7 +577,9 @@ describe("DirectoryScanner", () => { await parseScanner.scanDirectory("/test") - expect(failures.list()).toEqual([expect.objectContaining({ path: "test/file1.js", activity: "parse" })]) + expect(failures.list()).toEqual([ + expect.objectContaining({ path: nodePath.normalize("test/file1.js"), activity: "parse" }), + ]) expect(mockCacheManager.updateHash).not.toHaveBeenCalledWith("test/file1.js", expect.anything()) expect(mockCacheManager.updateHash).toHaveBeenCalledWith("test/file2.js", expect.any(String)) }, From fb682aaa880b818cddc8f22b203ffb27dcf1908b Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 18:54:15 -0800 Subject: [PATCH 29/42] fix(code-index): type failedFileSummary as optional and complete status test doubles --- src/__tests__/extension.spec.ts | 6 ++++++ .../workspace-indexing-enablement-manager.spec.ts | 3 +++ src/services/code-index/manager.ts | 7 +++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/__tests__/extension.spec.ts b/src/__tests__/extension.spec.ts index abcd3c28a0..28dba2649e 100644 --- a/src/__tests__/extension.spec.ts +++ b/src/__tests__/extension.spec.ts @@ -388,6 +388,9 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() @@ -434,6 +437,9 @@ describe("extension.ts", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: true, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], } scope["statusManager"]!["publishStatus"](status) await Promise.resolve() diff --git a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts index 838fffdc6e..f3a247096a 100644 --- a/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts +++ b/src/services/code-index/__tests__/workspace-indexing-enablement-manager.spec.ts @@ -21,6 +21,9 @@ describe("WorkspaceIndexingEnablementManager", () => { workspacePath: "/workspace", workspaceEnabled: true, autoEnableDefault: false, + failedFiles: [], + failedFileSummary: undefined, + providerWaits: [], }), } return { diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 789ec8c9b9..668ecc1477 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -16,7 +16,7 @@ import fs from "fs/promises" import ignore from "ignore" import path from "path" import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" +import { TelemetryEventName, type FailedFileSummary } from "@roo-code/types" import { t } from "../../i18n" export class CodeIndexManager { @@ -368,7 +368,10 @@ export class CodeIndexManager { state: failure.state, })) const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles[0] - const failedFileSummary = representative && { ...representative, count: failedFiles.length } + const failedFileSummary: FailedFileSummary | undefined = representative && { + ...representative, + count: failedFiles.length, + } const providerWaits = this.pacers.providerWaits() let systemStatus: IndexingState = status.systemStatus let message = status.message From e12706ac005253d75c5e1d2acf1b8bdd62dbdec3 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 20:00:17 -0800 Subject: [PATCH 30/42] fix(code-index): keep failedFileSummary optional in status type --- src/services/code-index/manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 668ecc1477..7946357f5c 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -367,7 +367,7 @@ export class CodeIndexManager { reason: failure.reason, state: failure.state, })) - const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles[0] + const representative = failedFiles.find((failure) => failure.state === "retrying") ?? failedFiles.at(0) const failedFileSummary: FailedFileSummary | undefined = representative && { ...representative, count: failedFiles.length, From 338c05803ca7b77b13faddf9f43c7c072edb428f Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 21:36:04 -0800 Subject: [PATCH 31/42] fix(code-index): settle watcher status, classify Qdrant 429s, clear recovered failures and serialize watcher batches --- .../__tests__/CodebaseSearchTool.spec.ts | 2 +- .../__tests__/webviewMessageHandler.spec.ts | 38 ++++++++-- src/core/webview/webviewMessageHandler.ts | 4 +- .../code-index-scan-executor.spec.ts | 1 + .../__tests__/file-index-writer.spec.ts | 29 +++++++- .../__tests__/manager.reliability.spec.ts | 24 ++++--- .../code-index/__tests__/orchestrator.spec.ts | 71 +++++++++++++++++++ .../__tests__/provider-pacer.spec.ts | 38 +++++++++- .../__tests__/search-service.spec.ts | 22 +++++- src/services/code-index/file-index-writer.ts | 8 ++- src/services/code-index/manager.ts | 2 + src/services/code-index/orchestrator.ts | 4 ++ .../processors/__tests__/file-watcher.spec.ts | 69 ++++++++++++++++++ .../processors/__tests__/parser.spec.ts | 10 +-- .../processors/__tests__/scanner.spec.ts | 26 +++++++ .../code-index/processors/file-preparation.ts | 13 ++-- .../code-index/processors/file-watcher.ts | 47 ++++++++---- src/services/code-index/processors/scanner.ts | 2 + src/services/code-index/provider-pacer.ts | 3 +- src/services/code-index/search-service.ts | 15 ++-- .../__tests__/qdrant-client.spec.ts | 20 +++++- .../code-index/vector-store/qdrant-client.ts | 15 ++-- 22 files changed, 405 insertions(+), 58 deletions(-) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts index 45decee1c6..b8889b5d0b 100644 --- a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -295,7 +295,7 @@ describe("CodebaseSearchTool", () => { }, ) - it.each([false, true])("warns about an incomplete index with matches: %s", async (hasMatches) => { + it.each([false, true])("CI-TL-01: warns about an incomplete index with matches: %s", async (hasMatches) => { const failedPaths = ["src/failed.ts", "other/unreadable.ts"] vi.mocked(manager.searchIndex).mockResolvedValue( searchResponse(hasMatches ? [result()] : [], failedPaths, [], true), diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index 5091dc5676..9007488c08 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -807,23 +807,51 @@ describe("CI-WV-01: code index start and status forwarding", () => { beforeEach(() => vi.clearAllMocks()) - it.each(["Standby", "Error"])("starts indexing once from %s", async (state) => { + it.each(["Standby", "Error"])("starts indexing once from %s and awaits the start", async (state) => { const { manager, provider } = withManager(state) - await webviewMessageHandler(provider, { type: "startIndexing" }) + let finishStart!: () => void + manager.startIndexing.mockReturnValue(new Promise((resolve) => (finishStart = resolve))) + let handled = false + const handling = webviewMessageHandler(provider, { type: "startIndexing" }).then(() => (handled = true)) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(manager.startIndexing).toHaveBeenCalledTimes(1) + expect(handled).toBe(false) + finishStart() + await handling + expect(handled).toBe(true) expect(manager.startIndexing).toHaveBeenCalledTimes(1) }) + it("logs a start failure instead of rejecting", async () => { + const { manager, provider } = withManager("Standby") + manager.startIndexing.mockRejectedValue(new Error("boom")) + await expect(webviewMessageHandler(provider, { type: "startIndexing" })).resolves.toBeUndefined() + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: boom") + }) + it("does not start again while indexing is active", async () => { const { manager, provider } = withManager("Indexing") await webviewMessageHandler(provider, { type: "startIndexing" }) expect(manager.startIndexing).not.toHaveBeenCalled() }) - it("forwards failed files and provider waits after a stop", async () => { + it("forwards failed files and provider waits only after the stop finishes", async () => { const { manager, provider } = withManager("Indexing") - await webviewMessageHandler(provider, { type: "stopIndexing" }) + let finishStop!: () => void + manager.stopIndexing.mockReturnValue(new Promise((resolve) => (finishStop = resolve))) + manager.getCurrentStatus.mockReturnValue({ ...status, systemStatus: "Stopping" }) + const handling = webviewMessageHandler(provider, { type: "stopIndexing" }) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(provider.postMessageToWebview).not.toHaveBeenCalled() + + manager.getCurrentStatus.mockReturnValue(status) + finishStop() + await handling expect(manager.stopIndexing).toHaveBeenCalledTimes(1) - expect(provider.postMessageToWebview).toHaveBeenCalledWith({ type: "indexingStatusUpdate", values: status }) + expect(provider.postMessageToWebview).toHaveBeenCalledExactlyOnceWith({ + type: "indexingStatusUpdate", + values: status, + }) }) }) diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 590d5a8acc..d292c7c24f 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3530,12 +3530,12 @@ export const webviewMessageHandler = async ( const currentState = manager.state if (currentState === "Standby" || currentState === "Error") { - void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + await manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) if (!manager.isInitialized) { await manager.initialize(provider.contextProxy) if (manager.state === "Standby" || manager.state === "Error") { - void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + await manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) } } } diff --git a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts index d9a0e39f8d..3fae7f607b 100644 --- a/src/services/code-index/__tests__/code-index-scan-executor.spec.ts +++ b/src/services/code-index/__tests__/code-index-scan-executor.spec.ts @@ -110,6 +110,7 @@ describe("CodeIndexScanExecutor", () => { const scanCases = (["runFullScan", "runIncrementalScan"] as const).flatMap((method) => [ { found: 0, indexed: 0, errors: 0, partial: false }, + { found: 0, indexed: 0, errors: 1, partial: true }, { found: 3, indexed: 3, errors: 0, partial: false }, { found: 3, indexed: 0, errors: 3, partial: true }, { found: 10, indexed: 1, errors: 9, partial: true }, diff --git a/src/services/code-index/__tests__/file-index-writer.spec.ts b/src/services/code-index/__tests__/file-index-writer.spec.ts index 5e2c071fb3..20ab49f337 100644 --- a/src/services/code-index/__tests__/file-index-writer.spec.ts +++ b/src/services/code-index/__tests__/file-index-writer.spec.ts @@ -115,11 +115,38 @@ describe("FileIndexWriter", () => { controller.abort() throw serverError() }) - await expect(writer.replace("/ws/a.ts", "h1", [block(1)], controller.signal)).rejects.toBeDefined() + await expect(writer.replace("/ws/a.ts", "h1", [block(1)], controller.signal)).rejects.toBe( + controller.signal.reason, + ) expect(cache.updateHash).not.toHaveBeenCalled() expect(failures.list()).toEqual([]) }) + it("drops the old hash once its points are deleted, so a failed replacement is re-indexed", async () => { + const { writer, embedder, store, cache } = setup() + embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("bad"), { status: 400 })) + await expect(writer.replace("/ws/a.ts", "h1", [block(1)])).resolves.toBe(false) + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(cache.deleteHash).toHaveBeenCalledWith("/ws/a.ts") + expect(cache.updateHash).not.toHaveBeenCalled() + }) + + it("deletes partial points from a failed first attempt that stored no hash", async () => { + const { writer, store, cache, failures } = setup() + cache.getHash.mockReturnValue(undefined) + store.upsertPoints + .mockResolvedValueOnce(undefined) + .mockRejectedValue(Object.assign(new Error("bad"), { status: 400 })) + const blocks = Array.from({ length: 4 }, (_unused, index) => block(index + 1)) + await expect(writer.replace("/ws/a.ts", "h1", blocks)).resolves.toBe(false) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(failures.list()).toHaveLength(1) + + store.upsertPoints.mockResolvedValue(undefined) + await expect(writer.replace("/ws/a.ts", "h2", [block(1)])).resolves.toBe(true) + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + }) + it("removal success deletes the hash and clears prior failures", async () => { const { writer, cache, failures } = setup() failures.renew("a.ts", "h1") diff --git a/src/services/code-index/__tests__/manager.reliability.spec.ts b/src/services/code-index/__tests__/manager.reliability.spec.ts index 01c66f4f74..a8cc11a338 100644 --- a/src/services/code-index/__tests__/manager.reliability.spec.ts +++ b/src/services/code-index/__tests__/manager.reliability.spec.ts @@ -24,6 +24,13 @@ vi.mock("vscode", () => { } }) vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: { captureEvent: vi.fn() } } })) +const replacementOrder = vi.hoisted(() => [] as string[]) +vi.mock("../semble", () => ({ + SembleProvider: vi.fn().mockImplementation(function () { + replacementOrder.push("recreate") + return { initialize: vi.fn(async () => {}), dispose: vi.fn(), stopIndexing: vi.fn() } + }), +})) vi.mock("../../../i18n", () => ({ t: (key: string) => key })) const workspacePath = path.join(path.sep, "test", "workspace") @@ -116,10 +123,14 @@ describe("CodeIndexManager reliability", () => { "CI-MG-02: %s stops the scan and watcher before services are recreated", async (entry) => { const { manager } = makeManager() - const order: string[] = [] + replacementOrder.length = 0 + // The stop settles a tick later, so replacement work that does not await it would run first. const orchestrator: OrchestratorDouble = { startIndexing: vi.fn(async () => {}), - stopIndexing: vi.fn(async () => void order.push("stop")), + stopIndexing: vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)) + replacementOrder.push("stop") + }), isScanning: false, state: "Indexed", } @@ -130,16 +141,11 @@ describe("CodeIndexManager reliability", () => { currentEmbedderProvider: "semble", loadConfiguration: async () => ({ requiresRestart: true }), } as unknown as (typeof manager)["_configManager"] - const recreate = vi.spyOn(manager as unknown as { _recreateServices(): Promise }, "_recreateServices") - recreate.mockImplementation(async () => { - await orchestrator.stopIndexing() - order.push("recreate") - }) await manager[entry]() - expect(order[0]).toBe("stop") - if (entry === "handleSettingsChange") expect(order).toEqual(["stop", "recreate"]) + expect(replacementOrder[0]).toBe("stop") + if (entry === "handleSettingsChange") expect(replacementOrder).toEqual(["stop", "recreate"]) else expect(manager["_orchestrator"]).toBeUndefined() }, ) diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 1e53939f02..8e47445b4a 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -84,6 +84,35 @@ function waitForAbort(...args: Parameters) { const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) +type Progress = { processedInBatch: number; totalInBatch: number; currentFile?: string } + +/** Wires the watcher double's three events to callable emitters. */ +function emitters(fileWatcher: ReturnType["fileWatcher"]) { + const listeners: { start?: () => void; progress?: (event: Progress) => void; finish?: () => void } = {} + fileWatcher.onDidStartBatchProcessing.mockImplementation((listener: () => void) => { + listeners.start = listener + return { dispose: vi.fn() } + }) + fileWatcher.onBatchProgressUpdate.mockImplementation((listener: (event: Progress) => void) => { + listeners.progress = listener + return { dispose: vi.fn() } + }) + fileWatcher.onDidFinishBatchProcessing.mockImplementation((listener: () => void) => { + listeners.finish = listener + return { dispose: vi.fn() } + }) + /** Same order as `FileWatcher.processBatch()`: start, progress, finish, then two trailing progress events. */ + return (total: number, duringBatch?: () => void) => { + listeners.start?.() + listeners.progress?.({ processedInBatch: 0, totalInBatch: total }) + duringBatch?.() + listeners.progress?.({ processedInBatch: total, totalInBatch: total, currentFile: "/w/a.ts" }) + listeners.finish?.() + listeners.progress?.({ processedInBatch: total, totalInBatch: total }) + listeners.progress?.({ processedInBatch: 0, totalInBatch: 0 }) + } +} + describe("CodeIndexOrchestrator", () => { beforeEach(() => clearAllMocks()) @@ -123,6 +152,48 @@ describe("CodeIndexOrchestrator", () => { expect(context.orchestrator.state).toBe("Indexed") }) + describe("watcher batch final state", () => { + function progressSetup() { + const context = setup() + context.stateManager.reportFileQueueProgress.mockImplementation(() => { + context.stateManager.setSystemState("Indexing") + }) + return { context, runBatch: emitters(context.fileWatcher) } + } + + it("ends in Indexed after batch-finished and the two trailing progress events", async () => { + const { context, runBatch } = progressSetup() + await context.orchestrator.startIndexing() + expect(context.orchestrator.state).toBe("Indexed") + runBatch(2) + expect(context.orchestrator.state).toBe("Indexed") + }) + + it("ends in partial Standby when the batch recorded a failure", async () => { + const { context, runBatch } = progressSetup() + await context.orchestrator.startIndexing() + runBatch(1, () => { + context.failures.renew("a.ts", "h") + context.failures.record("a.ts", "h", "embed", "down", 3, "exhausted") + }) + expect(context.orchestrator.state).toBe("Standby") + expect(context.stateManager.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingFilesFailed", + ) + }) + + it("reuses the watcher events after stop and restart", async () => { + const { context, runBatch } = progressSetup() + await context.orchestrator.startIndexing() + await context.orchestrator.stopIndexing() + await context.orchestrator.startIndexing() + runBatch(1) + expect(context.orchestrator.state).toBe("Indexed") + expect(context.fileWatcher.initialize).toHaveBeenCalledTimes(2) + }) + }) + it("does not clean up when initialize() fails and ends in Error", async () => { const context = setup() context.vectorStore.initialize.mockRejectedValue(new Error("Qdrant unreachable")) diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 512b15cba3..4f359c85d5 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -80,14 +80,46 @@ describe("ProviderPacer (CI-PP-02)", () => { await next }) - it("never shortens an existing pause", async () => { + it("never shortens an existing pause when a later 429 asks for less", async () => { const pacer = new ProviderPacer("openai:1") + let releaseShort: (() => void) | undefined + const shortRequest = vi + .fn() + .mockImplementationOnce( + () => new Promise((_resolve, reject) => (releaseShort = () => reject(rateLimit(1000)))), + ) + .mockResolvedValue("ok") + const short = settle(pacer.run(undefined, undefined, shortRequest)) const slow = settle(pacer.run(undefined, undefined, vi.fn().mockRejectedValueOnce(rateLimit(60_000)))) await vi.advanceTimersByTimeAsync(0) - const short = settle(pacer.run(undefined, undefined, vi.fn().mockResolvedValue("ok"))) expect(pacer.resumeAt).toBe(60_000) - await vi.advanceTimersByTimeAsync(60_000) + releaseShort?.() + await vi.advanceTimersByTimeAsync(0) + expect(pacer.resumeAt).toBe(60_000) + await vi.advanceTimersByTimeAsync(59_999) + expect(shortRequest).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) await Promise.all([slow, short]) + expect(shortRequest).toHaveBeenCalledTimes(2) + }) + + it("counts concurrent 429s during one pause as a single backoff step", async () => { + const pacer = new ProviderPacer("openai:1") + const releases: Array<() => void> = [] + const request = () => + vi + .fn() + .mockImplementationOnce( + () => new Promise((_resolve, reject) => releases.push(() => reject(rateLimit()))), + ) + .mockResolvedValue("ok") + const runs = [1, 2, 3, 4].map(() => settle(pacer.run(undefined, undefined, request()))) + await vi.advanceTimersByTimeAsync(0) + for (const release of releases) release() + await vi.advanceTimersByTimeAsync(0) + expect(pacer.resumeAt! - Date.now()).toBe(1000) + await vi.advanceTimersByTimeAsync(1000) + await Promise.all(runs) }) it("rejects a waiter at once with the abort reason and drops its path", async () => { diff --git a/src/services/code-index/__tests__/search-service.spec.ts b/src/services/code-index/__tests__/search-service.spec.ts index 49a157f14b..69543e0c56 100644 --- a/src/services/code-index/__tests__/search-service.spec.ts +++ b/src/services/code-index/__tests__/search-service.spec.ts @@ -42,7 +42,7 @@ function prepare(state: IndexingState, { failed = false, exists = true, enabled } const pacers = new ProviderPacerRegistry() const service = new CodeIndexSearchService(config, stateManager, embedder, store, failures, pacers) - return { service, failures, pacers, store, embedder, setSystemState } + return { service, failures, pacers, store, embedder, setSystemState, stateManager } } async function withWaitingPath(pacers: ProviderPacerRegistry, file: string) { @@ -117,6 +117,26 @@ describe("CodeIndexSearchService (CI-SS-01)", () => { }, ) + it("rejects when the state moves to Stopping during the collection lookup", async () => { + const { service, store, embedder, stateManager } = prepare("Standby", { failed: true }) + store.collectionExists.mockImplementation(async () => { + stateManager.setSystemState("Stopping", "stopping") + return true + }) + await expect(service.searchIndex("query")).rejects.toThrow("Current state: Stopping") + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("reports incomplete when indexing stops to Standby while the query is in flight", async () => { + const { service, store, stateManager } = prepare("Indexing") + store.search.mockImplementation(async () => { + stateManager.setSystemState("Standby", "stopped") + return [] + }) + const response = await service.searchIndex("query") + expect(response).toMatchObject({ incomplete: true, failedPaths: [], waitingPaths: [] }) + }) + it("does not set Error when a query fails", async () => { const { service, embedder, setSystemState } = prepare("Indexed") embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("bad"), { status: 400 })) diff --git a/src/services/code-index/file-index-writer.ts b/src/services/code-index/file-index-writer.ts index eb3cd22cd8..ac82580290 100644 --- a/src/services/code-index/file-index-writer.ts +++ b/src/services/code-index/file-index-writer.ts @@ -22,13 +22,19 @@ export class FileIndexWriter { /** Replaces the file's points with `blocks`; returns false when the file failed and was recorded. */ async replace(filePath: string, hash: string, blocks: CodeBlock[], signal?: AbortSignal): Promise { const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + // A failed earlier attempt may have upserted some batches before any hash was stored. + const mayHavePoints = + this.cache.getHash(filePath) !== undefined || + this.failures.list().some((failure) => failure.path === relativePath) this.failures.renew(relativePath, hash) const selected = blocks.filter((block) => block.content.trim()) try { - if (this.cache.getHash(filePath) !== undefined) { + if (mayHavePoints) { await this.attempt(relativePath, hash, "delete", this.store.rateLimitScope, signal, () => this.store.deletePointsByFilePath(filePath), ) + // The old points are gone, so the old hash must not let an unchanged file skip re-indexing. + if (!signal?.aborted) this.cache.deleteHash(filePath) } for (let offset = 0; offset < selected.length; offset += this.batchSize) { const batch = selected.slice(offset, offset + this.batchSize) diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 7946357f5c..dc668ee357 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -216,6 +216,8 @@ export class CodeIndexManager { } private startIndexingInBackground(): void { + // Initialization awaits the cache read; indexing may have been disabled meanwhile. + if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) return void this.startOrchestrator().catch((error) => { console.error("[CodeIndexManager] Background indexing failed:", error) }) diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 0151fedb51..e2700b2396 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -93,6 +93,10 @@ export class CodeIndexOrchestrator { totalInBatch, currentFile ? path.basename(currentFile) : undefined, ) + // The watcher reports progress after the batch-finished event; settle the status again. + if (processedInBatch === totalInBatch && !this.watcherBatchPending) { + this.updateRecoveryStatus(signal) + } }), this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { if (summary.batchError) { diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 9bd04a4b32..f3824ed678 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -453,6 +453,64 @@ describe("FileWatcher", () => { expect(summary.processedFiles).toEqual([{ path, status: "success" }]) }) + it("runs a later delete batch only after an earlier change batch finishes", async () => { + watcherWithRegistry() + await fileWatcher.initialize() + const order: string[] = [] + let finishUpsert!: () => void + mockVectorStore.upsertPoints.mockImplementation( + () => + new Promise((resolve) => { + finishUpsert = () => { + order.push("upsert") + resolve() + } + }), + ) + mockVectorStore.deletePointsByFilePath.mockImplementation(async () => void order.push("delete")) + mockCacheManager.getHash.mockReturnValue(undefined) + await mockOnDidChange({ fsPath: path }) + await flushBatch() + await mockOnDidDelete({ fsPath: path }) + await flushBatch() + expect(order).toEqual([]) + finishUpsert() + await vi.advanceTimersByTimeAsync(0) + expect(order).toEqual(["upsert", "delete"]) + }) + + it("clears a stale read failure when the content is unchanged", async () => { + const failures = watcherWithRegistry() + const relativePath = nodePath.normalize("src/file.ts") + failures.renew(relativePath, "unread") + failures.record(relativePath, "unread", "read", "busy", 1, "exhausted") + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ + path, + status: "skipped", + reason: "File has not changed", + }) + await runChange() + expect(failures.list()).toEqual([]) + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + }) + + it.each([ + ["File is too large", "h", 1], + ["File is ignored by .rooignore or .gitignore", "h", 1], + ["File is too large", undefined, 0], + ])("a change skipped as %s with cached hash %s removes old points %i times", async (reason, hash, removals) => { + watcherWithRegistry() + mockCacheManager.getHash.mockReturnValue(hash) + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ + path, + status: "skipped", + reason, + }) + await runChange() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledTimes(removals) + expect(mockCacheManager.deleteHash).toHaveBeenCalledTimes(removals) + }) + it("writes nothing after the session signal aborts", async () => { watcherWithRegistry() const controller = new AbortController() @@ -472,5 +530,16 @@ describe("FileWatcher", () => { expect(mockWatcher.dispose).toHaveBeenCalled() }) + + it("keeps batch events working after dispose and re-initialize (restart)", async () => { + await fileWatcher.initialize() + fileWatcher.dispose() + await fileWatcher.initialize() + const batchPromise = waitForNextBatch() + await mockOnDidChange({ fsPath: "/mock/workspace/src/file.ts" }) + await flushBatch() + const summary = await batchPromise + expect(summary.processedFiles).toEqual([{ path: "/mock/workspace/src/file.ts", status: "success" }]) + }) }) }) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 864025d732..39acdeb5a4 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -322,10 +322,12 @@ describe("CodeParser", () => { describe("singleton instance", () => { it("should maintain parser state across calls", async () => { - const result1 = await codeParser.parseFile("test.js", { content: "const a = 1" }) - const result2 = await codeParser.parseFile("test.js", { content: "const b = 2" }) - expect(result1).toBeDefined() - expect(result2).toBeDefined() + const first = "const firstValue = 'singleton parser first input with enough text'" + const second = "const secondValue = 'singleton parser second input with enough text'" + const result1 = await codeParser.parseFile("test.js", { content: first }) + const result2 = await codeParser.parseFile("test.js", { content: second }) + expect(result1.map((block) => block.content)).toEqual([first]) + expect(result2.map((block) => block.content)).toEqual([second]) }) }) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 9b1e5e8caa..dc88949993 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -5,6 +5,7 @@ import { ParserLoadError } from "../parser" import { FileFailureRegistry } from "../../file-failure-registry" import { ProviderPacerRegistry } from "../../provider-pacer" import { stat } from "fs/promises" +import { createHash } from "crypto" import nodePath from "path" import * as vscode from "vscode" @@ -543,6 +544,31 @@ describe("DirectoryScanner", () => { expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() }) + it("clears a read failure once the read succeeds with unchanged content", async () => { + const failures = new FileFailureRegistry() + const recoveryScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/cached.js"], false]) + mockCacheManager.getHash.mockReturnValue(createHash("sha256").update("test content").digest("hex")) + vi.mocked(stat).mockRejectedValueOnce(Object.assign(new Error("busy"), { code: "EBUSY" })) + + await recoveryScanner.scanDirectory("/test") + expect(failures.list()).toEqual([expect.objectContaining({ activity: "read" })]) + + const result = await recoveryScanner.scanDirectory("/test") + expect(result.stats.skipped).toBe(1) + expect(failures.list()).toEqual([]) + }) + it.each([ ["parser load", new ParserLoadError(".js", new Error("wasm"))], ["parse", new Error("syntax")], diff --git a/src/services/code-index/processors/file-preparation.ts b/src/services/code-index/processors/file-preparation.ts index 1f67e58b7a..50d8858b65 100644 --- a/src/services/code-index/processors/file-preparation.ts +++ b/src/services/code-index/processors/file-preparation.ts @@ -7,6 +7,11 @@ import { generateRelativeFilePath } from "../shared/get-relative-path" import { readSource, SOURCE_MISSING } from "../shared/indexing-errors" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" +export const SKIP_UNCHANGED = "File has not changed" +export const SKIP_IGNORED_DIRECTORY = "File is in an ignored directory" +export const SKIP_IGNORED = "File is ignored by .rooignore or .gitignore" +export const SKIP_TOO_LARGE = "File is too large" + /** Prepares one file for batching without embedding, writing points or mutating the hash cache. */ export class FilePreparation { constructor(private readonly dependencies: FilePreparationDependencies) {} @@ -29,7 +34,7 @@ export class FilePreparation { try { const newHash = createHash("sha256").update(content).digest("hex") if (dependencies.cacheManager.getHash(filePath) === newHash) { - return this.skippedResult(filePath, "File has not changed") + return this.skippedResult(filePath, SKIP_UNCHANGED) } const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash }) return { path: filePath, status: "processed_for_batching", newHash, blocks } @@ -43,20 +48,20 @@ export class FilePreparation { // Use relative paths so ignored directories outside the workspace do not exclude the file. const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath) if (isPathInIgnoredDirectory(relativeFilePath)) { - return "File is in an ignored directory" + return SKIP_IGNORED_DIRECTORY } if ( !dependencies.ignoreController.validateAccess(filePath) || dependencies.ignoreInstance?.ignores(relativeFilePath) ) { - return "File is ignored by .rooignore or .gitignore" + return SKIP_IGNORED } const fileStat = await readSource(dependencies.fileSystem.stat(Uri.file(filePath))) if (fileStat === SOURCE_MISSING) return SOURCE_MISSING if (fileStat.size > MAX_FILE_SIZE_BYTES) { - return "File is too large" + return SKIP_TOO_LARGE } return undefined } diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 254085ab6a..ec21117512 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -5,7 +5,7 @@ import { Ignore } from "ignore" import { scannerExtensions } from "../shared/supported-extensions" import { IFileWatcher, FileProcessingResult, IEmbedder, IVectorStore, BatchProcessingSummary } from "../interfaces" import { codeParser } from "./parser" -import { FilePreparation } from "./file-preparation" +import { FilePreparation, SKIP_UNCHANGED } from "./file-preparation" import { CacheManager } from "../cache-manager" import { Package } from "../../../shared/package" import { type FileFailureRegistry, UNKNOWN_REVISION } from "../file-failure-registry" @@ -24,6 +24,7 @@ export class FileWatcher implements IFileWatcher { private fileWatcher?: vscode.FileSystemWatcher private ignoreController: RooIgnoreController private accumulatedEvents: Map = new Map() + private batchQueue: Promise = Promise.resolve() private batchProcessDebounceTimer?: NodeJS.Timeout private readonly BATCH_DEBOUNCE_DELAY_MS = 500 private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10 @@ -130,16 +131,15 @@ export class FileWatcher implements IFileWatcher { } /** - * Disposes the file watcher + * Stops watching. The event emitters stay usable because the orchestrator reuses this watcher on restart; + * it disposes its own subscriptions. */ dispose(): void { this.fileWatcher?.dispose() + this.fileWatcher = undefined if (this.batchProcessDebounceTimer) { clearTimeout(this.batchProcessDebounceTimer) } - this._onDidStartBatchProcessing.dispose() - this._onBatchProgressUpdate.dispose() - this._onDidFinishBatchProcessing.dispose() this.accumulatedEvents.clear() } @@ -194,22 +194,28 @@ export class FileWatcher implements IFileWatcher { const filePathsInBatch = Array.from(eventsToProcess.keys()) this._onDidStartBatchProcessing.fire(filePathsInBatch) - await this.processBatch(eventsToProcess) + // Batches run one at a time so a later change or delete of a path cannot finish before an earlier one. + // Each batch keeps its own session signal, so a restart's new signal never authorizes a stale write. + const signal = this.signal + this.batchQueue = this.batchQueue + .then(() => this.processBatch(eventsToProcess, signal)) + .catch((error) => console.error("[FileWatcher] Batch processing failed:", error)) + await this.batchQueue } - private async removeFile(filePath: string): Promise { + private async removeFile(filePath: string, signal: AbortSignal | undefined): Promise { if (!this.writer) return { path: filePath, status: "success" } - const removed = await this.writer.remove(filePath, this.signal) + const removed = await this.writer.remove(filePath, signal) return removed ? { path: filePath, status: "success" } : { path: filePath, status: "error", error: new Error(`Failed to remove ${filePath} from the index`) } } - private async indexFile(filePath: string): Promise { + private async indexFile(filePath: string, signal: AbortSignal | undefined): Promise { const result = await this.processFile(filePath) - if (result.status === "missing") return this.removeFile(filePath) + if (result.status === "missing") return this.removeFile(filePath, signal) if (result.status === "local_error") { - if (!this.signal?.aborted) { + if (!signal?.aborted) { const relativePath = generateRelativeFilePath(filePath, this.workspacePath) const revision = result.newHash ?? UNKNOWN_REVISION const reason = result.error instanceof Error ? result.error.message : String(result.error) @@ -218,8 +224,18 @@ export class FileWatcher implements IFileWatcher { } return result } + if (result.status === "skipped") { + if (result.reason === SKIP_UNCHANGED) { + // Content matches the stored hash, so the file is indexed; an earlier read failure is stale. + if (!signal?.aborted) this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath)) + return result + } + // A file that became ignored or too large must not keep its old points. + if (this.cacheManager.getHash(filePath) !== undefined) return this.removeFile(filePath, signal) + return result + } if (result.status !== "processed_for_batching" || !result.newHash || !this.writer) return result - const indexed = await this.writer.replace(filePath, result.newHash, result.blocks ?? [], this.signal) + const indexed = await this.writer.replace(filePath, result.newHash, result.blocks ?? [], signal) return indexed ? { path: filePath, status: "success" } : { path: filePath, status: "error", error: new Error(`Failed to index ${filePath}`) } @@ -227,6 +243,7 @@ export class FileWatcher implements IFileWatcher { private async processBatch( eventsToProcess: Map, + signal: AbortSignal | undefined, ): Promise { const batchResults: FileProcessingResult[] = [] const totalFilesInBatch = eventsToProcess.size @@ -239,15 +256,15 @@ export class FileWatcher implements IFileWatcher { const events = [...eventsToProcess.values()] for (let offset = 0; offset < events.length; offset += this.FILE_PROCESSING_CONCURRENCY_LIMIT) { - if (this.signal?.aborted) break + if (signal?.aborted) break const chunk = events.slice(offset, offset + this.FILE_PROCESSING_CONCURRENCY_LIMIT) const results = await Promise.all( chunk.map(async (event) => { const filePath = event.uri.fsPath try { return event.type === "delete" - ? await this.removeFile(filePath) - : await this.indexFile(filePath) + ? await this.removeFile(filePath, signal) + : await this.indexFile(filePath, signal) } catch (error) { return { path: filePath, status: "error", error: error as Error } satisfies FileProcessingResult } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index c6adc50342..a5ccc948ae 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -159,6 +159,8 @@ export class DirectoryScanner implements IDirectoryScanner { processedFiles.add(filePath) if (this.cacheManager.getHash(filePath) === currentFileHash) { skippedCount++ + // The stored hash proves the file is indexed; an earlier read failure is stale. + if (!signal?.aborted) this.failures.clear(generateRelativeFilePath(filePath, scanWorkspace)) return } diff --git a/src/services/code-index/provider-pacer.ts b/src/services/code-index/provider-pacer.ts index 6ddcdf9eec..fe04441ba8 100644 --- a/src/services/code-index/provider-pacer.ts +++ b/src/services/code-index/provider-pacer.ts @@ -94,7 +94,8 @@ export class ProviderPacer { private pause(error: unknown): void { const now = Date.now() - this.consecutiveRateLimits++ + // Concurrent 429s from requests sent before the pause belong to the same episode. + if (now >= this.pausedUntil) this.consecutiveRateLimits++ const requested = new EmbeddingError("Provider rate limited", error).retryAfterMs const delay = typeof requested === "number" && requested > 0 && Number.isFinite(new Date(now + requested).getTime()) diff --git a/src/services/code-index/search-service.ts b/src/services/code-index/search-service.ts index 30339a81d9..308d29aa02 100644 --- a/src/services/code-index/search-service.ts +++ b/src/services/code-index/search-service.ts @@ -50,10 +50,14 @@ export class CodeIndexSearchService { const minScore = this.configManager.currentSearchMinScore const maxResults = this.configManager.currentSearchMaxResults - const currentState = this.stateManager.getCurrentStatus().systemStatus + const initialState = this.stateManager.getCurrentStatus().systemStatus const partialStandby = - currentState === "Standby" && this.hasOutstandingWork() && (await this.vectorStore.collectionExists()) - if (currentState !== "Indexed" && currentState !== "Indexing" && !partialStandby) { + initialState === "Standby" && this.hasOutstandingWork() && (await this.vectorStore.collectionExists()) + // The collection lookup is awaited, so a stop or error may have started meanwhile. + const currentState = this.stateManager.getCurrentStatus().systemStatus + const ready = + currentState === "Indexed" || currentState === "Indexing" || (partialStandby && currentState === "Standby") + if (!ready) { throw new Error(`Code index is not ready for search. Current state: ${currentState}`) } @@ -84,10 +88,11 @@ export class CodeIndexSearchService { const results = await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults) const failedPaths = [...new Set(this.failures.list().map((failure) => failure.path))] const waitingPaths = this.providerPacers.waitingPaths() - const scanning = this.stateManager.getCurrentStatus().systemStatus === "Indexing" + // Only Indexed proves the scan finished; a stop during the query leaves Standby without that proof. + const complete = this.stateManager.getCurrentStatus().systemStatus === "Indexed" return { results, - incomplete: scanning || failedPaths.length > 0 || waitingPaths.length > 0, + incomplete: !complete || failedPaths.length > 0 || waitingPaths.length > 0, failedPaths, waitingPaths, } diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index 4f50eb5e84..e2f23815a7 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -1,4 +1,4 @@ -import { QdrantClient } from "@qdrant/js-client-rest" +import { QdrantClient, QdrantClientResourceExhaustedError } from "@qdrant/js-client-rest" import { createHash } from "crypto" import { QdrantVectorStore } from "../qdrant-client" @@ -839,6 +839,24 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(2) }) + it.each([ + ["with Retry-After: 2", 2, 2000], + ["without Retry-After", undefined, 1000], + ])("waits and recovers when collection creation gets a 429 %s", async (_label, retryAfter, waitMs) => { + // The module is automocked, so set the fields the patched client would assign. + const limited = Object.assign(new QdrantClientResourceExhaustedError("Too many requests", 0), { + message: "Too many requests", + retry_after: retryAfter, + }) + mockQdrantClientInstance.createCollection.mockRejectedValueOnce(limited) + const pending = vectorStore.initialize() + await vitest.advanceTimersByTimeAsync(waitMs - 1) + expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) + await vitest.advanceTimersByTimeAsync(1) + await expect(pending).resolves.toBe(true) + expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(2) + }) + it("treats an existing payload index as success", async () => { mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(new Error("index already exists")) await expect(vectorStore.initialize()).resolves.toBe(true) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 25ee35e560..084b7c0b31 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -112,11 +112,16 @@ export class QdrantVectorStore implements IVectorStore { /** Runs one creation request with bounded server/connection retries and the shared rate-limit wait. */ private async create(request: () => Promise, signal?: AbortSignal): Promise { - try { - return await retryWithPolicy(() => this.pacers.get(this.rateLimitScope).run(undefined, signal, request), { - signal, - budget: { retriesUsed: 0 }, + // Only the 429 is converted; other errors keep initialize()'s connection and dimension-mismatch messages. + const classified = () => + request().catch((error: unknown) => { + throw error instanceof QdrantClientResourceExhaustedError ? toVectorStoreError(error) : error }) + try { + return await retryWithPolicy( + () => this.pacers.get(this.rateLimitScope).run(undefined, signal, classified), + { signal, budget: { retriesUsed: 0 } }, + ) } catch (error) { // The original error keeps initialize()'s existing connection and dimension-mismatch messages. throw error instanceof IndexingRetryError ? error.cause : error @@ -498,7 +503,7 @@ export class QdrantVectorStore implements IVectorStore { return filteredPoints as VectorStoreSearchResult[] } catch (error) { console.error("Failed to search points:", error) - throw error + throw error instanceof QdrantClientResourceExhaustedError ? toVectorStoreError(error) : error } } From 413ba2f0fbb51fb8741e376b4ba325aad7e70cd5 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 21:41:57 -0800 Subject: [PATCH 32/42] fix(i18n): translate code index cache and quota errors --- src/i18n/locales/ca/embeddings.json | 4 ++-- src/i18n/locales/de/embeddings.json | 4 ++-- src/i18n/locales/es/embeddings.json | 4 ++-- src/i18n/locales/fr/embeddings.json | 4 ++-- src/i18n/locales/hi/embeddings.json | 4 ++-- src/i18n/locales/id/embeddings.json | 4 ++-- src/i18n/locales/it/embeddings.json | 4 ++-- src/i18n/locales/ja/embeddings.json | 4 ++-- src/i18n/locales/ko/embeddings.json | 4 ++-- src/i18n/locales/nl/embeddings.json | 4 ++-- src/i18n/locales/pl/embeddings.json | 4 ++-- src/i18n/locales/pt-BR/embeddings.json | 4 ++-- src/i18n/locales/ru/embeddings.json | 4 ++-- src/i18n/locales/tr/embeddings.json | 4 ++-- src/i18n/locales/vi/embeddings.json | 4 ++-- src/i18n/locales/zh-CN/embeddings.json | 4 ++-- src/i18n/locales/zh-TW/embeddings.json | 4 ++-- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/i18n/locales/ca/embeddings.json b/src/i18n/locales/ca/embeddings.json index 87045e30f7..225925dc55 100644 --- a/src/i18n/locales/ca/embeddings.json +++ b/src/i18n/locales/ca/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Indexació requereix una carpeta de workspace oberta", "indexingStopped": "Indexació aturada per l'usuari.", "indexingStoppedPartial": "Indexació aturada. Dades d'índex parcials conservades.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "No s'ha pogut llegir la memòria cau de l'índex de codi. S'han conservat les dades de l'índex existent. Torna a iniciar la indexació per tornar-ho a provar." }, "semble": { "downloadingBinary": "Descarregant el binari de semble...", @@ -80,5 +80,5 @@ "checkFailed": "Comprovació de semble fallida: {{errorMessage}}", "providerReset": "Proveïdor de Semble restablert. La memòria cau al disc es conserva fins a la propera reconstrucció." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "S'ha arribat a la quota o al límit de facturació del proveïdor d'incrustacions. Revisa el teu pla o la facturació i torna a iniciar la indexació." } diff --git a/src/i18n/locales/de/embeddings.json b/src/i18n/locales/de/embeddings.json index a9891c4952..3b94542729 100644 --- a/src/i18n/locales/de/embeddings.json +++ b/src/i18n/locales/de/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Indexierung erfordert einen offenen Workspace-Ordner", "indexingStopped": "Indexierung vom Benutzer gestoppt.", "indexingStoppedPartial": "Indexierung gestoppt. Teilweise Indexdaten beibehalten.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Der Cache des Code-Index konnte nicht gelesen werden. Vorhandene Indexdaten wurden beibehalten. Starte die Indexierung erneut, um es noch einmal zu versuchen." }, "semble": { "downloadingBinary": "Semble-Binary wird heruntergeladen...", @@ -80,5 +80,5 @@ "checkFailed": "Semble-Überprüfung fehlgeschlagen: {{errorMessage}}", "providerReset": "Semble-Anbieter zurückgesetzt. Der Cache auf der Festplatte bleibt bis zum nächsten Neuaufbau erhalten." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Kontingent oder Abrechnungslimit des Einbettungsanbieters erreicht. Überprüfe deinen Tarif oder deine Abrechnung und starte die Indexierung erneut." } diff --git a/src/i18n/locales/es/embeddings.json b/src/i18n/locales/es/embeddings.json index dbce931bed..62cec8343e 100644 --- a/src/i18n/locales/es/embeddings.json +++ b/src/i18n/locales/es/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "La indexación requiere una carpeta de workspace abierta", "indexingStopped": "Indexación detenida por el usuario.", "indexingStoppedPartial": "Indexación detenida. Datos de índice parciales conservados.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "No se pudo leer la caché del índice de código. Se conservaron los datos del índice existente. Vuelve a iniciar la indexación para reintentarlo." }, "semble": { "downloadingBinary": "Descargando el binario de semble...", @@ -80,5 +80,5 @@ "checkFailed": "Verificación de semble fallida: {{errorMessage}}", "providerReset": "Proveedor de Semble restablecido. La caché en disco se conserva hasta la próxima reconstrucción." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Se ha alcanzado la cuota o el límite de facturación del proveedor de incrustaciones. Comprueba tu plan o la facturación y vuelve a iniciar la indexación." } diff --git a/src/i18n/locales/fr/embeddings.json b/src/i18n/locales/fr/embeddings.json index 5f81c08006..9cb920d34a 100644 --- a/src/i18n/locales/fr/embeddings.json +++ b/src/i18n/locales/fr/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "L'indexation nécessite l'ouverture d'un dossier workspace", "indexingStopped": "Indexation arrêtée par l'utilisateur.", "indexingStoppedPartial": "Indexation arrêtée. Données d'index partielles conservées.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Impossible de lire le cache de l'index du code. Les données de l'index existant ont été conservées. Relance l'indexation pour réessayer." }, "semble": { "downloadingBinary": "Téléchargement du binaire semble...", @@ -80,5 +80,5 @@ "checkFailed": "Vérification de semble échouée : {{errorMessage}}", "providerReset": "Fournisseur Semble réinitialisé. Le cache sur disque reste jusqu'à la prochaine reconstruction." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Le quota ou la limite de facturation du fournisseur d'embeddings a été atteint. Vérifie ton forfait ou ta facturation, puis relance l'indexation." } diff --git a/src/i18n/locales/hi/embeddings.json b/src/i18n/locales/hi/embeddings.json index ccdad44e1f..9b54b240bb 100644 --- a/src/i18n/locales/hi/embeddings.json +++ b/src/i18n/locales/hi/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "इंडेक्सिंग के लिए एक खुला वर्कस्पेस फ़ोल्डर आवश्यक है", "indexingStopped": "उपयोगकर्ता द्वारा इंडेक्सिंग रोकी गई।", "indexingStoppedPartial": "इंडेक्सिंग रोकी गई। आंशिक इंडेक्स डेटा संरक्षित।", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "कोड इंडेक्स कैश पढ़ा नहीं जा सका। मौजूदा इंडेक्स डेटा सुरक्षित रखा गया है। फिर से कोशिश करने के लिए इंडेक्सिंग दोबारा शुरू करो।" }, "semble": { "downloadingBinary": "semble बाइनरी डाउनलोड हो रहा है...", @@ -80,5 +80,5 @@ "checkFailed": "Semble जाँच विफल: {{errorMessage}}", "providerReset": "Semble प्रदाता रीसेट किया गया। डिस्क कैश अगले रीबिल्ड तक बना रहता है।" }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "एम्बेडिंग प्रदाता का कोटा या बिलिंग सीमा पूरी हो गई है। अपना प्लान या बिलिंग जाँचो, फिर इंडेक्सिंग दोबारा शुरू करो।" } diff --git a/src/i18n/locales/id/embeddings.json b/src/i18n/locales/id/embeddings.json index 27e68b0ffd..aae9e6b5eb 100644 --- a/src/i18n/locales/id/embeddings.json +++ b/src/i18n/locales/id/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Pengindeksan memerlukan folder workspace yang terbuka", "indexingStopped": "Pengindeksan dihentikan oleh pengguna.", "indexingStoppedPartial": "Pengindeksan dihentikan. Data indeks parsial dipertahankan.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Tidak dapat membaca cache indeks kode. Data indeks yang ada tetap disimpan. Mulai pengindeksan lagi untuk mencoba ulang." }, "semble": { "downloadingBinary": "Mengunduh biner semble...", @@ -80,5 +80,5 @@ "checkFailed": "Pemeriksaan semble gagal: {{errorMessage}}", "providerReset": "Penyedia Semble direset. Cache di disk tetap ada hingga pembangunan ulang berikutnya." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Kuota atau batas penagihan penyedia embeddings telah tercapai. Periksa paket atau penagihanmu, lalu mulai pengindeksan lagi." } diff --git a/src/i18n/locales/it/embeddings.json b/src/i18n/locales/it/embeddings.json index 578edc93bc..6c8516aebe 100644 --- a/src/i18n/locales/it/embeddings.json +++ b/src/i18n/locales/it/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "L'indicizzazione richiede una cartella di workspace aperta", "indexingStopped": "Indicizzazione interrotta dall'utente.", "indexingStoppedPartial": "Indicizzazione interrotta. Dati di indice parziali conservati.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Impossibile leggere la cache dell'indice del codice. I dati dell'indice esistente sono stati conservati. Riavvia l'indicizzazione per riprovare." }, "semble": { "downloadingBinary": "Download del binario semble in corso...", @@ -80,5 +80,5 @@ "checkFailed": "Verifica di semble fallita: {{errorMessage}}", "providerReset": "Provider Semble ripristinato. La cache su disco rimane fino alla prossima ricostruzione." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Quota o limite di fatturazione del provider di embedding raggiunto. Controlla il tuo piano o la fatturazione, poi riavvia l'indicizzazione." } diff --git a/src/i18n/locales/ja/embeddings.json b/src/i18n/locales/ja/embeddings.json index 3742e0e04b..935ca6cb09 100644 --- a/src/i18n/locales/ja/embeddings.json +++ b/src/i18n/locales/ja/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "インデックス作成には、開かれたワークスペースフォルダーが必要です", "indexingStopped": "ユーザーによりインデックス作成が停止されました。", "indexingStoppedPartial": "インデックス作成が停止されました。部分的なインデックスデータは保持されています。", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "コードインデックスのキャッシュを読み取れませんでした。既存のインデックスデータは保持されています。再試行するには、インデックス作成をもう一度開始してください。" }, "semble": { "downloadingBinary": "sembleバイナリをダウンロード中...", @@ -80,5 +80,5 @@ "checkFailed": "Sembleの確認に失敗しました: {{errorMessage}}", "providerReset": "Sembleプロバイダーをリセットしました。次回の再構築までディスクキャッシュは残ります。" }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "埋め込みプロバイダーのクォータまたは請求上限に達しました。プランまたは請求情報を確認してから、インデックス作成をもう一度開始してください。" } diff --git a/src/i18n/locales/ko/embeddings.json b/src/i18n/locales/ko/embeddings.json index ef18cda970..9c3e75169d 100644 --- a/src/i18n/locales/ko/embeddings.json +++ b/src/i18n/locales/ko/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "인덱싱에는 열린 워크스페이스 폴더가 필요합니다", "indexingStopped": "사용자에 의해 인덱싱이 중지되었습니다.", "indexingStoppedPartial": "인덱싱이 중지되었습니다. 부분 인덱스 데이터가 보존되었습니다.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "코드 인덱스 캐시를 읽지 못했습니다. 기존 인덱스 데이터는 보존되었습니다. 다시 시도하려면 인덱싱을 다시 시작하세요." }, "semble": { "downloadingBinary": "semble 바이너리 다운로드 중...", @@ -80,5 +80,5 @@ "checkFailed": "Semble 확인 실패: {{errorMessage}}", "providerReset": "Semble 공급자가 재설정되었습니다. 디스크 캐시는 다음 재구축까지 유지됩니다." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "임베딩 공급자의 할당량 또는 결제 한도에 도달했습니다. 요금제나 결제 정보를 확인한 다음 인덱싱을 다시 시작하세요." } diff --git a/src/i18n/locales/nl/embeddings.json b/src/i18n/locales/nl/embeddings.json index b98b5149ac..a3e9459da6 100644 --- a/src/i18n/locales/nl/embeddings.json +++ b/src/i18n/locales/nl/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Indexering vereist een geopende workspace map", "indexingStopped": "Indexering gestopt door gebruiker.", "indexingStoppedPartial": "Indexering gestopt. Gedeeltelijke indexgegevens bewaard.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "De cache van de code-index kon niet worden gelezen. Bestaande indexgegevens zijn bewaard. Start de indexering opnieuw om het nog eens te proberen." }, "semble": { "downloadingBinary": "Semble-binary downloaden...", @@ -80,5 +80,5 @@ "checkFailed": "Semble-controle mislukt: {{errorMessage}}", "providerReset": "Semble-provider gereset. Schijfcache blijft behouden tot de volgende heropbouw." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Het quotum of de factureringslimiet van de insluitingsprovider is bereikt. Controleer je abonnement of facturering en start de indexering opnieuw." } diff --git a/src/i18n/locales/pl/embeddings.json b/src/i18n/locales/pl/embeddings.json index d7ebfa3576..4517cd7a90 100644 --- a/src/i18n/locales/pl/embeddings.json +++ b/src/i18n/locales/pl/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Indeksowanie wymaga otwartego folderu workspace", "indexingStopped": "Indeksowanie zatrzymane przez użytkownika.", "indexingStoppedPartial": "Indeksowanie zatrzymane. Częściowe dane indeksu zachowane.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Nie udało się odczytać pamięci podręcznej indeksu kodu. Istniejące dane indeksu zostały zachowane. Uruchom indeksowanie ponownie, aby spróbować jeszcze raz." }, "semble": { "downloadingBinary": "Pobieranie binariów semble...", @@ -80,5 +80,5 @@ "checkFailed": "Sprawdzenie semble nie powiodło się: {{errorMessage}}", "providerReset": "Dostawca Semble zresetowany. Pamięć podręczna na dysku pozostaje do kolejnej przebudowy." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Osiągnięto limit wykorzystania lub rozliczeń u dostawcy osadzeń. Sprawdź swój plan lub rozliczenia, a następnie uruchom indeksowanie ponownie." } diff --git a/src/i18n/locales/pt-BR/embeddings.json b/src/i18n/locales/pt-BR/embeddings.json index e9f4561c9f..e43d21bcc6 100644 --- a/src/i18n/locales/pt-BR/embeddings.json +++ b/src/i18n/locales/pt-BR/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "A indexação requer uma pasta de workspace aberta", "indexingStopped": "Indexação interrompida pelo usuário.", "indexingStoppedPartial": "Indexação interrompida. Dados de índice parciais preservados.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Não foi possível ler o cache do índice de código. Os dados do índice existente foram preservados. Inicie a indexação novamente para tentar de novo." }, "semble": { "downloadingBinary": "Baixando o binário do semble...", @@ -80,5 +80,5 @@ "checkFailed": "Verificação do semble falhou: {{errorMessage}}", "providerReset": "Provedor Semble redefinido. O cache em disco permanece até a próxima reconstrução." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "A cota ou o limite de cobrança do provedor de embeddings foi atingido. Verifique seu plano ou sua cobrança e inicie a indexação novamente." } diff --git a/src/i18n/locales/ru/embeddings.json b/src/i18n/locales/ru/embeddings.json index bb667b9976..1185f49c54 100644 --- a/src/i18n/locales/ru/embeddings.json +++ b/src/i18n/locales/ru/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Для индексации требуется открытая папка рабочего пространства", "indexingStopped": "Индексация остановлена пользователем.", "indexingStoppedPartial": "Индексация остановлена. Частичные данные индекса сохранены.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Не удалось прочитать кэш индекса кода. Существующие данные индекса сохранены. Запусти индексацию снова, чтобы повторить попытку." }, "semble": { "downloadingBinary": "Загрузка бинарного файла semble...", @@ -80,5 +80,5 @@ "checkFailed": "Проверка semble не удалась: {{errorMessage}}", "providerReset": "Провайдер Semble сброшен. Дисковый кеш сохраняется до следующей пересборки." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Достигнута квота или лимит расходов у провайдера вложений. Проверь свой тариф или платёжные данные, затем запусти индексацию снова." } diff --git a/src/i18n/locales/tr/embeddings.json b/src/i18n/locales/tr/embeddings.json index 3afc96150f..cec2a5703e 100644 --- a/src/i18n/locales/tr/embeddings.json +++ b/src/i18n/locales/tr/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "İndeksleme açık bir workspace klasörü gerektirir", "indexingStopped": "İndeksleme kullanıcı tarafından durduruldu.", "indexingStoppedPartial": "İndeksleme durduruldu. Kısmi indeks verileri korundu.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Kod indeksi önbelleği okunamadı. Mevcut indeks verileri korundu. Yeniden denemek için indekslemeyi tekrar başlat." }, "semble": { "downloadingBinary": "semble ikili dosyası indiriliyor...", @@ -80,5 +80,5 @@ "checkFailed": "Semble kontrolü başarısız: {{errorMessage}}", "providerReset": "Semble sağlayıcısı sıfırlandı. Diskteki önbellek bir sonraki yeniden yapılandırmaya kadar korunur." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Gömülme sağlayıcısının kotasına veya faturalandırma sınırına ulaşıldı. Planını veya faturalandırma bilgilerini kontrol et, ardından indekslemeyi tekrar başlat." } diff --git a/src/i18n/locales/vi/embeddings.json b/src/i18n/locales/vi/embeddings.json index 5fe21bd5b9..db875d4a21 100644 --- a/src/i18n/locales/vi/embeddings.json +++ b/src/i18n/locales/vi/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "Lập chỉ mục yêu cầu một thư mục workspace đang mở", "indexingStopped": "Lập chỉ mục đã bị dừng bởi người dùng.", "indexingStoppedPartial": "Lập chỉ mục đã dừng. Dữ liệu chỉ mục một phần được bảo toàn.", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "Không thể đọc bộ nhớ đệm của chỉ mục mã. Dữ liệu chỉ mục hiện có đã được giữ lại. Bắt đầu lập chỉ mục lại để thử lại." }, "semble": { "downloadingBinary": "Đang tải xuống tệp nhị phân semble...", @@ -80,5 +80,5 @@ "checkFailed": "Kiểm tra semble thất bại: {{errorMessage}}", "providerReset": "Nhà cung cấp Semble đã đặt lại. Bộ nhớ đệm trên đĩa vẫn còn cho đến lần xây dựng lại tiếp theo." }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "Đã đạt hạn ngạch hoặc giới hạn thanh toán của nhà cung cấp nhúng. Kiểm tra gói dịch vụ hoặc thông tin thanh toán của bạn, rồi bắt đầu lập chỉ mục lại." } diff --git a/src/i18n/locales/zh-CN/embeddings.json b/src/i18n/locales/zh-CN/embeddings.json index 912fcfc300..f89bd2fb25 100644 --- a/src/i18n/locales/zh-CN/embeddings.json +++ b/src/i18n/locales/zh-CN/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "索引需要打开的工作区文件夹", "indexingStopped": "用户已停止索引。", "indexingStoppedPartial": "索引已停止。部分索引数据已保留。", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "无法读取代码索引缓存。现有索引数据已保留。重新启动索引以重试。" }, "semble": { "downloadingBinary": "正在下载 semble 二进制文件...", @@ -80,5 +80,5 @@ "checkFailed": "Semble 检查失败：{{errorMessage}}", "providerReset": "Semble 提供程序已重置。磁盘缓存将保留至下次重建。" }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "已达到嵌入提供商的配额或计费上限。检查套餐或计费信息，然后重新启动索引。" } diff --git a/src/i18n/locales/zh-TW/embeddings.json b/src/i18n/locales/zh-TW/embeddings.json index 2e4f45ebb3..756367e912 100644 --- a/src/i18n/locales/zh-TW/embeddings.json +++ b/src/i18n/locales/zh-TW/embeddings.json @@ -70,7 +70,7 @@ "indexingRequiresWorkspace": "索引需要開啟的工作區資料夾", "indexingStopped": "使用者已停止索引。", "indexingStoppedPartial": "索引已停止。部分索引資料已保留。", - "cacheReadFailed": "Could not read the code index cache. Existing index data was kept. Start indexing again to retry." + "cacheReadFailed": "無法讀取程式碼索引快取。現有索引資料已保留。重新啟動索引以重試。" }, "semble": { "downloadingBinary": "正在下載 semble 二進位檔...", @@ -80,5 +80,5 @@ "checkFailed": "Semble 檢查失敗：{{errorMessage}}", "providerReset": "Semble 提供者已重設。磁碟快取將保留至下次重建。" }, - "quotaExceeded": "Embedding provider quota or billing limit reached. Check your plan or billing, then start indexing again." + "quotaExceeded": "已達內嵌供應商的配額或計費上限。檢查方案或帳務資訊，然後重新啟動索引。" } From c6ee22c31a36d67467787313ec30936d7e6e77c5 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 22:51:32 -0800 Subject: [PATCH 33/42] fix(code-index): isolate Qdrant pacer per test and fix CI test failures --- .../__tests__/file-index-writer.spec.ts | 8 +++++--- src/services/code-index/__tests__/manager.spec.ts | 3 +++ .../code-index/__tests__/orchestrator.spec.ts | 12 ++++++++---- src/services/code-index/orchestrator.ts | 4 ++-- .../processors/__tests__/parser.spec.ts | 15 +++++++++++++++ .../vector-store/__tests__/qdrant-client.spec.ts | 6 ++++-- 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/services/code-index/__tests__/file-index-writer.spec.ts b/src/services/code-index/__tests__/file-index-writer.spec.ts index 20ab49f337..84a643262b 100644 --- a/src/services/code-index/__tests__/file-index-writer.spec.ts +++ b/src/services/code-index/__tests__/file-index-writer.spec.ts @@ -115,9 +115,11 @@ describe("FileIndexWriter", () => { controller.abort() throw serverError() }) - await expect(writer.replace("/ws/a.ts", "h1", [block(1)], controller.signal)).rejects.toBe( - controller.signal.reason, - ) + // Read the reason after the call: it is undefined until the embed aborts the controller. + const outcome = await writer + .replace("/ws/a.ts", "h1", [block(1)], controller.signal) + .catch((error: unknown) => error) + expect(outcome).toBe(controller.signal.reason) expect(cache.updateHash).not.toHaveBeenCalled() expect(failures.list()).toEqual([]) }) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index eb30afce76..ac10d087ac 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -242,6 +242,9 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { value: { startIndexing, stopIndexing: vi.fn() }, configurable: true, }) + // The background start re-checks enablement, which a bare manager reports as off. + Object.defineProperty(manager, "isFeatureEnabled", { get: () => true, configurable: true }) + Object.defineProperty(manager, "isWorkspaceEnabled", { get: () => true, configurable: true }) manager["startIndexingInBackground"]() diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 8e47445b4a..f5bab1999a 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -3,7 +3,7 @@ import { FileFailureRegistry } from "../file-failure-registry" import { ProviderPacerRegistry } from "../provider-pacer" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" -import type { IDirectoryScanner } from "../interfaces" +import type { BatchProcessingSummary, IDirectoryScanner } from "../interfaces" import { clearAllMocks } from "../../../test-utils/reset" @@ -88,7 +88,11 @@ type Progress = { processedInBatch: number; totalInBatch: number; currentFile?: /** Wires the watcher double's three events to callable emitters. */ function emitters(fileWatcher: ReturnType["fileWatcher"]) { - const listeners: { start?: () => void; progress?: (event: Progress) => void; finish?: () => void } = {} + const listeners: { + start?: () => void + progress?: (event: Progress) => void + finish?: (summary: BatchProcessingSummary) => void + } = {} fileWatcher.onDidStartBatchProcessing.mockImplementation((listener: () => void) => { listeners.start = listener return { dispose: vi.fn() } @@ -97,7 +101,7 @@ function emitters(fileWatcher: ReturnType["fileWatcher"]) { listeners.progress = listener return { dispose: vi.fn() } }) - fileWatcher.onDidFinishBatchProcessing.mockImplementation((listener: () => void) => { + fileWatcher.onDidFinishBatchProcessing.mockImplementation((listener: (summary: BatchProcessingSummary) => void) => { listeners.finish = listener return { dispose: vi.fn() } }) @@ -107,7 +111,7 @@ function emitters(fileWatcher: ReturnType["fileWatcher"]) { listeners.progress?.({ processedInBatch: 0, totalInBatch: total }) duringBatch?.() listeners.progress?.({ processedInBatch: total, totalInBatch: total, currentFile: "/w/a.ts" }) - listeners.finish?.() + listeners.finish?.({ processedFiles: [] }) listeners.progress?.({ processedInBatch: total, totalInBatch: total }) listeners.progress?.({ processedInBatch: 0, totalInBatch: 0 }) } diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index e2700b2396..5cb8c6b9bf 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -98,8 +98,8 @@ export class CodeIndexOrchestrator { this.updateRecoveryStatus(signal) } }), - this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { - if (summary.batchError) { + this.fileWatcher.onDidFinishBatchProcessing((summary?: BatchProcessingSummary) => { + if (summary?.batchError) { console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) } this.watcherBatchPending = false diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 39acdeb5a4..c5dbb908ec 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -322,6 +322,21 @@ describe("CodeParser", () => { describe("singleton instance", () => { it("should maintain parser state across calls", async () => { + // Capture each parsed root, so the result reflects the input rather than an earlier fixed capture. + const echoRoot = (node: { text: string }) => [ + { + node: { + text: node.text, + startPosition: { row: 0 }, + endPosition: { row: 0 }, + type: "function_declaration", + childForFieldName: vi.fn().mockReturnValue(null), + children: [], + }, + name: "definition.function", + }, + ] + mockLanguageParser.js.query.captures.mockImplementationOnce(echoRoot).mockImplementationOnce(echoRoot) const first = "const firstValue = 'singleton parser first input with enough text'" const second = "const secondValue = 'singleton parser second input with enough text'" const result1 = await codeParser.parseFile("test.js", { content: first }) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index e2f23815a7..eb3b58aaf5 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -43,7 +43,8 @@ const mockQdrantClientInstance = { delete: vitest.fn(), } -const pacers = new ProviderPacerRegistry() +// Recreated per test: a 429 pause left in a shared registry would delay every later request. +let pacers: ProviderPacerRegistry const mockCreateHashInstance = { update: vitest.fn().mockReturnThis(), @@ -61,6 +62,7 @@ describe("QdrantVectorStore", () => { beforeEach(() => { clearAllMocks() + pacers = new ProviderPacerRegistry() // Mock QdrantClient constructor ;(QdrantClient as any).mockImplementation(function () { @@ -844,7 +846,7 @@ describe("QdrantVectorStore", () => { ["without Retry-After", undefined, 1000], ])("waits and recovers when collection creation gets a 429 %s", async (_label, retryAfter, waitMs) => { // The module is automocked, so set the fields the patched client would assign. - const limited = Object.assign(new QdrantClientResourceExhaustedError("Too many requests", 0), { + const limited = Object.assign(new QdrantClientResourceExhaustedError("Too many requests", "0"), { message: "Too many requests", retry_after: retryAfter, }) From e8a075f4cee185c11e9c1d279df4c3655b23e8d8 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 13 Nov 2026 23:57:46 -0800 Subject: [PATCH 34/42] fix(code-index): restore detached start in webview startIndexing handler --- .../__tests__/webviewMessageHandler.spec.ts | 15 ++++----------- src/core/webview/webviewMessageHandler.ts | 5 +++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index 9007488c08..18836fa440 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -807,18 +807,10 @@ describe("CI-WV-01: code index start and status forwarding", () => { beforeEach(() => vi.clearAllMocks()) - it.each(["Standby", "Error"])("starts indexing once from %s and awaits the start", async (state) => { + it.each(["Standby", "Error"])("starts indexing once from %s without awaiting the start", async (state) => { const { manager, provider } = withManager(state) - let finishStart!: () => void - manager.startIndexing.mockReturnValue(new Promise((resolve) => (finishStart = resolve))) - let handled = false - const handling = webviewMessageHandler(provider, { type: "startIndexing" }).then(() => (handled = true)) - await new Promise((resolve) => setTimeout(resolve, 0)) - expect(manager.startIndexing).toHaveBeenCalledTimes(1) - expect(handled).toBe(false) - finishStart() - await handling - expect(handled).toBe(true) + manager.startIndexing.mockReturnValue(new Promise(() => {})) + await expect(webviewMessageHandler(provider, { type: "startIndexing" })).resolves.toBeUndefined() expect(manager.startIndexing).toHaveBeenCalledTimes(1) }) @@ -826,6 +818,7 @@ describe("CI-WV-01: code index start and status forwarding", () => { const { manager, provider } = withManager("Standby") manager.startIndexing.mockRejectedValue(new Error("boom")) await expect(webviewMessageHandler(provider, { type: "startIndexing" })).resolves.toBeUndefined() + await new Promise((resolve) => setTimeout(resolve, 0)) expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: boom") }) diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index d292c7c24f..e1716aea70 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3530,12 +3530,13 @@ export const webviewMessageHandler = async ( const currentState = manager.state if (currentState === "Standby" || currentState === "Error") { - await manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + // Detached: a full scan must not block the message handler; the manager joins repeat starts. + void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) if (!manager.isInitialized) { await manager.initialize(provider.contextProxy) if (manager.state === "Standby" || manager.state === "Error") { - await manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) + void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) } } } From 5061b15651332672a8c343cf509636f4a6e82404 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 01:34:35 -0800 Subject: [PATCH 35/42] fix(code-index): start from Error in one step, count pending watcher batches, remove uncached failed files - startIndexing() from Error now recovers, re-initializes and starts; the webview handler no longer checks isInitialized in the same tick --- .../webview/__tests__/ClineProvider.spec.ts | 14 ++- src/core/webview/webviewMessageHandler.ts | 10 +-- .../__tests__/file-index-writer.spec.ts | 13 +++ .../__tests__/manager.reliability.spec.ts | 51 +++++++++++ .../code-index/__tests__/orchestrator.spec.ts | 89 +++++++++++++------ .../__tests__/provider-pacer.spec.ts | 8 +- src/services/code-index/file-index-writer.ts | 5 +- src/services/code-index/manager.ts | 14 +-- src/services/code-index/orchestrator.ts | 13 +-- .../processors/__tests__/file-watcher.spec.ts | 53 +++++++---- .../processors/__tests__/scanner.spec.ts | 27 ++++++ .../code-index/processors/file-watcher.ts | 9 +- src/services/code-index/processors/scanner.ts | 26 ++++-- 13 files changed, 244 insertions(+), 88 deletions(-) diff --git a/src/core/webview/__tests__/ClineProvider.spec.ts b/src/core/webview/__tests__/ClineProvider.spec.ts index 464889418e..45fd7b11d9 100644 --- a/src/core/webview/__tests__/ClineProvider.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.spec.ts @@ -6022,13 +6022,11 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { expect(provider.log).toHaveBeenCalledWith("Cannot start indexing: No workspace folder open") }) - it("catches both start-indexing calls during error recovery", async () => { + it("starts once from Error and leaves recovery to the manager", async () => { const manager = createIndexManager({ + state: "Error", isInitialized: false, - startIndexing: vi - .fn() - .mockRejectedValueOnce(new Error("first failure")) - .mockRejectedValueOnce(new Error("second failure")), + startIndexing: vi.fn().mockRejectedValueOnce(new Error("recovery failure")), }) const provider = createProvider({ getCurrentWorkspaceCodeIndexScope: vi.fn().mockReturnValue({ codeIndexManager: manager }), @@ -6037,9 +6035,9 @@ describe("webviewMessageHandler no-floating-promises coverage", () => { await webviewMessageHandler(provider, { type: "startIndexing" }) await Promise.resolve() - expect(manager.startIndexing).toHaveBeenCalledTimes(2) - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: first failure") - expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: second failure") + expect(manager.initialize).toHaveBeenCalledOnce() + expect(manager.startIndexing).toHaveBeenCalledOnce() + expect(provider.log).toHaveBeenCalledWith("Indexing error: Error: recovery failure") }) it("covers changed stop, toggle, and detached toggle rejection paths", async () => { diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index e1716aea70..65916383be 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -3530,15 +3530,9 @@ export const webviewMessageHandler = async ( const currentState = manager.state if (currentState === "Standby" || currentState === "Error") { - // Detached: a full scan must not block the message handler; the manager joins repeat starts. + // Detached: a full scan must not block the message handler. From Error, the manager + // itself recovers, re-initializes and starts. void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) - - if (!manager.isInitialized) { - await manager.initialize(provider.contextProxy) - if (manager.state === "Standby" || manager.state === "Error") { - void manager.startIndexing().catch((err) => provider.log(`Indexing error: ${err}`)) - } - } } } } catch (error) { diff --git a/src/services/code-index/__tests__/file-index-writer.spec.ts b/src/services/code-index/__tests__/file-index-writer.spec.ts index 84a643262b..45780ed34a 100644 --- a/src/services/code-index/__tests__/file-index-writer.spec.ts +++ b/src/services/code-index/__tests__/file-index-writer.spec.ts @@ -124,6 +124,19 @@ describe("FileIndexWriter", () => { expect(failures.list()).toEqual([]) }) + it("drops the old hash when a stop lands while the delete is in flight", async () => { + const { writer, store, cache, failures } = setup() + const controller = new AbortController() + store.deletePointsByFilePath.mockImplementation(async () => controller.abort()) + const outcome = await writer + .replace("/ws/a.ts", "h1", [block(1)], controller.signal) + .catch((error: unknown) => error) + expect(outcome).toBe(controller.signal.reason) + expect(cache.deleteHash).toHaveBeenCalledExactlyOnceWith("/ws/a.ts") + expect(cache.updateHash).not.toHaveBeenCalled() + expect(failures.list()).toEqual([]) + }) + it("drops the old hash once its points are deleted, so a failed replacement is re-indexed", async () => { const { writer, embedder, store, cache } = setup() embedder.createEmbeddings.mockRejectedValue(Object.assign(new Error("bad"), { status: 400 })) diff --git a/src/services/code-index/__tests__/manager.reliability.spec.ts b/src/services/code-index/__tests__/manager.reliability.spec.ts index a8cc11a338..146871f14c 100644 --- a/src/services/code-index/__tests__/manager.reliability.spec.ts +++ b/src/services/code-index/__tests__/manager.reliability.spec.ts @@ -32,6 +32,17 @@ vi.mock("../semble", () => ({ }), })) vi.mock("../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../config-manager", () => ({ + CodeIndexConfigManager: vi.fn().mockImplementation(function (contextProxy: unknown) { + return { + isFeatureEnabled: true, + isFeatureConfigured: true, + currentEmbedderProvider: "openai", + loadConfiguration: async () => ({ requiresRestart: false }), + getContextProxy: () => contextProxy, + } + }), +})) const workspacePath = path.join(path.sep, "test", "workspace") @@ -150,6 +161,46 @@ describe("CodeIndexManager reliability", () => { }, ) + it("CI-MG-02: one start from Error recovers, re-initializes and starts indexing", async () => { + const { manager, stateManager } = makeManager() + const contextProxy = {} + manager["_configManager"] = { + isFeatureEnabled: true, + isFeatureConfigured: true, + getContextProxy: () => contextProxy, + } as unknown as (typeof manager)["_configManager"] + const stale: OrchestratorDouble = { + startIndexing: vi.fn(async () => {}), + // Settles a tick later, as the real stop does, so a same-tick caller check would miss recovery. + stopIndexing: vi.fn(() => new Promise((resolve) => setTimeout(resolve, 0))), + isScanning: false, + state: "Error", + } + attach(manager, stale) + stateManager.setSystemState("Error", "scan failed") + const fresh: OrchestratorDouble = { + startIndexing: vi.fn(async () => void stateManager.setSystemState("Indexing", "scanning")), + stopIndexing: vi.fn(async () => {}), + isScanning: false, + state: "Standby", + } + vi.spyOn(CacheManager.prototype, "initialize").mockResolvedValue(undefined) + vi.spyOn(manager as unknown as { _recreateServices(): Promise }, "_recreateServices").mockImplementation( + async () => { + attach(manager, fresh) + manager["_searchService"] = {} as unknown as (typeof manager)["_searchService"] + }, + ) + + await manager.startIndexing() + + expect(stale.stopIndexing).toHaveBeenCalledOnce() + expect(stale.startIndexing).not.toHaveBeenCalled() + expect(fresh.startIndexing).toHaveBeenCalledOnce() + expect(manager.isInitialized).toBe(true) + expect(stateManager.state).toBe("Indexing") + }) + it("CI-MG-02: a replacement failure ends in Error and stays restartable", async () => { const { manager, stateManager } = makeManager() manager["_configManager"] = { diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index f5bab1999a..cfadaef0f8 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -86,35 +86,43 @@ const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) type Progress = { processedInBatch: number; totalInBatch: number; currentFile?: string } -/** Wires the watcher double's three events to callable emitters. */ +function liveListeners() { + const listeners = new Set<(value: T) => void>() + return { + listeners, + subscribe: (listener: (value: T) => void) => { + listeners.add(listener) + return { dispose: () => void listeners.delete(listener) } + }, + fire: (value: T) => listeners.forEach((listener) => listener(value)), + } +} + +/** Wires the watcher double's three events to emitters whose subscriptions really dispose. */ function emitters(fileWatcher: ReturnType["fileWatcher"]) { - const listeners: { - start?: () => void - progress?: (event: Progress) => void - finish?: (summary: BatchProcessingSummary) => void - } = {} - fileWatcher.onDidStartBatchProcessing.mockImplementation((listener: () => void) => { - listeners.start = listener - return { dispose: vi.fn() } - }) - fileWatcher.onBatchProgressUpdate.mockImplementation((listener: (event: Progress) => void) => { - listeners.progress = listener - return { dispose: vi.fn() } - }) - fileWatcher.onDidFinishBatchProcessing.mockImplementation((listener: (summary: BatchProcessingSummary) => void) => { - listeners.finish = listener - return { dispose: vi.fn() } - }) + const start = liveListeners() + const progress = liveListeners() + const finish = liveListeners() + fileWatcher.onDidStartBatchProcessing.mockImplementation(start.subscribe) + fileWatcher.onBatchProgressUpdate.mockImplementation(progress.subscribe) + fileWatcher.onDidFinishBatchProcessing.mockImplementation(finish.subscribe) + const events = { + start: () => start.fire(), + progress: progress.fire, + finish: () => finish.fire({ processedFiles: [] }), + listenerCounts: () => [start.listeners.size, progress.listeners.size, finish.listeners.size], + } /** Same order as `FileWatcher.processBatch()`: start, progress, finish, then two trailing progress events. */ - return (total: number, duringBatch?: () => void) => { - listeners.start?.() - listeners.progress?.({ processedInBatch: 0, totalInBatch: total }) + const runBatch = (total: number, duringBatch?: () => void) => { + events.start() + events.progress({ processedInBatch: 0, totalInBatch: total }) duringBatch?.() - listeners.progress?.({ processedInBatch: total, totalInBatch: total, currentFile: "/w/a.ts" }) - listeners.finish?.({ processedFiles: [] }) - listeners.progress?.({ processedInBatch: total, totalInBatch: total }) - listeners.progress?.({ processedInBatch: 0, totalInBatch: 0 }) + events.progress({ processedInBatch: total, totalInBatch: total, currentFile: "/w/a.ts" }) + events.finish() + events.progress({ processedInBatch: total, totalInBatch: total }) + events.progress({ processedInBatch: 0, totalInBatch: 0 }) } + return { runBatch, events } } describe("CodeIndexOrchestrator", () => { @@ -162,7 +170,7 @@ describe("CodeIndexOrchestrator", () => { context.stateManager.reportFileQueueProgress.mockImplementation(() => { context.stateManager.setSystemState("Indexing") }) - return { context, runBatch: emitters(context.fileWatcher) } + return { context, ...emitters(context.fileWatcher) } } it("ends in Indexed after batch-finished and the two trailing progress events", async () => { @@ -187,15 +195,38 @@ describe("CodeIndexOrchestrator", () => { ) }) - it("reuses the watcher events after stop and restart", async () => { - const { context, runBatch } = progressSetup() + it("subscribes new watcher listeners after stop and restart", async () => { + const { context, runBatch, events } = progressSetup() await context.orchestrator.startIndexing() await context.orchestrator.stopIndexing() + expect(events.listenerCounts()).toEqual([0, 0, 0]) + await context.orchestrator.startIndexing() - runBatch(1) + expect(events.listenerCounts()).toEqual([1, 1, 1]) + expect(context.fileWatcher.onDidStartBatchProcessing).toHaveBeenCalledTimes(2) + runBatch(1, () => expect(context.orchestrator.state).toBe("Indexing")) expect(context.orchestrator.state).toBe("Indexed") expect(context.fileWatcher.initialize).toHaveBeenCalledTimes(2) }) + + it("stays Indexing while a second queued batch is pending after the first finishes", async () => { + const { context, events } = progressSetup() + await context.orchestrator.startIndexing() + events.start() + events.start() + events.progress({ processedInBatch: 0, totalInBatch: 1 }) + events.finish() + events.progress({ processedInBatch: 1, totalInBatch: 1 }) + + context.failures.renew("a.ts", "h") + context.failures.record("a.ts", "h", "embed", "down", 1, "retrying") + context.failures.clear("a.ts", "h") + expect(context.orchestrator.state).toBe("Indexing") + + events.progress({ processedInBatch: 0, totalInBatch: 1 }) + events.finish() + expect(context.orchestrator.state).toBe("Indexed") + }) }) it("does not clean up when initialize() fails and ends in Error", async () => { diff --git a/src/services/code-index/__tests__/provider-pacer.spec.ts b/src/services/code-index/__tests__/provider-pacer.spec.ts index 4f359c85d5..5f455f94b0 100644 --- a/src/services/code-index/__tests__/provider-pacer.spec.ts +++ b/src/services/code-index/__tests__/provider-pacer.spec.ts @@ -99,7 +99,7 @@ describe("ProviderPacer (CI-PP-02)", () => { await vi.advanceTimersByTimeAsync(59_999) expect(shortRequest).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(1) - await Promise.all([slow, short]) + await expect(Promise.all([slow, short])).resolves.toEqual([{ value: undefined }, { value: "ok" }]) expect(shortRequest).toHaveBeenCalledTimes(2) }) @@ -119,7 +119,7 @@ describe("ProviderPacer (CI-PP-02)", () => { await vi.advanceTimersByTimeAsync(0) expect(pacer.resumeAt! - Date.now()).toBe(1000) await vi.advanceTimersByTimeAsync(1000) - await Promise.all(runs) + await expect(Promise.all(runs)).resolves.toEqual([1, 2, 3, 4].map(() => ({ value: "ok" }))) }) it("rejects a waiter at once with the abort reason and drops its path", async () => { @@ -136,7 +136,7 @@ describe("ProviderPacer (CI-PP-02)", () => { expect(pacer.waitingPaths).not.toContain("b.ts") expect(request).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(60_000) - await first + await expect(first).resolves.toEqual({ value: undefined }) }) it("rethrows non-rate-limit errors unchanged", async () => { @@ -180,7 +180,7 @@ describe("ProviderPacerRegistry", () => { expect(changed).toHaveBeenCalled() await vi.advanceTimersByTimeAsync(5000) - await Promise.all([paused, same]) + await expect(Promise.all([paused, same])).resolves.toEqual([{ value: undefined }, { value: "ok" }]) expect(registry.providerWaits()).toEqual([]) expect(registry.hasWaitingRequests()).toBe(false) }) diff --git a/src/services/code-index/file-index-writer.ts b/src/services/code-index/file-index-writer.ts index ac82580290..77775a67cc 100644 --- a/src/services/code-index/file-index-writer.ts +++ b/src/services/code-index/file-index-writer.ts @@ -33,8 +33,9 @@ export class FileIndexWriter { await this.attempt(relativePath, hash, "delete", this.store.rateLimitScope, signal, () => this.store.deletePointsByFilePath(filePath), ) - // The old points are gone, so the old hash must not let an unchanged file skip re-indexing. - if (!signal?.aborted) this.cache.deleteHash(filePath) + // The old points are gone, so the old hash must not let an unchanged file skip re-indexing, + // even when a stop landed while the delete was in flight. + this.cache.deleteHash(filePath) } for (let offset = 0; offset < selected.length; offset += this.batchSize) { const batch = selected.slice(offset, offset + this.batchSize) diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index dc668ee357..5ec3ee501c 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -169,6 +169,8 @@ export class CodeIndexManager { // 5. CacheManager Initialization; an unreadable cache ends here so existing index data is kept. if (!(await this.ensureCacheManager())) return { requiresRestart } + // The cache read yields; indexing may have been disabled meanwhile. + if (!this.isFeatureEnabled || !this.isWorkspaceEnabled) return { requiresRestart } // 6. Determine if Core Services Need Recreation const needsServiceRecreation = (!this._serviceFactory && !this._sembleProvider) || requiresRestart @@ -252,13 +254,13 @@ export class CodeIndexManager { if (this.startInFlight) return this.startInFlight this.failureRegistry.resetAllBudgets() - // Check if we're in error state and recover if needed - const currentStatus = this.getCurrentStatus() - if (currentStatus.systemStatus === "Error") { + if (this.getCurrentStatus().systemStatus === "Error") { + // Recovery drops the config manager, so keep its proxy to re-initialize; initialize() starts indexing. + const contextProxy = this._configManager?.getContextProxy() await this.recoverFromError() - - // After recovery, we need to reinitialize since recoverFromError clears all services - // This will be handled by the caller (webviewMessageHandler) checking isInitialized + if (!contextProxy) return + await this.initialize(contextProxy) + await this.startInFlight return } diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 5cb8c6b9bf..8c192c88d1 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -25,7 +25,8 @@ export class CodeIndexOrchestrator { private scanPromise?: Promise private stopPromise?: Promise private scanDone = false - private watcherBatchPending = false + // A count, not a flag: the watcher queues batches, so one batch can finish while another is pending. + private pendingWatcherBatches = 0 private recoverySubscriptions: vscode.Disposable[] = [] private readonly scanExecutor: CodeIndexScanExecutor @@ -59,7 +60,7 @@ export class CodeIndexOrchestrator { this.stateManager.setSystemState("Indexing", formatProviderWaitMessage(this.pacers.providerWaits())) return } - if (!this.scanDone || this.watcherBatchPending) return + if (!this.scanDone || this.pendingWatcherBatches > 0) return if (this.failures.list().length > 0) { this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingFilesFailed")) return @@ -81,7 +82,7 @@ export class CodeIndexOrchestrator { this._fileWatcherSubscriptions = [ this.fileWatcher.onDidStartBatchProcessing(() => { - this.watcherBatchPending = true + this.pendingWatcherBatches++ }), this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { if (signal.aborted || this.stateManager.state === "Stopping") return @@ -94,7 +95,7 @@ export class CodeIndexOrchestrator { currentFile ? path.basename(currentFile) : undefined, ) // The watcher reports progress after the batch-finished event; settle the status again. - if (processedInBatch === totalInBatch && !this.watcherBatchPending) { + if (processedInBatch === totalInBatch && this.pendingWatcherBatches === 0) { this.updateRecoveryStatus(signal) } }), @@ -102,7 +103,7 @@ export class CodeIndexOrchestrator { if (summary?.batchError) { console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) } - this.watcherBatchPending = false + this.pendingWatcherBatches = Math.max(0, this.pendingWatcherBatches - 1) this.updateRecoveryStatus(signal) }), ] @@ -271,7 +272,7 @@ export class CodeIndexOrchestrator { this._fileWatcherSubscriptions = [] this.recoverySubscriptions.forEach((sub) => sub.dispose()) this.recoverySubscriptions = [] - this.watcherBatchPending = false + this.pendingWatcherBatches = 0 } /** diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index f3824ed678..c04be5c935 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -469,6 +469,8 @@ describe("FileWatcher", () => { ) mockVectorStore.deletePointsByFilePath.mockImplementation(async () => void order.push("delete")) mockCacheManager.getHash.mockReturnValue(undefined) + mockCacheManager.updateHash.mockImplementation(() => void order.push("hash")) + mockCacheManager.deleteHash.mockImplementation(() => void order.push("unhash")) await mockOnDidChange({ fsPath: path }) await flushBatch() await mockOnDidDelete({ fsPath: path }) @@ -476,7 +478,8 @@ describe("FileWatcher", () => { expect(order).toEqual([]) finishUpsert() await vi.advanceTimersByTimeAsync(0) - expect(order).toEqual(["upsert", "delete"]) + // The delete wins: its points and hash removal come after the change's writes. + expect(order).toEqual(["upsert", "hash", "delete", "unhash"]) }) it("clears a stale read failure when the content is unchanged", async () => { @@ -495,21 +498,39 @@ describe("FileWatcher", () => { }) it.each([ - ["File is too large", "h", 1], - ["File is ignored by .rooignore or .gitignore", "h", 1], - ["File is too large", undefined, 0], - ])("a change skipped as %s with cached hash %s removes old points %i times", async (reason, hash, removals) => { - watcherWithRegistry() - mockCacheManager.getHash.mockReturnValue(hash) - vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ - path, - status: "skipped", - reason, - }) - await runChange() - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledTimes(removals) - expect(mockCacheManager.deleteHash).toHaveBeenCalledTimes(removals) - }) + { reason: "File is too large", hash: "h", failed: false, removed: true }, + { reason: "File is ignored by .rooignore or .gitignore", hash: "h", failed: false, removed: true }, + { reason: "File is too large", hash: undefined, failed: true, removed: true }, + { reason: "File is too large", hash: undefined, failed: false, removed: false }, + ])( + "a change skipped as $reason (hash $hash, failure entry $failed) removes points: $removed", + async ({ reason, hash, failed, removed }) => { + const failures = watcherWithRegistry() + const relativePath = nodePath.normalize("src/file.ts") + if (failed) { + failures.renew(relativePath, "h0") + failures.record(relativePath, "h0", "upsert", "down", 3, "exhausted") + } + mockCacheManager.getHash.mockReturnValue(hash) + vi.spyOn(FilePreparation.prototype, "prepareFile").mockResolvedValueOnce({ + path, + status: "skipped", + reason, + }) + const summary = await runChange() + if (removed) { + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledExactlyOnceWith(path) + expect(mockCacheManager.deleteHash).toHaveBeenCalledExactlyOnceWith(path) + expect(summary.processedFiles).toEqual([{ path, status: "success" }]) + } else { + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + expect(mockCacheManager.deleteHash).not.toHaveBeenCalled() + expect(summary.processedFiles).toEqual([{ path, status: "skipped", reason }]) + } + expect(failures.list()).toEqual([]) + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) it("writes nothing after the session signal aborts", async () => { watcherWithRegistry() diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index dc88949993..50799f9f97 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -522,6 +522,33 @@ describe("DirectoryScanner", () => { }, ) + it("removes points and the entry of a failed file that has no hash and is gone", async () => { + const failures = new FileFailureRegistry() + const removalScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([[], false]) + const relativePath = nodePath.normalize("src/failed.js") + failures.renew(relativePath, "h") + failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") + + await removalScanner.scanDirectory("/test") + + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() + const [[removedPath]] = vi.mocked(mockVectorStore.deletePointsByFilePath).mock.calls + expect(nodePath.isAbsolute(removedPath)).toBe(true) + expect(removedPath.endsWith(relativePath)).toBe(true) + expect(failures.list()).toEqual([]) + }) + it("records a non-missing read error as a read failure", async () => { const failures = new FileFailureRegistry() const readScanner = new DirectoryScanner( diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index ec21117512..04f388cc0e 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -230,8 +230,13 @@ export class FileWatcher implements IFileWatcher { if (!signal?.aborted) this.failures.clear(generateRelativeFilePath(filePath, this.workspacePath)) return result } - // A file that became ignored or too large must not keep its old points. - if (this.cacheManager.getHash(filePath) !== undefined) return this.removeFile(filePath, signal) + // A file that became ignored or too large must not keep its old points; a failure entry + // without a hash means a failed first write may have left partial points. + const relativePath = generateRelativeFilePath(filePath, this.workspacePath) + const mayHavePoints = + this.cacheManager.getHash(filePath) !== undefined || + this.failures.list().some((failure) => failure.path === relativePath) + if (mayHavePoints) return this.removeFile(filePath, signal) return result } if (result.status !== "processed_for_batching" || !result.newHash || !this.writer) return result diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index a5ccc948ae..79a4c9b369 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -3,7 +3,7 @@ import { Ignore } from "ignore" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { stat } from "fs/promises" import * as path from "path" -import { generateRelativeFilePath } from "../shared/get-relative-path" +import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" import { getWorkspacePathForContext } from "../../../utils/path" import { scannerExtensions } from "../shared/supported-extensions" import * as vscode from "vscode" @@ -197,16 +197,28 @@ export class DirectoryScanner implements IDirectoryScanner { return { stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount } } - const oldHashes = this.cacheManager.getAllHashes() - for (const cachedFilePath of Object.keys(oldHashes)) { - if (processedFiles.has(cachedFilePath)) continue + const removalCandidates = new Map() + for (const cachedFilePath of Object.keys(this.cacheManager.getAllHashes())) { + removalCandidates.set(generateRelativeFilePath(cachedFilePath, scanWorkspace), cachedFilePath) + } + // A file whose first write failed has no hash, but its failure entry means partial points may exist. + for (const failure of this.failures.list()) { + if (!removalCandidates.has(failure.path)) { + removalCandidates.set(failure.path, generateNormalizedAbsolutePath(failure.path, scanWorkspace)) + } + } + const processedRelative = new Set( + [...processedFiles].map((filePath) => generateRelativeFilePath(filePath, scanWorkspace)), + ) + for (const [relativePath, removedFilePath] of removalCandidates) { + if (processedRelative.has(relativePath)) continue if (signal?.aborted) break - const removed = await writer.remove(cachedFilePath, signal) + const removed = await writer.remove(removedFilePath, signal) if (!removed && !signal?.aborted) { this.reportError( - new Error(t("embeddings:scanner.unknownErrorDeletingPoints", { filePath: cachedFilePath })), + new Error(t("embeddings:scanner.unknownErrorDeletingPoints", { filePath: removedFilePath })), "scanDirectory:deleteRemovedFiles", - cachedFilePath, + removedFilePath, scanWorkspace, onError, ) From 2d2f878a02143263029e2fe4f0498d6821cd280c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 02:43:22 -0800 Subject: [PATCH 36/42] test(code-index): use providerIdentifiers.openai in manager reliability spec --- src/services/code-index/__tests__/manager.reliability.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/__tests__/manager.reliability.spec.ts b/src/services/code-index/__tests__/manager.reliability.spec.ts index 146871f14c..0b59827277 100644 --- a/src/services/code-index/__tests__/manager.reliability.spec.ts +++ b/src/services/code-index/__tests__/manager.reliability.spec.ts @@ -1,4 +1,5 @@ import * as path from "path" +import { providerIdentifiers } from "@roo-code/types/provider-identifiers" import type * as vscode from "vscode" import { CodeIndexManager } from "../manager" import { CodeIndexStateManager } from "../state-manager" @@ -37,7 +38,7 @@ vi.mock("../config-manager", () => ({ return { isFeatureEnabled: true, isFeatureConfigured: true, - currentEmbedderProvider: "openai", + currentEmbedderProvider: providerIdentifiers.openai, loadConfiguration: async () => ({ requiresRestart: false }), getContextProxy: () => contextProxy, } From df9f19b73be5c672d04427a3d6b98d896efd5ede Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 03:57:40 -0800 Subject: [PATCH 37/42] fix(code-index): keep failure-only entries when file listing is truncated --- .../processors/__tests__/scanner.spec.ts | 30 +++++++++++++++++++ src/services/code-index/processors/scanner.ts | 5 ++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 50799f9f97..abd9afc9cb 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -549,6 +549,36 @@ describe("DirectoryScanner", () => { expect(failures.list()).toEqual([]) }) + it.each([ + { truncated: true, removed: false }, + { truncated: false, removed: true }, + ])( + "failure-only file with truncated listing=$truncated is removed=$removed", + async ({ truncated, removed }) => { + const failures = new FileFailureRegistry() + const truncationScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([[], truncated]) + const relativePath = nodePath.normalize("src/unlisted.js") + failures.renew(relativePath, "h") + failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") + + await truncationScanner.scanDirectory("/test") + + expect(vi.mocked(mockVectorStore.deletePointsByFilePath).mock.calls.length).toBe(removed ? 1 : 0) + expect(failures.list()).toHaveLength(removed ? 0 : 1) + }, + ) + it("records a non-missing read error as a read failure", async () => { const failures = new FileFailureRegistry() const readScanner = new DirectoryScanner( diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 79a4c9b369..13c198e959 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -83,7 +83,7 @@ export class DirectoryScanner implements IDirectoryScanner { ) // Get all files recursively (handles .gitignore automatically) - const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) + const [allPaths, listingTruncated] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) // Filter out directories (marked with trailing '/') const filePaths = allPaths.filter((p) => !p.endsWith("/")) @@ -202,7 +202,8 @@ export class DirectoryScanner implements IDirectoryScanner { removalCandidates.set(generateRelativeFilePath(cachedFilePath, scanWorkspace), cachedFilePath) } // A file whose first write failed has no hash, but its failure entry means partial points may exist. - for (const failure of this.failures.list()) { + // A truncated listing cannot prove a failure-only file is gone, so those are kept. + for (const failure of listingTruncated ? [] : this.failures.list()) { if (!removalCandidates.has(failure.path)) { removalCandidates.set(failure.path, generateNormalizedAbsolutePath(failure.path, scanWorkspace)) } From 851b8440ed91bd902fadd690c12e14c4cc10d495 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 04:21:01 -0800 Subject: [PATCH 38/42] fix(code-index): remove confirmed-missing failure files on truncated scans and handle write rejections at queue time --- .../processors/__tests__/scanner.spec.ts | 61 +++++++++++++++++++ src/services/code-index/processors/scanner.ts | 21 +++++-- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index abd9afc9cb..8acfcc9ee0 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -579,6 +579,67 @@ describe("DirectoryScanner", () => { }, ) + it("removes a listed failure-only file confirmed missing even when the listing is truncated", async () => { + const failures = new FileFailureRegistry() + const truncationScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/vanished.js"], true]) + vi.mocked(stat).mockRejectedValueOnce(Object.assign(new Error("gone"), { code: "ENOENT" })) + const relativePath = nodePath.normalize("test/vanished.js") + failures.renew(relativePath, "h") + failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") + + await truncationScanner.scanDirectory("/test") + + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + }) + + it("drains queued writes and propagates an aborted write's error while a parse is pending", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) + const controller = new AbortController() + const abortError = new Error("aborted write") + let releaseSecondParse: () => void = () => {} + const secondParse = new Promise((resolve) => { + releaseSecondParse = resolve + }) + const block = { + file_path: "test/file1.js", + content: "function hello() {}", + start_line: 1, + end_line: 1, + identifier: "hello", + type: "function", + fileHash: "hash1", + segmentHash: "seg-hash-1", + } + vi.mocked(mockCodeParser.parseFile) + .mockResolvedValueOnce([block]) + .mockImplementationOnce(async () => { + await secondParse + return [] + }) + vi.mocked(mockEmbedder.createEmbeddings).mockImplementationOnce(async () => { + controller.abort() + setTimeout(releaseSecondParse, 0) + throw abortError + }) + + await expect( + scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), + ).rejects.toBe(abortError) + }) + it("records a non-missing read error as a read failure", async () => { const failures = new FileFailureRegistry() const readScanner = new DirectoryScanner( diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 13c198e959..4e059065ac 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -111,6 +111,7 @@ export class DirectoryScanner implements IDirectoryScanner { }) const processedFiles = new Set() + const missingFiles = new Set() let processedCount = 0 let skippedCount = 0 let totalBlockCount = 0 @@ -118,6 +119,7 @@ export class DirectoryScanner implements IDirectoryScanner { const parseLimiter = pLimit(PARSING_CONCURRENCY) const writeLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) const writes: Promise[] = [] + let writeFailure: { error: unknown } | undefined const recordLocalFailure = ( filePath: string, @@ -141,13 +143,19 @@ export class DirectoryScanner implements IDirectoryScanner { try { const stats = await readSource(stat(filePath)) // A file deleted after listing is a removal: leaving it out lets the removal phase delete it. - if (stats === SOURCE_MISSING) return + if (stats === SOURCE_MISSING) { + missingFiles.add(generateRelativeFilePath(filePath, scanWorkspace)) + return + } if (stats.size > MAX_FILE_SIZE_BYTES) { skippedCount++ return } const buffer = await readSource(vscode.workspace.fs.readFile(vscode.Uri.file(filePath))) - if (buffer === SOURCE_MISSING) return + if (buffer === SOURCE_MISSING) { + missingFiles.add(generateRelativeFilePath(filePath, scanWorkspace)) + return + } content = Buffer.from(buffer).toString("utf-8") } catch (error) { processedFiles.add(filePath) @@ -180,11 +188,14 @@ export class DirectoryScanner implements IDirectoryScanner { } totalBlockCount += blocks.length + // The handler is attached at queue time; parses still pending must not leave a rejection unhandled. writes.push( writeLimiter(async () => { if (signal?.aborted) return const indexed = await writer.replace(filePath, currentFileHash, blocks, signal) if (indexed) onBlocksIndexed?.(blocks.filter((block) => block.content.trim()).length) + }).catch((error: unknown) => { + writeFailure ??= { error } }), ) }), @@ -192,6 +203,7 @@ export class DirectoryScanner implements IDirectoryScanner { await Promise.all(parsePromises) await Promise.all(writes) + if (writeFailure) throw writeFailure.error if (signal?.aborted) { return { stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount } @@ -202,8 +214,9 @@ export class DirectoryScanner implements IDirectoryScanner { removalCandidates.set(generateRelativeFilePath(cachedFilePath, scanWorkspace), cachedFilePath) } // A file whose first write failed has no hash, but its failure entry means partial points may exist. - // A truncated listing cannot prove a failure-only file is gone, so those are kept. - for (const failure of listingTruncated ? [] : this.failures.list()) { + // A truncated listing cannot prove an unlisted failure-only file is gone; a confirmed-missing one is. + for (const failure of this.failures.list()) { + if (listingTruncated && !missingFiles.has(failure.path)) continue if (!removalCandidates.has(failure.path)) { removalCandidates.set(failure.path, generateNormalizedAbsolutePath(failure.path, scanWorkspace)) } From e70cb6d859fd21ac52665566a4310fcf167b6a5b Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 04:31:03 -0800 Subject: [PATCH 39/42] test(code-index): assert abort reason, cover truncated listing at stat and read --- .../processors/__tests__/scanner.spec.ts | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 8acfcc9ee0..1e49e29d58 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -549,23 +549,28 @@ describe("DirectoryScanner", () => { expect(failures.list()).toEqual([]) }) + function createScannerWithFailures() { + const failures = new FileFailureRegistry() + const failureScanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + undefined, + failures, + new ProviderPacerRegistry(), + ) + return { failures, failureScanner } + } + it.each([ { truncated: true, removed: false }, { truncated: false, removed: true }, ])( "failure-only file with truncated listing=$truncated is removed=$removed", async ({ truncated, removed }) => { - const failures = new FileFailureRegistry() - const truncationScanner = new DirectoryScanner( - mockEmbedder, - mockVectorStore, - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - undefined, - failures, - new ProviderPacerRegistry(), - ) + const { failures, failureScanner: truncationScanner } = createScannerWithFailures() const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([[], truncated]) const relativePath = nodePath.normalize("src/unlisted.js") @@ -579,32 +584,27 @@ describe("DirectoryScanner", () => { }, ) - it("removes a listed failure-only file confirmed missing even when the listing is truncated", async () => { - const failures = new FileFailureRegistry() - const truncationScanner = new DirectoryScanner( - mockEmbedder, - mockVectorStore, - mockCodeParser, - mockCacheManager, - mockIgnoreInstance, - undefined, - failures, - new ProviderPacerRegistry(), - ) - const { listFiles } = await import("../../../glob/list-files") - vi.mocked(listFiles).mockResolvedValue([["test/vanished.js"], true]) - vi.mocked(stat).mockRejectedValueOnce(Object.assign(new Error("gone"), { code: "ENOENT" })) - const relativePath = nodePath.normalize("test/vanished.js") - failures.renew(relativePath, "h") - failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") + it.each(["stat", "read"] as const)( + "removes a listed failure-only file missing at %s even when the listing is truncated", + async (stage) => { + const { failures, failureScanner: truncationScanner } = createScannerWithFailures() + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/vanished.js"], true]) + const missing = Object.assign(new Error("gone"), { code: "ENOENT" }) + if (stage === "stat") vi.mocked(stat).mockRejectedValueOnce(missing) + else vi.mocked(vscode.workspace.fs.readFile).mockRejectedValueOnce(missing) + const relativePath = nodePath.normalize("test/vanished.js") + failures.renew(relativePath, "h") + failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") - await truncationScanner.scanDirectory("/test") + await truncationScanner.scanDirectory("/test") - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() - expect(failures.list()).toEqual([]) - }) + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + }, + ) - it("drains queued writes and propagates an aborted write's error while a parse is pending", async () => { + it("drains queued writes and rejects with the abort reason while a parse is pending", async () => { const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) const controller = new AbortController() @@ -637,7 +637,7 @@ describe("DirectoryScanner", () => { await expect( scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), - ).rejects.toBe(abortError) + ).rejects.toBe(controller.signal.reason) }) it("records a non-missing read error as a read failure", async () => { From 83e455a0275eecb5e60a2db69e8864b7f108e91b Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 04:33:41 -0800 Subject: [PATCH 40/42] fix(code-index): drop finish events from batches of a stopped session --- .../processors/__tests__/file-watcher.spec.ts | 15 +++++++++++++++ .../code-index/processors/file-watcher.ts | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index c04be5c935..3b2e8a557a 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -542,6 +542,21 @@ describe("FileWatcher", () => { expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() expect(mockCacheManager.updateHash).not.toHaveBeenCalled() }) + + it("fires no batch-finished or progress events for a batch whose session was stopped", async () => { + watcherWithRegistry() + const controller = new AbortController() + await fileWatcher.initialize(controller.signal) + await mockOnDidChange({ fsPath: path }) + const finished = vi.fn() + const progress = vi.fn() + fileWatcher.onDidFinishBatchProcessing(finished) + fileWatcher.onBatchProgressUpdate(progress) + controller.abort() + await flushBatch() + expect(finished).not.toHaveBeenCalled() + expect(progress).not.toHaveBeenCalledWith(expect.objectContaining({ processedInBatch: 1 })) + }) }) describe("dispose", () => { diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 04f388cc0e..0bc38e494f 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -286,6 +286,8 @@ export class FileWatcher implements IFileWatcher { } } + // The orchestrator reset its counter on stop; a stale finish would decrement the next session's count. + if (signal?.aborted) return this._onDidFinishBatchProcessing.fire({ processedFiles: batchResults }) this._onBatchProgressUpdate.fire({ processedInBatch: totalFilesInBatch, totalInBatch: totalFilesInBatch }) if (this.accumulatedEvents.size === 0) { From 198520cf6e04d127f323bbb8d75f02fc50185af5 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 04:48:22 -0800 Subject: [PATCH 41/42] test(code-index): read abort reason after scan settles in scanner abort test --- .../processors/__tests__/scanner.spec.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 1e49e29d58..c991b5fba3 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -635,9 +635,17 @@ describe("DirectoryScanner", () => { throw abortError }) - await expect( - scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), - ).rejects.toBe(controller.signal.reason) + const rejection = await scanner + .scanDirectory("/test", undefined, undefined, undefined, controller.signal) + .then( + () => undefined, + (error: unknown) => error, + ) + + // Read the reason only after the scan settles; it is undefined until abort() runs. + expect(controller.signal.aborted).toBe(true) + expect(rejection).toBeInstanceOf(DOMException) + expect(rejection).toBe(controller.signal.reason) }) it("records a non-missing read error as a read failure", async () => { From a3ae407096efbe854de04b8405ac2b91d7328d72 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 14 Nov 2026 05:14:03 -0800 Subject: [PATCH 42/42] fix(code-index): drop progress from stopped watcher chunks; clean up oversized failure-only files on truncated scans --- .../processors/__tests__/file-watcher.spec.ts | 23 +++++++++++++++++++ .../processors/__tests__/scanner.spec.ts | 15 ++++++++++++ .../code-index/processors/file-watcher.ts | 1 + src/services/code-index/processors/scanner.ts | 2 ++ 4 files changed, 41 insertions(+) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 3b2e8a557a..7c5b777f3f 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -557,6 +557,29 @@ describe("FileWatcher", () => { expect(finished).not.toHaveBeenCalled() expect(progress).not.toHaveBeenCalledWith(expect.objectContaining({ processedInBatch: 1 })) }) + + it("fires no processed-file progress when the session stops while a chunk is in flight", async () => { + watcherWithRegistry() + const controller = new AbortController() + let releasePrepare: () => void = () => {} + vi.spyOn(FilePreparation.prototype, "prepareFile").mockImplementationOnce( + () => + new Promise((resolve) => { + releasePrepare = () => + resolve({ path, status: "local_error", activity: "read", error: new Error("x") }) + }), + ) + await fileWatcher.initialize(controller.signal) + const progress = vi.fn() + fileWatcher.onBatchProgressUpdate(progress) + await mockOnDidChange({ fsPath: path }) + await flushBatch() + expect(progress).toHaveBeenCalledWith(expect.objectContaining({ processedInBatch: 0, totalInBatch: 1 })) + controller.abort() + releasePrepare() + await vi.advanceTimersByTimeAsync(0) + expect(progress).not.toHaveBeenCalledWith(expect.objectContaining({ processedInBatch: 1 })) + }) }) describe("dispose", () => { diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index c991b5fba3..867874c985 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -604,6 +604,21 @@ describe("DirectoryScanner", () => { }, ) + it("removes a listed oversized failure-only file even when the listing is truncated", async () => { + const { failures, failureScanner: truncationScanner } = createScannerWithFailures() + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/big.js"], true]) + vi.mocked(stat).mockResolvedValueOnce({ ...mockStats, size: 2 * 1024 * 1024 }) + const relativePath = nodePath.normalize("test/big.js") + failures.renew(relativePath, "h") + failures.record(relativePath, "h", "upsert", "down", 3, "exhausted") + + await truncationScanner.scanDirectory("/test") + + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(failures.list()).toEqual([]) + }) + it("drains queued writes and rejects with the abort reason while a parse is pending", async () => { const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 0bc38e494f..ff45562236 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -275,6 +275,7 @@ export class FileWatcher implements IFileWatcher { } }), ) + if (signal?.aborted) break for (const result of results) { batchResults.push(result) processedCountInBatch++ diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 4e059065ac..ec8360f092 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -148,6 +148,8 @@ export class DirectoryScanner implements IDirectoryScanner { return } if (stats.size > MAX_FILE_SIZE_BYTES) { + // Confirmed ineligible, so a truncated listing may still remove its partial points. + missingFiles.add(generateRelativeFilePath(filePath, scanWorkspace)) skippedCount++ return }