From 8d1f30a32bb713b8aad7eca33428bdf53d7118f0 Mon Sep 17 00:00:00 2001 From: Roomote Date: Thu, 5 Nov 2026 20:49:32 +0000 Subject: [PATCH 1/3] fix(telemetry): capture one exception per API error occurrence --- packages/telemetry/src/TelemetryService.ts | 92 ++++++++++ ...ryService.exception-single-capture.test.ts | 128 +++++++++++++ packages/types/src/telemetry.ts | 11 ++ .../providers/__tests__/openrouter.spec.ts | 10 +- src/api/providers/openrouter.ts | 16 +- src/core/task/__tests__/Task.spec.ts | 168 ++++++++++++++++++ 6 files changed, 419 insertions(+), 6 deletions(-) create mode 100644 packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts diff --git a/packages/telemetry/src/TelemetryService.ts b/packages/telemetry/src/TelemetryService.ts index 4b4115e9db..3de061bbde 100644 --- a/packages/telemetry/src/TelemetryService.ts +++ b/packages/telemetry/src/TelemetryService.ts @@ -7,6 +7,7 @@ import { TelemetryEventName, type TelemetrySetting, type ToolUsage, + isApiProviderError, } from "@roo-code/types" /** @@ -27,6 +28,43 @@ const CIRCUIT_BREAKER_WINDOW_MS = 10 * 60 * 1000 /** How long a tripped breaker stays tripped before allowing captures again. */ const CIRCUIT_BREAKER_COOLDOWN_MS = 10 * 60 * 1000 +/** + * How long an exception fingerprint is remembered for the single-capture + * policy below. Provider retry loops re-enter the provider per attempt and each + * attempt builds a fresh error, so identical errors re-thrown across attempts + * collapse into the first capture within this window. A recurring identical + * error keeps the window alive while it keeps happening, so one persistent + * failure surfaces as one $exception event instead of one per retry. + */ +const EXCEPTION_DEDUPE_WINDOW_MS = 30 * 1000 + +/** + * Upper bound on remembered exception fingerprints (oldest evicted) so a long + * session streaming many distinct errors cannot grow the map without bound. + */ +const EXCEPTION_DEDUPE_CACHE_LIMIT = 100 + +/** + * Stable identity for an exception under the single-capture policy: everything + * providers put on an ApiProviderError (provider, model, operation, code, + * message), or type plus message for other errors. Two errors with the same + * fingerprint within the dedupe window are treated as one occurrence. + */ +function exceptionFingerprint(error: Error): string { + if (isApiProviderError(error)) { + return [ + "ApiProviderError", + error.provider, + error.modelId, + error.operation, + error.errorCode ?? "", + error.message, + ].join("|") + } + + return [error.name, error.message].join("|") +} + /** * Upper bound applied separately to each phase of shutdown(): draining in-flight capture * calls, then awaiting client.shutdown(). deactivate() awaits shutdown() before terminal @@ -49,6 +87,13 @@ export class TelemetryService { private guardedEventOccurrences = new Map() private trippedUntil = new Map() + // Exception fingerprint -> time of the latest capture attempt, oldest first. + // Backs the single-capture policy in captureException(); entries are re-dated + // (and moved to the back) on every dedupe hit so the window slides while an + // identical error keeps recurring, and the oldest entry is the eviction + // candidate when the cache outgrows EXCEPTION_DEDUPE_CACHE_LIMIT. + private recentExceptionFingerprints = new Map() + // In-flight client.capture()/captureException() promises. captureEvent/captureException are // synchronous (void-returning) for callers, but the underlying client calls are async (e.g. // PostHogTelemetryClient awaits property enrichment before enqueueing). Tracked here so @@ -171,6 +216,53 @@ export class TelemetryService { return } + // Single-capture policy (issue #513): one error occurrence must produce one + // $exception event, but provider catch blocks capture and re-throw, and the + // retry loop in Task#attemptApiRequest re-enters the provider per attempt, + // so the same failure reaches this method several times. Two guards, in + // order: + // + // 1. Identity. An ApiProviderError is marked on its first capture; any + // later capture of the same instance (enclosing catch, re-throw) is a + // no-op. Marking happens only after the readiness check above, so an + // error captured while telemetry is down can still be captured later. + // + // 2. Fingerprint window. Each retry attempt constructs a fresh + // ApiProviderError, which identity cannot see, so identical errors + // within EXCEPTION_DEDUPE_WINDOW_MS collapse into their first capture. + if (isApiProviderError(error)) { + if (error.captured) { + return + } + + error.captured = true + } + + const fingerprint = exceptionFingerprint(error) + const now = Date.now() + const lastSeenAt = this.recentExceptionFingerprints.get(fingerprint) + + if (lastSeenAt !== undefined && now - lastSeenAt < EXCEPTION_DEDUPE_WINDOW_MS) { + // Re-dating an existing key must move it to the back so the oldest + // entry stays the correct eviction candidate. + this.recentExceptionFingerprints.delete(fingerprint) + this.recentExceptionFingerprints.set(fingerprint, now) + return + } + + this.recentExceptionFingerprints.delete(fingerprint) + this.recentExceptionFingerprints.set(fingerprint, now) + + while (this.recentExceptionFingerprints.size > EXCEPTION_DEDUPE_CACHE_LIMIT) { + const oldest = this.recentExceptionFingerprints.keys().next().value + + if (oldest === undefined) { + break + } + + this.recentExceptionFingerprints.delete(oldest) + } + this.clients.forEach((client) => this.trackPendingClientCall(client.captureException(error, additionalProperties)), ) diff --git a/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts b/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts new file mode 100644 index 0000000000..c4f6bff65b --- /dev/null +++ b/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts @@ -0,0 +1,128 @@ +// pnpm --filter @roo-code/telemetry test src/__tests__/TelemetryService.exception-single-capture.test.ts + +import { ApiProviderError, type TelemetryClient } from "@roo-code/types" + +import { TelemetryService } from "../TelemetryService" + +function makeApiProviderError(message = "upstream failure", operation = "createMessage"): ApiProviderError { + return new ApiProviderError(message, "anthropic", "claude-3-5-sonnet-20241022", operation) +} + +function makeRecordingClient(): TelemetryClient { + return { + setProvider: vi.fn(), + capture: vi.fn().mockResolvedValue(undefined), + captureException: vi.fn().mockResolvedValue(undefined), + updateTelemetryState: vi.fn(), + isTelemetryEnabled: vi.fn().mockReturnValue(true), + shutdown: vi.fn().mockResolvedValue(undefined), + } +} + +describe("TelemetryService.captureException single-capture policy", () => { + let mockClient: TelemetryClient + + beforeEach(() => { + vi.useFakeTimers() + mockClient = makeRecordingClient() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it("marks a captured ApiProviderError so re-throws of the same instance are no-ops", () => { + const service = new TelemetryService([mockClient]) + const apiError = makeApiProviderError() + + service.captureException(apiError) + service.captureException(apiError) + service.captureException(apiError) + + expect(mockClient.captureException).toHaveBeenCalledTimes(1) + expect(mockClient.captureException).toHaveBeenCalledWith(apiError, undefined) + expect(apiError.captured).toBe(true) + }) + + it("keeps ignoring the same instance after the dedupe window (identity beats the window)", () => { + const service = new TelemetryService([mockClient]) + const apiError = makeApiProviderError() + + service.captureException(apiError) + vi.advanceTimersByTime(60_000) + service.captureException(apiError) + + expect(mockClient.captureException).toHaveBeenCalledTimes(1) + }) + + it("collapses fresh identical ApiProviderErrors built per retry attempt within the window", () => { + const service = new TelemetryService([mockClient]) + + // Simulates the retry loop: each attempt catches a fresh failure, builds + // a fresh ApiProviderError with the same shape, captures it, and re-throws. + for (let attempt = 0; attempt < 5; attempt++) { + service.captureException(makeApiProviderError()) + } + + expect(mockClient.captureException).toHaveBeenCalledTimes(1) + }) + + it("captures an identical fresh error again once the window has passed", () => { + const service = new TelemetryService([mockClient]) + + service.captureException(makeApiProviderError()) + vi.advanceTimersByTime(31_000) + service.captureException(makeApiProviderError()) + + expect(mockClient.captureException).toHaveBeenCalledTimes(2) + }) + + it("slides the window while an identical error keeps recurring", () => { + const service = new TelemetryService([mockClient]) + + service.captureException(makeApiProviderError()) + for (const gapMs of [10_000, 10_000, 10_000, 10_000]) { + vi.advanceTimersByTime(gapMs) + service.captureException(makeApiProviderError()) + } + + expect(mockClient.captureException).toHaveBeenCalledTimes(1) + }) + + it("does not collapse distinct errors", () => { + const service = new TelemetryService([mockClient]) + + service.captureException(makeApiProviderError("first failure")) + service.captureException(makeApiProviderError("second failure")) + service.captureException(makeApiProviderError("first failure", "completePrompt")) + + expect(mockClient.captureException).toHaveBeenCalledTimes(3) + }) + + it("collapses plain Errors with the same name and message within the window", () => { + const service = new TelemetryService([mockClient]) + + service.captureException(new Error("connection reset")) + service.captureException(new Error("connection reset")) + service.captureException(new Error("connection refused")) + + expect(mockClient.captureException).toHaveBeenCalledTimes(2) + }) + + it("leaves the error unmarked when no clients are registered so it can be captured later", () => { + const idleService = new TelemetryService([]) + const apiError = makeApiProviderError() + + idleService.captureException(apiError) + + expect(mockClient.captureException).not.toHaveBeenCalled() + expect(apiError.captured).toBeUndefined() + + // The readiness check runs before the flag is set, so an error captured + // while telemetry was down is not consumed by the failed attempt. + const readyService = new TelemetryService([mockClient]) + readyService.captureException(apiError) + + expect(mockClient.captureException).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/types/src/telemetry.ts b/packages/types/src/telemetry.ts index fce105e0e5..5dfa32703e 100644 --- a/packages/types/src/telemetry.ts +++ b/packages/types/src/telemetry.ts @@ -469,6 +469,17 @@ export function shouldReportApiErrorToTelemetry(errorCode?: number, errorMessage * Can be reused by any API provider. */ export class ApiProviderError extends Error { + /** + * Set by TelemetryService#captureException once this instance has been sent + * to the telemetry clients. Provider catch blocks re-throw the errors they + * captured, so the same instance can reach captureException again (through + * an enclosing catch or a retry loop). Checking this flag keeps one error + * occurrence at one $exception event instead of inflating the count. + * + * Optional: an instance that was never captured reads as `undefined`. + */ + public captured?: boolean + constructor( message: string, public readonly provider: string, diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index fec3e4a2e2..93bedbfa60 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -608,7 +608,9 @@ describe("OpenRouterHandler", () => { expect(mockCaptureException).toHaveBeenCalledWith( expect.objectContaining({ - message: "API Error", + // handleStreamingError throws the instance it captured, so the + // payload carries the same caller-facing message. + message: "OpenRouter API Error 500: API Error", provider: providerIdentifiers.openrouter, modelId: mockOptions.openRouterModelId, operation: "createMessage", @@ -717,7 +719,7 @@ describe("OpenRouterHandler", () => { expect(mockCaptureException).toHaveBeenCalledWith( expect.objectContaining({ - message: "Rate limit exceeded", + message: "OpenRouter API Error 429: Rate limit exceeded", provider: providerIdentifiers.openrouter, modelId: mockOptions.openRouterModelId, operation: "createMessage", @@ -1018,7 +1020,7 @@ describe("OpenRouterHandler", () => { // Verify telemetry was captured expect(mockCaptureException).toHaveBeenCalledWith( expect.objectContaining({ - message: "API Error", + message: "OpenRouter API Error 500: API Error", provider: providerIdentifiers.openrouter, modelId: mockOptions.openRouterModelId, operation: "completePrompt", @@ -1134,7 +1136,7 @@ describe("OpenRouterHandler", () => { // captureException is called, but PostHogTelemetryClient filters out 429 errors internally expect(mockCaptureException).toHaveBeenCalledWith( expect.objectContaining({ - message: "Rate limit exceeded", + message: "OpenRouter API Error 429: Rate limit exceeded", provider: providerIdentifiers.openrouter, modelId: mockOptions.openRouterModelId, operation: "completePrompt", diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 6d2057a7a3..eec790d23d 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -194,6 +194,12 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH /** * Handle OpenRouter streaming error response and report to telemetry. * OpenRouter may include metadata.raw with the actual upstream provider error. + * This method is the single capture point for mid-stream OpenRouter errors: + * it throws the same ApiProviderError instance it captured, and + * TelemetryService#captureException marks that instance, so an enclosing + * catch that captures the re-thrown error is a no-op instead of a second + * $exception event (issue #513, Bug 4). The message matches what this method + * always threw, so caller-facing behavior is unchanged. * @param error The error object (not wrapped - receives the error directly) */ private handleStreamingError(error: OpenRouterError, modelId: string, operation: string): never { @@ -202,13 +208,19 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH const rawErrorMessage = parsedError || error?.message || "Unknown error" const apiError = Object.assign( - new ApiProviderError(rawErrorMessage, providerIdentifiers.openrouter, modelId, operation, error?.code), + new ApiProviderError( + `OpenRouter API Error ${error?.code}: ${rawErrorMessage}`, + providerIdentifiers.openrouter, + modelId, + operation, + error?.code, + ), { status: error?.code, error }, ) TelemetryService.instance.captureException(apiError) - throw new Error(`OpenRouter API Error ${error?.code}: ${rawErrorMessage}`) + throw apiError } override async *createMessage( diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 739b7418a2..ceccb22b83 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -11,8 +11,10 @@ import { Anthropic } from "@anthropic-ai/sdk" import type { Mock } from "vitest" import { + ApiProviderError, providerIdentifiers, RooCodeEventName, + TelemetryEventName, type ClineMessage, type GlobalState, type HistoryItem, @@ -20,6 +22,7 @@ import { type ProviderSettings, type ModelInfo, type TaskLike, + type TelemetryClient, } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" @@ -670,6 +673,171 @@ describe("Cline", () => { }) }) + /** + * Issue #513 regression coverage. Provider catch blocks capture the failure + * and re-throw, and the retry loop in Task#attemptApiRequest re-enters the + * provider per attempt, so one persistent API error used to be captured once + * per attempt. These tests pin the single-capture policy at the seam where + * the amplification happens. + */ + describe("api error telemetry single-capture", () => { + let recordingClient: TelemetryClient + let telemetryInstanceSlot: { _instance: TelemetryService | null } + + beforeEach(() => { + recordingClient = { + setProvider: vi.fn(), + capture: vi.fn().mockResolvedValue(undefined), + captureException: vi.fn().mockResolvedValue(undefined), + updateTelemetryState: vi.fn(), + isTelemetryEnabled: vi.fn().mockReturnValue(true), + shutdown: vi.fn().mockResolvedValue(undefined), + } + + // Swap the shared singleton for one wired to the recording client. The + // cast names the one private static member it touches; TelemetryService + // has no public reset, and afterEach hands the slot back by nulling it + // so the shared beforeEach re-creates the default instance. + telemetryInstanceSlot = TelemetryService as unknown as { _instance: TelemetryService | null } + telemetryInstanceSlot._instance = null + TelemetryService.createInstance([recordingClient]) + }) + + afterEach(() => { + telemetryInstanceSlot._instance = null + }) + + async function createAutoApprovedTask() { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: true, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + consecutiveMistakeLimit: 3, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + return task + } + + it("captures exactly one exception when a provider captures and throws across retry attempts", async () => { + const task = await createAutoApprovedTask() + vi.spyOn(getTaskTestAccess(task), "getSystemPrompt").mockResolvedValue("mock system prompt") + + // Spies observe without replacing: this counts provider-side captures + // while the real single-capture policy decides what reaches the client. + const serviceCaptureSpy = vi.spyOn(TelemetryService.instance, "captureException") + + // Fault injection at the provider seam: each failed attempt mimics the + // provider contract (build a fresh ApiProviderError, capture it, throw + // it) and the third attempt recovers so the real retry loop terminates. + let attempts = 0 + vi.spyOn(task.api, "createMessage").mockImplementation(() => { + attempts++ + if (attempts <= 2) { + const apiError = new ApiProviderError( + "upstream exploded", + providerIdentifiers.anthropic, + "claude-3-5-sonnet-20241022", + "createMessage", + ) + TelemetryService.instance.captureException(apiError) + return { + // eslint-disable-next-line require-yield -- fault-injection stream: the provider contract under test throws on the first chunk + async *[Symbol.asyncIterator]() { + throw apiError + }, + async next() { + throw apiError + }, + async return() { + return { done: true, value: undefined } + }, + async throw(error: unknown) { + throw error + }, + async [Symbol.asyncDispose]() {}, + } as AsyncGenerator + } + return { + async *[Symbol.asyncIterator]() { + yield { type: "text", text: "recovered" } + }, + async next() { + return { done: false, value: { type: "text", text: "recovered" } as ApiStreamChunk } + }, + async return() { + return { done: true, value: undefined } + }, + async throw(error: unknown) { + throw error + }, + async [Symbol.asyncDispose]() {}, + } as AsyncGenerator + }) + + // Drive the real retry loop: attempts 1 and 2 throw after capturing, + // the catch backs off (requestDelaySeconds is 0) and recurses, and + // attempt 3 succeeds. + const chunks: ApiStreamChunk[] = [] + for await (const chunk of task.attemptApiRequest(0)) { + chunks.push(chunk) + } + + expect(attempts).toBe(3) + expect(serviceCaptureSpy).toHaveBeenCalledTimes(2) + expect(recordingClient.captureException).toHaveBeenCalledTimes(1) + expect(recordingClient.captureException).toHaveBeenCalledWith( + expect.objectContaining({ + message: "upstream exploded", + provider: providerIdentifiers.anthropic, + operation: "createMessage", + captured: true, + }), + undefined, + ) + expect(chunks).toEqual([{ type: "text", text: "recovered" }]) + }) + + it("reports the consecutive mistake limit as a named event without capturing an exception", async () => { + const task = await createAutoApprovedTask() + // The guard fires at the top of the request loop, before any provider + // call. Answering it sets abort, and the interstitial api_req_started + // say() then refuses to run on an aborted task, which deterministically + // ends the loop without ever issuing the underlying API request. + task.consecutiveMistakeCount = 3 + vi.spyOn(getTaskTestAccess(task), "getSystemPrompt").mockResolvedValue("mock system prompt") + + const askSpy = vi.spyOn(task, "ask").mockImplementation(async (type) => { + if (type === "mistake_limit_reached") { + task.abort = true + return { response: "yesButtonClicked" } satisfies TaskAskResult + } + throw new Error(`unexpected ask: ${type}`) + }) + const mistakeEventSpy = vi.spyOn(TelemetryService.instance, "captureConsecutiveMistakeError") + const createMessageSpy = vi.spyOn(task.api, "createMessage") + + await expect( + task.recursivelyMakeClineRequests([{ type: "text", text: "original request" }]), + ).rejects.toThrow("aborted") + + expect(askSpy).toHaveBeenCalledWith("mistake_limit_reached", expect.any(String)) + expect(mistakeEventSpy).toHaveBeenCalledWith(task.taskId, "unknown", 3, task.consecutiveMistakeLimit) + expect(createMessageSpy).not.toHaveBeenCalled() + // The mistake limit is control flow: it reaches telemetry as the named + // event only, never as a $exception. + expect(recordingClient.captureException).not.toHaveBeenCalled() + expect(recordingClient.capture).toHaveBeenCalledWith( + expect.objectContaining({ event: TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR }), + ) + }) + }) + describe("native tool-call request isolation", () => { it("derives readiness only from a closed, idle, fully paired tool turn", () => { const task = new Task({ From 5a4822619697cc34a4daf81acac9df21c7485004 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 20 Nov 2026 04:41:46 +0000 Subject: [PATCH 2/3] fix(telemetry): model exception deduplication and address review feedback --- .../tla/ExceptionCapture.Joined.cfg | 4 + .../tla/ExceptionCapture.Mutating.cfg | 4 + .../tla/ExceptionCapture.NoIdentity.cfg | 4 + .../tla/ExceptionCapture.NoWindow.cfg | 4 + .../tla/ExceptionCapture.Unbounded.cfg | 4 + docs/architecture/tla/ExceptionCapture.cfg | 4 + docs/architecture/tla/ExceptionCapture.tla | 66 ++++++++++++++++ .../telemetry/src/PostHogTelemetryClient.ts | 9 ++- packages/telemetry/src/TelemetryService.ts | 14 ++-- ...lemetryClient.message-preservation.test.ts | 40 ++++++++++ .../__tests__/PostHogTelemetryClient.test.ts | 26 ++++--- ...ryService.exception-single-capture.test.ts | 76 +++++++++++++++++++ src/core/task/__tests__/Task.spec.ts | 4 +- 13 files changed, 236 insertions(+), 23 deletions(-) create mode 100644 docs/architecture/tla/ExceptionCapture.Joined.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.Mutating.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.NoIdentity.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.NoWindow.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.Unbounded.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.cfg create mode 100644 docs/architecture/tla/ExceptionCapture.tla create mode 100644 packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts diff --git a/docs/architecture/tla/ExceptionCapture.Joined.cfg b/docs/architecture/tla/ExceptionCapture.Joined.cfg new file mode 100644 index 0000000000..f1db0cde3a --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.Joined.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "Joined" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.Mutating.cfg b/docs/architecture/tla/ExceptionCapture.Mutating.cfg new file mode 100644 index 0000000000..26ee816747 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.Mutating.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "Mutating" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg b/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg new file mode 100644 index 0000000000..27210bd304 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "NoIdentity" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.NoWindow.cfg b/docs/architecture/tla/ExceptionCapture.NoWindow.cfg new file mode 100644 index 0000000000..704d46f650 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.NoWindow.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "NoWindow" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.Unbounded.cfg b/docs/architecture/tla/ExceptionCapture.Unbounded.cfg new file mode 100644 index 0000000000..e8b0486247 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.Unbounded.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "Unbounded" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.cfg b/docs/architecture/tla/ExceptionCapture.cfg new file mode 100644 index 0000000000..62b54e0ac7 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANTS Mode = "Fixed" Limit = 2 Steps = 3 +INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.tla b/docs/architecture/tla/ExceptionCapture.tla new file mode 100644 index 0000000000..e5273c8dd4 --- /dev/null +++ b/docs/architecture/tla/ExceptionCapture.tla @@ -0,0 +1,66 @@ +---- MODULE ExceptionCapture ---- +\* Source: TelemetryService.captureException. Sink: TelemetryClient.captureException. +\* Initial rules were recorded before review, after reading the current fix. +\* Failures: repeats emit, distinct fields collide, blocked calls consume state, or the cache grows. +\* Invariants: IdentityOnce, WindowDecision, FieldsDistinct, BlockedUnchanged, Bounded. +\* Expected: field collisions or identity and window boundary errors. No prior telemetry spec exists. +\* Review: PR 2330 discussion_r4486068932 adds CallerMessage. It was an initial gap. +\* Review: PR 2330 discussion_r4486070563 is covered by WindowDecision, including expiry. +\* The window is 30 seconds. A retry after a longer gap emits again. This is not retry-chain identity. +\* Client mutation is an oracle. The model does not copy PostHog message extraction. +\* Result: Fixed holds in 67 states. All five variants fail their invariants. Each run takes less than 2 seconds. +\* Cost: about 6 minutes of agent work for the spec, tests, replay, and fixes. Review: 1 covered, 1 added, 0 gap. +\* New finding: joined fields collide (a|b, c versus a, b|c). A real-code test failed before the JSON tuple fix. +\* Replay: the real PostHog client changed the source message. Three inputs failed before the detached-copy fix. +\* Replay: removal of identity, window, and size guards each makes its real-code test fail. +\* Null result: no new window boundary bug. Tests accept a new event at 30 seconds and at a 600-second retry gap. +\* No oracle counterexample was rejected as an impossible input. This checks the model, not code correctness. +\* Property: TelemetryService.exception-single-capture.test.ts checks all 256 four-call traces on real code. +\* Sink replay: PostHogTelemetryClient.message-preservation.test.ts uses the real client and a fake PostHog transport. +\* ready is the service gate, not the client opt-in flag. A registered client can still filter an event. +\* Each gap is time since the last cache write. Ages saturate at expiry. The model covers API errors only. +\* One id denotes one field tuple; fresh chooses a new instance. Client network success is outside this model. +\* Eviction can admit a repeat early. The window rule applies only to a retained fingerprint. + +EXTENDS Naturals, Sequences, FiniteSets +CONSTANTS Mode, Limit, Steps +VARIABLES seen, cache, step, checks +vars == <> +Ids == 1..3 +\* Two realistic field tuples have the same delimiter-joined text. +Fields(i) == CASE i = 1 -> <<"a|b", "c">> [] i = 2 -> <<"a", "b|c">> [] OTHER -> <<"a", "c">> +Key(i) == IF Mode = "Joined" /\ i \in {1, 2} THEN Fields(1) ELSE Fields(i) +Drop(c, k) == SelectSeq(c, LAMBDA e : e.key # k) +Init == seen = {} /\ cache = <<>> /\ step = 0 /\ checks = [identity |-> TRUE, window |-> TRUE, blocked |-> TRUE, caller |-> TRUE] +\* A gap of 2 is the expiry boundary; 3 represents a longer backoff gap. +Call(i, fresh, gap, ready, mutate) == + LET k == Key(i) + entries == {n \in 1..Len(cache) : cache[n].key = k} + recent == \E n \in entries : cache[n].age + gap < 2 + identity == ~fresh /\ i \in seen + emit == ready /\ (~identity \/ Mode = "NoIdentity") /\ (Mode = "NoWindow" \/ ~recent) + expected == ready /\ ~identity /\ ~recent + aged == [n \in 1..Len(cache) |-> [cache[n] EXCEPT !.age = IF @ + gap > 2 THEN 2 ELSE @ + gap]] + touched == Append(Drop(aged, k), [key |-> k, age |-> 0]) + nextCache == IF Len(touched) > Limit /\ Mode # "Unbounded" THEN Tail(touched) ELSE touched + writes == ready /\ ~identity + IN /\ step < Steps + /\ step' = step + 1 + /\ seen' = IF ready THEN seen \cup {i} ELSE seen + /\ cache' = IF writes THEN nextCache ELSE cache + /\ checks' = [identity |-> checks.identity /\ ~(emit /\ identity), + window |-> checks.window /\ (emit = expected), + blocked |-> checks.blocked /\ (~ready => seen' = seen /\ cache' = cache), + caller |-> checks.caller /\ ~(emit /\ mutate /\ Mode = "Mutating")] +Next == \E i \in Ids, fresh \in BOOLEAN, gap \in 0..3, ready \in BOOLEAN, mutate \in BOOLEAN : + Call(i, fresh, gap, ready, mutate) +Spec == Init /\ [][Next]_vars +TypeOK == seen \subseteq Ids /\ step \in 0..Steps /\ checks \in [ {"identity", "window", "blocked", "caller"} -> BOOLEAN ] +FieldsDistinct == step >= 0 => \A i, j \in Ids : Fields(i) # Fields(j) => Key(i) # Key(j) +\* checks records each decision and source preservation at each call. +WindowDecision == checks.window +IdentityOnce == checks.identity +BlockedUnchanged == checks.blocked +CallerMessage == checks.caller +Bounded == Len(cache) <= Limit +==== diff --git a/packages/telemetry/src/PostHogTelemetryClient.ts b/packages/telemetry/src/PostHogTelemetryClient.ts index 77f0bd2aa5..e321f4c89e 100644 --- a/packages/telemetry/src/PostHogTelemetryClient.ts +++ b/packages/telemetry/src/PostHogTelemetryClient.ts @@ -116,8 +116,11 @@ export class PostHogTelemetryClient extends BaseTelemetryClient { mergedProperties = { ...extractedProperties, ...additionalProperties } } - // Override the error message with the extracted error message. - error.message = errorMessage + // Normalize a detached error so telemetry cannot change the caller-facing message. + const stack = error.stack + const telemetryError = Object.defineProperties(new Error(), Object.getOwnPropertyDescriptors(error)) + telemetryError.stack = stack + telemetryError.message = errorMessage const provider = this.providerRef?.deref() let telemetryProperties: TelemetryProperties | undefined = undefined @@ -135,7 +138,7 @@ export class PostHogTelemetryClient extends BaseTelemetryClient { $app_version: telemetryProperties?.appVersion, } - this.client.captureException(error, this.distinctId, exceptionProperties) + this.client.captureException(telemetryError, this.distinctId, exceptionProperties) } /** diff --git a/packages/telemetry/src/TelemetryService.ts b/packages/telemetry/src/TelemetryService.ts index 3de061bbde..fa402c6b2f 100644 --- a/packages/telemetry/src/TelemetryService.ts +++ b/packages/telemetry/src/TelemetryService.ts @@ -32,9 +32,11 @@ const CIRCUIT_BREAKER_COOLDOWN_MS = 10 * 60 * 1000 * How long an exception fingerprint is remembered for the single-capture * policy below. Provider retry loops re-enter the provider per attempt and each * attempt builds a fresh error, so identical errors re-thrown across attempts - * collapse into the first capture within this window. A recurring identical - * error keeps the window alive while it keeps happening, so one persistent - * failure surfaces as one $exception event instead of one per retry. + * collapse into the first capture within this window. Each fresh identical + * error refreshes the window. A gap of 30 seconds or more permits a new event. + * Exponential backoff, server retry delays, and request duration can exceed + * this window, so a persistent retry chain can emit again after such a gap. + * This bounds bursts; it does not identify an entire retry chain. */ const EXCEPTION_DEDUPE_WINDOW_MS = 30 * 1000 @@ -52,17 +54,17 @@ const EXCEPTION_DEDUPE_CACHE_LIMIT = 100 */ function exceptionFingerprint(error: Error): string { if (isApiProviderError(error)) { - return [ + return JSON.stringify([ "ApiProviderError", error.provider, error.modelId, error.operation, error.errorCode ?? "", error.message, - ].join("|") + ]) } - return [error.name, error.message].join("|") + return JSON.stringify([error.name, error.message]) } /** diff --git a/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts b/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts new file mode 100644 index 0000000000..ed1d53e338 --- /dev/null +++ b/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts @@ -0,0 +1,40 @@ +import * as vscode from "vscode" +import { PostHog } from "posthog-node" +import { ApiProviderError } from "@roo-code/types" +import { PostHogTelemetryClient } from "../PostHogTelemetryClient" +import { TelemetryService } from "../TelemetryService" + +vi.mock("posthog-node") +vi.mock("vscode", () => ({ + env: { machineId: "test" }, + workspace: { getConfiguration: vi.fn() }, +})) + +it.each(["upstream failure", "provider|failure", "error\nwith details"])( + "preserves the caller message while PostHog normalizes %s", + async (raw) => { + const captureException = vi.fn() + vi.mocked(PostHog).mockImplementation(function () { + // The SDK double supplies only the methods used by this capture path. + return { captureException, optIn: vi.fn() } as unknown as PostHog + }) + vi.mocked(vscode.workspace.getConfiguration).mockReturnValue({ + get: () => "all", + // The configuration double supplies get for the telemetry preference. + } as unknown as vscode.WorkspaceConfiguration) + const client = new PostHogTelemetryClient() + client.updateTelemetryState(true) + const service = new TelemetryService([client]) + const message = `OpenRouter API Error 500: ${raw}` + const error = Object.assign(new ApiProviderError(message, "openrouter", "model", "createMessage", 500), { + status: 500, + error: { message: raw }, + }) + service.captureException(error) + expect(error.message).toBe(message) + service.captureException(error) + expect(captureException).toHaveBeenCalledTimes(1) + expect(captureException.mock.calls[0][0]).not.toBe(error) + expect(captureException.mock.calls[0][0]).toMatchObject({ message: raw, stack: error.stack }) + }, +) diff --git a/packages/telemetry/src/__tests__/PostHogTelemetryClient.test.ts b/packages/telemetry/src/__tests__/PostHogTelemetryClient.test.ts index 1feeb1688f..97e4b8afff 100644 --- a/packages/telemetry/src/__tests__/PostHogTelemetryClient.test.ts +++ b/packages/telemetry/src/__tests__/PostHogTelemetryClient.test.ts @@ -520,7 +520,7 @@ describe("PostHogTelemetryClient", () => { expect(mockPostHogClient.captureException).not.toHaveBeenCalled() }) - it("should capture errors with nested metadata and override error.message with extracted message", () => { + it("should normalize nested metadata without changing the source message", () => { const client = new PostHogTelemetryClient() client.updateTelemetryState(true) @@ -535,14 +535,15 @@ describe("PostHogTelemetryClient", () => { client.captureException(error) - // The implementation overrides error.message with the extracted message from getErrorMessage - expect(error.message).toBe("Upstream provider error: model overloaded") - expect(mockPostHogClient.captureException).toHaveBeenCalledWith(error, "test-machine-id", { - $app_version: undefined, - }) + expect(error.message).toBe("Generic request failed") + expect(mockPostHogClient.captureException).toHaveBeenCalledWith( + expect.objectContaining({ message: "Upstream provider error: model overloaded", stack: error.stack }), + "test-machine-id", + { $app_version: undefined }, + ) }) - it("should capture errors with nested error.message and override error.message with extracted message", () => { + it("should normalize nested error.message without changing the source message", () => { const client = new PostHogTelemetryClient() client.updateTelemetryState(true) @@ -556,11 +557,12 @@ describe("PostHogTelemetryClient", () => { client.captureException(error) - // The implementation overrides error.message with the extracted message from getErrorMessage - expect(error.message).toBe("Upstream provider: connection timeout") - expect(mockPostHogClient.captureException).toHaveBeenCalledWith(error, "test-machine-id", { - $app_version: undefined, - }) + expect(error.message).toBe("Generic request failed") + expect(mockPostHogClient.captureException).toHaveBeenCalledWith( + expect.objectContaining({ message: "Upstream provider: connection timeout", stack: error.stack }), + "test-machine-id", + { $app_version: undefined }, + ) }) it("should use primary message when no nested error structure exists", () => { diff --git a/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts b/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts index c4f6bff65b..ed16b93dbe 100644 --- a/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts +++ b/packages/telemetry/src/__tests__/TelemetryService.exception-single-capture.test.ts @@ -126,3 +126,79 @@ describe("TelemetryService.captureException single-capture policy", () => { expect(mockClient.captureException).toHaveBeenCalledTimes(1) }) }) + +// FieldsDistinct: delimiters in real provider fields must not merge distinct errors. +it.each(["|", "||", "model|suffix"])("keeps field boundaries for %s", (separator) => { + const client = makeRecordingClient() + const service = new TelemetryService([client]) + service.captureException(new ApiProviderError("failure", `a${separator}b`, "c", "createMessage")) + service.captureException(new ApiProviderError("failure", "a", `b${separator}c`, "createMessage")) + expect(client.captureException).toHaveBeenCalledTimes(2) +}) + +it.each([29_999, 30_000, 40_000, 600_000])("applies the window at a %i ms retry gap", (gap) => { + vi.useFakeTimers() + try { + const client = makeRecordingClient() + const service = new TelemetryService([client]) + service.captureException(makeApiProviderError()) + vi.advanceTimersByTime(gap) + service.captureException(makeApiProviderError()) + expect(client.captureException).toHaveBeenCalledTimes(gap < 30_000 ? 1 : 2) + } finally { + vi.useRealTimers() + } +}) + +it("bounds the cache and evicts the least recently observed fingerprint", () => { + const client = makeRecordingClient() + const service = new TelemetryService([client]) + for (let i = 0; i < 100; i++) service.captureException(makeApiProviderError(`failure ${i}`)) + service.captureException(makeApiProviderError("failure 0")) + service.captureException(makeApiProviderError("failure 100")) + expect(service["recentExceptionFingerprints"].size).toBe(100) + service.captureException(makeApiProviderError("failure 0")) + expect(client.captureException).toHaveBeenCalledTimes(101) + service.captureException(makeApiProviderError("failure 1")) + expect(client.captureException).toHaveBeenCalledTimes(102) + expect(service["recentExceptionFingerprints"].size).toBe(100) +}) + +it("does not consume an error during shutdown", async () => { + const client = makeRecordingClient() + const service = new TelemetryService([client]) + await service.shutdown() + const error = makeApiProviderError() + service.captureException(error) + expect(error.captured).toBeUndefined() + expect(service["recentExceptionFingerprints"].size).toBe(0) + expect(client.captureException).not.toHaveBeenCalled() +}) + +// WindowDecision: enumerate all short traces over equal, distinct, and expired fresh errors. +it("matches the sliding-window invariant over every four-call trace", () => { + vi.useFakeTimers() + try { + for (let trace = 0; trace < 256; trace++) { + const client = makeRecordingClient() + const service = new TelemetryService([client]) + const lastSeen = new Map() + let expected = 0 + let choices = trace + for (let step = 0; step < 4; step++) { + const choice = choices % 4 + choices = Math.floor(choices / 4) + const message = choice === 1 ? "different" : "failure" + vi.advanceTimersByTime(choice === 2 ? 30_000 : choice === 3 ? 10_000 : 0) + const now = Date.now() + const previous = lastSeen.get(message) + if (previous === undefined || now - previous >= 30_000) expected++ + lastSeen.set(message, now) + service.captureException(makeApiProviderError(message)) + expect(client.captureException, `trace ${trace}, step ${step}`).toHaveBeenCalledTimes(expected) + } + } + } finally { + vi.useRealTimers() + } +}) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index ceccb22b83..12ea754c65 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -814,7 +814,7 @@ describe("Cline", () => { const askSpy = vi.spyOn(task, "ask").mockImplementation(async (type) => { if (type === "mistake_limit_reached") { - task.abort = true + vi.spyOn(task, "abort", "get").mockReturnValue(true) return { response: "yesButtonClicked" } satisfies TaskAskResult } throw new Error(`unexpected ask: ${type}`) @@ -824,7 +824,7 @@ describe("Cline", () => { await expect( task.recursivelyMakeClineRequests([{ type: "text", text: "original request" }]), - ).rejects.toThrow("aborted") + ).rejects.toThrow(new Error(`[RooCode#say] task ${task.taskId}.${task.instanceId} aborted`)) expect(askSpy).toHaveBeenCalledWith("mistake_limit_reached", expect.any(String)) expect(mistakeEventSpy).toHaveBeenCalledWith(task.taskId, "unknown", 3, task.consecutiveMistakeLimit) From 17181aabc90f04de4ef21fe9d287ef66b03bd0e4 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 20 Nov 2026 05:07:41 +0000 Subject: [PATCH 3/3] refactor(telemetry): simplify deduplication and strengthen TLA+ models --- .../tla/ExceptionCapture.Joined.cfg | 4 - .../tla/ExceptionCapture.Mutating.cfg | 4 - .../tla/ExceptionCapture.NoIdentity.cfg | 4 - .../tla/ExceptionCapture.NoWindow.cfg | 4 - .../tla/ExceptionCapture.Unbounded.cfg | 4 - docs/architecture/tla/ExceptionCapture.cfg | 4 - docs/architecture/tla/ExceptionCapture.tla | 128 +++++++++--------- .../tla/ExceptionCaptureMC.Joined.cfg | 14 ++ .../tla/ExceptionCaptureMC.NoIdentity.cfg | 14 ++ .../tla/ExceptionCaptureMC.NoWindow.cfg | 14 ++ .../tla/ExceptionCaptureMC.Unbounded.cfg | 14 ++ docs/architecture/tla/ExceptionCaptureMC.cfg | 14 ++ docs/architecture/tla/ExceptionCaptureMC.tla | 16 +++ docs/architecture/tla/ExceptionCopy.tla | 47 +++++++ .../tla/ExceptionCopyMC.Mutating.cfg | 7 + .../tla/ExceptionCopyMC.PlainCopy.cfg | 7 + docs/architecture/tla/ExceptionCopyMC.cfg | 7 + docs/architecture/tla/ExceptionCopyMC.tla | 15 ++ .../telemetry/src/PostHogTelemetryClient.ts | 4 +- packages/telemetry/src/TelemetryService.ts | 14 +- ...lemetryClient.message-preservation.test.ts | 70 ++++++++-- 21 files changed, 306 insertions(+), 103 deletions(-) delete mode 100644 docs/architecture/tla/ExceptionCapture.Joined.cfg delete mode 100644 docs/architecture/tla/ExceptionCapture.Mutating.cfg delete mode 100644 docs/architecture/tla/ExceptionCapture.NoIdentity.cfg delete mode 100644 docs/architecture/tla/ExceptionCapture.NoWindow.cfg delete mode 100644 docs/architecture/tla/ExceptionCapture.Unbounded.cfg delete mode 100644 docs/architecture/tla/ExceptionCapture.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.Joined.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.NoIdentity.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.NoWindow.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.Unbounded.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.cfg create mode 100644 docs/architecture/tla/ExceptionCaptureMC.tla create mode 100644 docs/architecture/tla/ExceptionCopy.tla create mode 100644 docs/architecture/tla/ExceptionCopyMC.Mutating.cfg create mode 100644 docs/architecture/tla/ExceptionCopyMC.PlainCopy.cfg create mode 100644 docs/architecture/tla/ExceptionCopyMC.cfg create mode 100644 docs/architecture/tla/ExceptionCopyMC.tla diff --git a/docs/architecture/tla/ExceptionCapture.Joined.cfg b/docs/architecture/tla/ExceptionCapture.Joined.cfg deleted file mode 100644 index f1db0cde3a..0000000000 --- a/docs/architecture/tla/ExceptionCapture.Joined.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "Joined" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.Mutating.cfg b/docs/architecture/tla/ExceptionCapture.Mutating.cfg deleted file mode 100644 index 26ee816747..0000000000 --- a/docs/architecture/tla/ExceptionCapture.Mutating.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "Mutating" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg b/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg deleted file mode 100644 index 27210bd304..0000000000 --- a/docs/architecture/tla/ExceptionCapture.NoIdentity.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "NoIdentity" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.NoWindow.cfg b/docs/architecture/tla/ExceptionCapture.NoWindow.cfg deleted file mode 100644 index 704d46f650..0000000000 --- a/docs/architecture/tla/ExceptionCapture.NoWindow.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "NoWindow" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.Unbounded.cfg b/docs/architecture/tla/ExceptionCapture.Unbounded.cfg deleted file mode 100644 index e8b0486247..0000000000 --- a/docs/architecture/tla/ExceptionCapture.Unbounded.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "Unbounded" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.cfg b/docs/architecture/tla/ExceptionCapture.cfg deleted file mode 100644 index 62b54e0ac7..0000000000 --- a/docs/architecture/tla/ExceptionCapture.cfg +++ /dev/null @@ -1,4 +0,0 @@ -SPECIFICATION Spec -CONSTANTS Mode = "Fixed" Limit = 2 Steps = 3 -INVARIANTS TypeOK FieldsDistinct WindowDecision IdentityOnce BlockedUnchanged CallerMessage Bounded -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCapture.tla b/docs/architecture/tla/ExceptionCapture.tla index e5273c8dd4..47b8818743 100644 --- a/docs/architecture/tla/ExceptionCapture.tla +++ b/docs/architecture/tla/ExceptionCapture.tla @@ -1,66 +1,72 @@ ---- MODULE ExceptionCapture ---- -\* Source: TelemetryService.captureException. Sink: TelemetryClient.captureException. -\* Initial rules were recorded before review, after reading the current fix. -\* Failures: repeats emit, distinct fields collide, blocked calls consume state, or the cache grows. -\* Invariants: IdentityOnce, WindowDecision, FieldsDistinct, BlockedUnchanged, Bounded. -\* Expected: field collisions or identity and window boundary errors. No prior telemetry spec exists. -\* Review: PR 2330 discussion_r4486068932 adds CallerMessage. It was an initial gap. -\* Review: PR 2330 discussion_r4486070563 is covered by WindowDecision, including expiry. -\* The window is 30 seconds. A retry after a longer gap emits again. This is not retry-chain identity. -\* Client mutation is an oracle. The model does not copy PostHog message extraction. -\* Result: Fixed holds in 67 states. All five variants fail their invariants. Each run takes less than 2 seconds. -\* Cost: about 6 minutes of agent work for the spec, tests, replay, and fixes. Review: 1 covered, 1 added, 0 gap. -\* New finding: joined fields collide (a|b, c versus a, b|c). A real-code test failed before the JSON tuple fix. -\* Replay: the real PostHog client changed the source message. Three inputs failed before the detached-copy fix. -\* Replay: removal of identity, window, and size guards each makes its real-code test fail. -\* Null result: no new window boundary bug. Tests accept a new event at 30 seconds and at a 600-second retry gap. -\* No oracle counterexample was rejected as an impossible input. This checks the model, not code correctness. -\* Property: TelemetryService.exception-single-capture.test.ts checks all 256 four-call traces on real code. -\* Sink replay: PostHogTelemetryClient.message-preservation.test.ts uses the real client and a fake PostHog transport. -\* ready is the service gate, not the client opt-in flag. A registered client can still filter an event. -\* Each gap is time since the last cache write. Ages saturate at expiry. The model covers API errors only. -\* One id denotes one field tuple; fresh chooses a new instance. Client network success is outside this model. -\* Eviction can admit a repeat early. The window rule applies only to a retained fingerprint. +\* Gate: TLA_JAR= ./check.sh ExceptionCaptureMC ExceptionCopyMC +\* Source: TelemetryService.captureException. Sink: each registered TelemetryClient.captureException. +\* Failures: repeats emit, distinct fields collide, blocked calls change state, or the cache grows. +\* Rules: WindowDecision, FieldsDistinct, BlockedUnchanged, and Bounded prevent those failures. +\* Expected before simplification: the shared cache refresh preserves all rules. No new code bug is expected. +\* Initial rules preceded review comments, but followed inspection of the original fix. +\* Review: PR 2330 discussion_r4486070563 is covered by WindowDecision, including window expiry. +\* Review counts for this module: 1 covered, 0 added, 0 gap. Copy findings are in ExceptionCopy. +\* Prior replay found delimiter collisions. The JSON tuple fix keeps (a|b, c) distinct from (a, b|c). +\* Prior replay removed identity, window, and size guards. Each real-code regression test failed. +\* Property tests: TelemetryService.exception-single-capture.test.ts checks 256 four-call traces and cache eviction. +\* A time advance saturates cache ages at expiry. It is separate from a capture call. +\* Instance identity is separate from field equality. Two instances can have the same fields. +\* The window applies to retained entries. Expiry or eviction permits a fresh instance to emit again. +\* ready is the service gate, not client opt-in. Client filtering and network success are outside the model. +\* Historical cost: about 6 minutes for the first spec, replay, tests, and fixes. +\* Result: ExceptionCaptureMC holds in 1,079 states. Joined, NoIdentity, NoWindow, and Unbounded each fail. +\* Cost: about 4 minutes across both new models, the code refactor, and validation. Each TLC run takes less than 2 seconds. +\* Null result: no new production bug. The shared refresh passes the property tests and eviction regression. +\* WindowDecision covers repeat identity, so the redundant identity history flag was removed. +\* BlockedUnchanged is an action property; it needs no history flag. The instance set bounds marked. +\* The copy model has its own states. Combining it with this cache would add unrelated state combinations. +\* Guidance: https://lamport.azurewebsites.net/tla/book-02-02-27.pdf (Chapter 7). +\* MC layout: https://github.com/tlaplus/vscode-tlaplus/blob/master/resources/knowledgebase/tlc-config-files.md +\* TLC checks the model, not code correctness. The prior trace-limited model visited 67 states. -EXTENDS Naturals, Sequences, FiniteSets -CONSTANTS Mode, Limit, Steps -VARIABLES seen, cache, step, checks -vars == <> -Ids == 1..3 -\* Two realistic field tuples have the same delimiter-joined text. -Fields(i) == CASE i = 1 -> <<"a|b", "c">> [] i = 2 -> <<"a", "b|c">> [] OTHER -> <<"a", "c">> -Key(i) == IF Mode = "Joined" /\ i \in {1, 2} THEN Fields(1) ELSE Fields(i) -Drop(c, k) == SelectSeq(c, LAMBDA e : e.key # k) -Init == seen = {} /\ cache = <<>> /\ step = 0 /\ checks = [identity |-> TRUE, window |-> TRUE, blocked |-> TRUE, caller |-> TRUE] -\* A gap of 2 is the expiry boundary; 3 represents a longer backoff gap. -Call(i, fresh, gap, ready, mutate) == - LET k == Key(i) - entries == {n \in 1..Len(cache) : cache[n].key = k} - recent == \E n \in entries : cache[n].age + gap < 2 - identity == ~fresh /\ i \in seen - emit == ready /\ (~identity \/ Mode = "NoIdentity") /\ (Mode = "NoWindow" \/ ~recent) - expected == ready /\ ~identity /\ ~recent - aged == [n \in 1..Len(cache) |-> [cache[n] EXCEPT !.age = IF @ + gap > 2 THEN 2 ELSE @ + gap]] - touched == Append(Drop(aged, k), [key |-> k, age |-> 0]) - nextCache == IF Len(touched) > Limit /\ Mode # "Unbounded" THEN Tail(touched) ELSE touched - writes == ready /\ ~identity - IN /\ step < Steps - /\ step' = step + 1 - /\ seen' = IF ready THEN seen \cup {i} ELSE seen - /\ cache' = IF writes THEN nextCache ELSE cache - /\ checks' = [identity |-> checks.identity /\ ~(emit /\ identity), - window |-> checks.window /\ (emit = expected), - blocked |-> checks.blocked /\ (~ready => seen' = seen /\ cache' = cache), - caller |-> checks.caller /\ ~(emit /\ mutate /\ Mode = "Mutating")] -Next == \E i \in Ids, fresh \in BOOLEAN, gap \in 0..3, ready \in BOOLEAN, mutate \in BOOLEAN : - Call(i, fresh, gap, ready, mutate) +EXTENDS Naturals, Sequences +CONSTANTS Instances, Fields, Fingerprint(_), Window, Limit, Gaps, + IdentityGuard, WindowGuard, SizeGuard +VARIABLES marked, cache, last +vars == <> + +NoCall == [kind |-> "tick", ready |-> FALSE, marked |-> FALSE, recent |-> FALSE, emitted |-> FALSE] +Drop(c, k) == SelectSeq(c, LAMBDA entry : entry.key # k) +Recent(c, k) == \E n \in 1..Len(c) : c[n].key = k /\ c[n].age < Window +Refresh(c, k) == Append(Drop(c, k), [key |-> k, age |-> 0]) +Trim(c) == IF SizeGuard /\ Len(c) > Limit THEN Tail(c) ELSE c + +Init == marked = {} /\ cache = <<>> /\ last = NoCall + +Call(i, ready) == + LET key == Fingerprint(Fields[i]) + known == i \in marked + recent == Recent(cache, key) + accepts == ready /\ (~IdentityGuard \/ ~known) + IN /\ marked' = IF ready THEN marked \cup {i} ELSE marked + /\ cache' = IF accepts THEN Trim(Refresh(cache, key)) ELSE cache + /\ last' = [kind |-> "call", ready |-> ready, marked |-> known, recent |-> recent, + emitted |-> accepts /\ (~WindowGuard \/ ~recent)] + +Tick(gap) == + /\ cache' = [n \in 1..Len(cache) |-> + [cache[n] EXCEPT !.age = IF @ + gap >= Window THEN Window ELSE @ + gap]] + /\ last' = NoCall + /\ UNCHANGED marked + +Next == (\E i \in Instances, ready \in BOOLEAN : Call(i, ready)) \/ (\E gap \in Gaps : Tick(gap)) Spec == Init /\ [][Next]_vars -TypeOK == seen \subseteq Ids /\ step \in 0..Steps /\ checks \in [ {"identity", "window", "blocked", "caller"} -> BOOLEAN ] -FieldsDistinct == step >= 0 => \A i, j \in Ids : Fields(i) # Fields(j) => Key(i) # Key(j) -\* checks records each decision and source preservation at each call. -WindowDecision == checks.window -IdentityOnce == checks.identity -BlockedUnchanged == checks.blocked -CallerMessage == checks.caller + +TypeOK == /\ marked \subseteq Instances + /\ cache \in Seq([key : {Fingerprint(Fields[i]) : i \in Instances}, age : 0..Window]) + /\ last \in [kind : {"call", "tick"}, ready : BOOLEAN, marked : BOOLEAN, + recent : BOOLEAN, emitted : BOOLEAN] +WindowDecision == last.kind = "call" => + (last.emitted = (last.ready /\ ~last.marked /\ ~last.recent)) +FieldsDistinct == \A i \in marked, j \in Instances : + Fields[i] # Fields[j] => Fingerprint(Fields[i]) # Fingerprint(Fields[j]) Bounded == Len(cache) <= Limit +\* A blocked call leaves both policy stores unchanged. Time advances are separate actions. +BlockedUnchanged == [][(last'.kind = "call" /\ ~last'.ready) => UNCHANGED <>]_vars ==== diff --git a/docs/architecture/tla/ExceptionCaptureMC.Joined.cfg b/docs/architecture/tla/ExceptionCaptureMC.Joined.cfg new file mode 100644 index 0000000000..1ad981fced --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.Joined.cfg @@ -0,0 +1,14 @@ +SPECIFICATION Spec +CONSTANTS + Instances <- MCInstances + Fields <- MCFields + Fingerprint <- JoinedKey + Window <- MCWindow + Limit <- MCLimit + Gaps <- MCGaps + IdentityGuard <- On + WindowGuard <- On + SizeGuard <- On +INVARIANTS TypeOK WindowDecision FieldsDistinct Bounded +PROPERTY BlockedUnchanged +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCaptureMC.NoIdentity.cfg b/docs/architecture/tla/ExceptionCaptureMC.NoIdentity.cfg new file mode 100644 index 0000000000..10645cef2d --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.NoIdentity.cfg @@ -0,0 +1,14 @@ +SPECIFICATION Spec +CONSTANTS + Instances <- MCInstances + Fields <- MCFields + Fingerprint <- TupleKey + Window <- MCWindow + Limit <- MCLimit + Gaps <- MCGaps + IdentityGuard <- Off + WindowGuard <- On + SizeGuard <- On +INVARIANTS TypeOK WindowDecision FieldsDistinct Bounded +PROPERTY BlockedUnchanged +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCaptureMC.NoWindow.cfg b/docs/architecture/tla/ExceptionCaptureMC.NoWindow.cfg new file mode 100644 index 0000000000..8600cd712b --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.NoWindow.cfg @@ -0,0 +1,14 @@ +SPECIFICATION Spec +CONSTANTS + Instances <- MCInstances + Fields <- MCFields + Fingerprint <- TupleKey + Window <- MCWindow + Limit <- MCLimit + Gaps <- MCGaps + IdentityGuard <- On + WindowGuard <- Off + SizeGuard <- On +INVARIANTS TypeOK WindowDecision FieldsDistinct Bounded +PROPERTY BlockedUnchanged +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCaptureMC.Unbounded.cfg b/docs/architecture/tla/ExceptionCaptureMC.Unbounded.cfg new file mode 100644 index 0000000000..18083f0281 --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.Unbounded.cfg @@ -0,0 +1,14 @@ +SPECIFICATION Spec +CONSTANTS + Instances <- MCInstances + Fields <- MCFields + Fingerprint <- TupleKey + Window <- MCWindow + Limit <- MCLimit + Gaps <- MCGaps + IdentityGuard <- On + WindowGuard <- On + SizeGuard <- Off +INVARIANTS TypeOK WindowDecision FieldsDistinct Bounded +PROPERTY BlockedUnchanged +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCaptureMC.cfg b/docs/architecture/tla/ExceptionCaptureMC.cfg new file mode 100644 index 0000000000..824dacf9a9 --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.cfg @@ -0,0 +1,14 @@ +SPECIFICATION Spec +CONSTANTS + Instances <- MCInstances + Fields <- MCFields + Fingerprint <- TupleKey + Window <- MCWindow + Limit <- MCLimit + Gaps <- MCGaps + IdentityGuard <- On + WindowGuard <- On + SizeGuard <- On +INVARIANTS TypeOK WindowDecision FieldsDistinct Bounded +PROPERTY BlockedUnchanged +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCaptureMC.tla b/docs/architecture/tla/ExceptionCaptureMC.tla new file mode 100644 index 0000000000..b53fff8aea --- /dev/null +++ b/docs/architecture/tla/ExceptionCaptureMC.tla @@ -0,0 +1,16 @@ +---- MODULE ExceptionCaptureMC ---- +\* Finite inputs and guard substitutions for ExceptionCapture. No trace counter is needed. +\* Two instances share fields. Two different tuples collide under delimiter joining. A third key forces eviction. +EXTENDS ExceptionCapture + +MCInstances == 1..4 +MCFields == [i \in MCInstances |-> + CASE i \in {1, 2} -> <<"a|b", "c">> [] i = 3 -> <<"a", "b|c">> [] OTHER -> <<"a", "c">>] +TupleKey(fields) == fields +JoinedKey(fields) == IF fields \in {MCFields[1], MCFields[3]} THEN MCFields[1] ELSE fields +MCWindow == 2 +MCLimit == 2 +MCGaps == {1, 2, 3} +On == TRUE +Off == FALSE +==== diff --git a/docs/architecture/tla/ExceptionCopy.tla b/docs/architecture/tla/ExceptionCopy.tla new file mode 100644 index 0000000000..512cbf8a6b --- /dev/null +++ b/docs/architecture/tla/ExceptionCopy.tla @@ -0,0 +1,47 @@ +---- MODULE ExceptionCopy ---- +\* Source: PostHogTelemetryClient.captureException(error). Sink: PostHog.captureException(payload). +\* Failures: normalization changes the caller error, loses error type or children, or sends the wrong message. +\* Rules: CallerMessage, SourceShape, and NormalizedMessage compare source and delivered records directly. +\* The transform is an oracle. ExceptionCopyMC supplies fixed and broken transforms for the finite check. +\* Expected before simplification: fixed records pass; mutation and prototype loss each fail. +\* Review: PR 2330 discussion_r4486068932 added CallerMessage to the original combined model. +\* Review: PR 2330 discussion_r4638627894 added SourceShape after the copy lost built-in prototypes. +\* Review counts for this module: 0 covered, 2 added, 0 gap. Dedupe feedback is in ExceptionCapture. +\* Replay: the real client changed three caller messages before the detached-copy fix. +\* Replay: TypeError, RangeError, and AggregateError each lost their prototype for three test messages. +\* Property tests: PostHogTelemetryClient.message-preservation.test.ts checks messages, types, stacks, causes, and children. +\* proto represents the inherited error type. Only an AggregateError exposes children to the sink. +\* Fields stand for data or references. This model does not copy JavaScript descriptors or PostHog parsing. +\* Historical cost: about 2 minutes for the prototype-loss spec, replay, fix, and checks. +\* Result: ExceptionCopyMC holds in 32 states. Mutating fails CallerMessage; PlainCopy fails SourceShape. +\* Cost: included in ExceptionCapture. Each TLC run takes less than 2 seconds. No new production bug was found. +\* Removed caller and shape success flags. The invariants now inspect the actual transform output. +\* Guidance: https://lamport.azurewebsites.net/tla/book-02-02-27.pdf (Chapter 7). +\* TLC checks the model, not code correctness. Real-code tests provide the source-to-sink replay. + +EXTENDS Naturals, Sequences +CONSTANTS Inputs, Messages, Transform(_, _) +VARIABLES source, raw, caller, delivered +vars == <> + +Children(error) == IF error.proto = "AggregateError" THEN error.own.children ELSE <<>> +Init == /\ source \in Inputs /\ raw \in Messages + /\ caller = source /\ delivered = <<>> +Capture == + LET output == Transform(source, raw) + IN /\ Len(delivered) = 0 + /\ caller' = output.caller /\ delivered' = <> + /\ UNCHANGED <> +Spec == Init /\ [][Capture]_vars + +TypeOK == /\ source \in Inputs /\ raw \in Messages /\ caller \in Inputs + /\ delivered \in Seq([proto : {error.proto : error \in Inputs}, own : {error.own : error \in Inputs}]) + /\ Len(delivered) <= 1 +CallerMessage == caller = source +SourceShape == Len(delivered) = 1 => + /\ delivered[1].proto = source.proto + /\ delivered[1].own.stack = source.own.stack + /\ delivered[1].own.cause = source.own.cause + /\ Children(delivered[1]) = Children(source) +NormalizedMessage == Len(delivered) = 1 => delivered[1].own.message = raw +==== diff --git a/docs/architecture/tla/ExceptionCopyMC.Mutating.cfg b/docs/architecture/tla/ExceptionCopyMC.Mutating.cfg new file mode 100644 index 0000000000..16f4458aef --- /dev/null +++ b/docs/architecture/tla/ExceptionCopyMC.Mutating.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec +CONSTANTS + Inputs <- MCInputs + Messages <- MCMessages + Transform <- Mutating +INVARIANTS TypeOK CallerMessage SourceShape NormalizedMessage +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCopyMC.PlainCopy.cfg b/docs/architecture/tla/ExceptionCopyMC.PlainCopy.cfg new file mode 100644 index 0000000000..df016f789c --- /dev/null +++ b/docs/architecture/tla/ExceptionCopyMC.PlainCopy.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec +CONSTANTS + Inputs <- MCInputs + Messages <- MCMessages + Transform <- PlainCopy +INVARIANTS TypeOK CallerMessage SourceShape NormalizedMessage +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCopyMC.cfg b/docs/architecture/tla/ExceptionCopyMC.cfg new file mode 100644 index 0000000000..f27873c2df --- /dev/null +++ b/docs/architecture/tla/ExceptionCopyMC.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec +CONSTANTS + Inputs <- MCInputs + Messages <- MCMessages + Transform <- Detached +INVARIANTS TypeOK CallerMessage SourceShape NormalizedMessage +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/ExceptionCopyMC.tla b/docs/architecture/tla/ExceptionCopyMC.tla new file mode 100644 index 0000000000..5d753c0fda --- /dev/null +++ b/docs/architecture/tla/ExceptionCopyMC.tla @@ -0,0 +1,15 @@ +---- MODULE ExceptionCopyMC ---- +\* Finite source records and copy transforms for ExceptionCopy. The two broken transforms are sabotage checks. +EXTENDS ExceptionCopy + +MCNames == {"Error", "TypeError", "RangeError", "AggregateError"} +MCMessages == {"wrapped", "raw"} +MCInputs == { [proto |-> kind, + own |-> [message |-> message, stack |-> "source stack", cause |-> "cause reference", + children |-> IF kind = "AggregateError" THEN <<"child reference">> ELSE <<>>]] : + kind \in MCNames, message \in MCMessages } +Normalized(error, message) == [error EXCEPT !.own.message = message] +Detached(error, message) == [caller |-> error, payload |-> Normalized(error, message)] +Mutating(error, message) == [caller |-> Normalized(error, message), payload |-> Normalized(error, message)] +PlainCopy(error, message) == [caller |-> error, payload |-> [Normalized(error, message) EXCEPT !.proto = "Error"]] +==== diff --git a/packages/telemetry/src/PostHogTelemetryClient.ts b/packages/telemetry/src/PostHogTelemetryClient.ts index e321f4c89e..6d2d7c866f 100644 --- a/packages/telemetry/src/PostHogTelemetryClient.ts +++ b/packages/telemetry/src/PostHogTelemetryClient.ts @@ -118,7 +118,9 @@ export class PostHogTelemetryClient extends BaseTelemetryClient { // Normalize a detached error so telemetry cannot change the caller-facing message. const stack = error.stack - const telemetryError = Object.defineProperties(new Error(), Object.getOwnPropertyDescriptors(error)) + const telemetryError = new Error() + Object.setPrototypeOf(telemetryError, Object.getPrototypeOf(error)) + Object.defineProperties(telemetryError, Object.getOwnPropertyDescriptors(error)) telemetryError.stack = stack telemetryError.message = errorMessage diff --git a/packages/telemetry/src/TelemetryService.ts b/packages/telemetry/src/TelemetryService.ts index fa402c6b2f..cb6fa6645a 100644 --- a/packages/telemetry/src/TelemetryService.ts +++ b/packages/telemetry/src/TelemetryService.ts @@ -244,14 +244,8 @@ export class TelemetryService { const now = Date.now() const lastSeenAt = this.recentExceptionFingerprints.get(fingerprint) - if (lastSeenAt !== undefined && now - lastSeenAt < EXCEPTION_DEDUPE_WINDOW_MS) { - // Re-dating an existing key must move it to the back so the oldest - // entry stays the correct eviction candidate. - this.recentExceptionFingerprints.delete(fingerprint) - this.recentExceptionFingerprints.set(fingerprint, now) - return - } - + // Every fresh instance refreshes recency, including a duplicate inside the window. + // Delete before set so Map iteration keeps the least recently observed key first. this.recentExceptionFingerprints.delete(fingerprint) this.recentExceptionFingerprints.set(fingerprint, now) @@ -265,6 +259,10 @@ export class TelemetryService { this.recentExceptionFingerprints.delete(oldest) } + if (lastSeenAt !== undefined && now - lastSeenAt < EXCEPTION_DEDUPE_WINDOW_MS) { + return + } + this.clients.forEach((client) => this.trackPendingClientCall(client.captureException(error, additionalProperties)), ) diff --git a/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts b/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts index ed1d53e338..bb56c5a357 100644 --- a/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts +++ b/packages/telemetry/src/__tests__/PostHogTelemetryClient.message-preservation.test.ts @@ -10,18 +10,23 @@ vi.mock("vscode", () => ({ workspace: { getConfiguration: vi.fn() }, })) +const captureException = vi.fn<(error: Error) => void>() + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(PostHog).mockImplementation(function () { + // The SDK double supplies only the methods used by this capture path. + return { captureException, optIn: vi.fn() } as unknown as PostHog + }) + vi.mocked(vscode.workspace.getConfiguration).mockReturnValue({ + get: () => "all", + // The configuration double supplies get for the telemetry preference. + } as unknown as vscode.WorkspaceConfiguration) +}) + it.each(["upstream failure", "provider|failure", "error\nwith details"])( "preserves the caller message while PostHog normalizes %s", async (raw) => { - const captureException = vi.fn() - vi.mocked(PostHog).mockImplementation(function () { - // The SDK double supplies only the methods used by this capture path. - return { captureException, optIn: vi.fn() } as unknown as PostHog - }) - vi.mocked(vscode.workspace.getConfiguration).mockReturnValue({ - get: () => "all", - // The configuration double supplies get for the telemetry preference. - } as unknown as vscode.WorkspaceConfiguration) const client = new PostHogTelemetryClient() client.updateTelemetryState(true) const service = new TelemetryService([client]) @@ -38,3 +43,50 @@ it.each(["upstream failure", "provider|failure", "error\nwith details"])( expect(captureException.mock.calls[0][0]).toMatchObject({ message: raw, stack: error.stack }) }, ) + +const errorKinds = [ + { name: "Error", create: (message: string) => new Error(message) }, + { name: "TypeError", create: (message: string) => new TypeError(message) }, + { name: "RangeError", create: (message: string) => new RangeError(message) }, + { + name: "AggregateError", + create: (message: string) => new AggregateError([new TypeError("child failure")], message), + }, +] + +// SourceShape: normalization preserves the source type, stack, cause, and aggregate children. +describe.each(errorKinds)("$name", ({ create }) => { + it.each(["upstream failure", "provider|failure", "error\nwith details"])( + "preserves the error shape while normalizing %s", + async (raw) => { + const client = new PostHogTelemetryClient() + client.updateTelemetryState(true) + const message = `Provider API Error 500: ${raw}` + const cause = new Error("original cause") + const error = Object.assign(create(message), { + cause, + status: 500, + error: { metadata: { raw } }, + }) + const stack = error.stack + + await client.captureException(error) + + expect(error.message).toBe(message) + expect(error.stack).toBe(stack) + expect(captureException).toHaveBeenCalledTimes(1) + const sent = captureException.mock.calls[0][0] + expect(sent).not.toBe(error) + expect(Object.getPrototypeOf(sent)).toBe(Object.getPrototypeOf(error)) + expect(sent.name).toBe(error.name) + expect(sent.message).toBe(raw) + expect(sent.stack).toBe(stack) + expect(sent.cause).toBe(cause) + if (error instanceof AggregateError) { + expect(sent).toBeInstanceOf(AggregateError) + if (!(sent instanceof AggregateError)) throw new Error("AggregateError prototype was lost") + expect(sent.errors).toBe(error.errors) + } + }, + ) +})