Skip to content

Commit 0540104

Browse files
authored
feat(telemetry): aggregate task completion telemetry with delta installments (#1071)
* feat(telemetry): aggregate task completion telemetry with delta installments * fix(telemetry): prevent idle timer from breaking tests with fake timers * fix(telemetry): guard user message decrement and fix idle re-fire logic * fix(telemetry): cover all task launch paths for idle timer; fix pop guard * fix(telemetry): add startIdleTelemetryCheck stub to task-run-dispatch spec * fix(telemetry): advance baseline before emit to prevent double-reporting on throw * fix(telemetry): address PR 1071 review feedback
1 parent cbdce3d commit 0540104

14 files changed

Lines changed: 932 additions & 102 deletions

‎packages/telemetry/src/TelemetryService.ts‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
type TelemetryPropertiesProvider,
66
TelemetryEventName,
77
type TelemetrySetting,
8+
type ToolUsage,
89
} from "@roo-code/types"
910

1011
/**
@@ -182,8 +183,31 @@ export class TelemetryService {
182183
this.captureEvent(TelemetryEventName.TASK_RESTARTED, { taskId })
183184
}
184185

185-
public captureTaskCompleted(taskId: string): void {
186-
this.captureEvent(TelemetryEventName.TASK_COMPLETED, { taskId })
186+
/**
187+
* Captures task completion, summarizing per-task tool and message counts that
188+
* were previously reported as separate per-turn events to reduce event volume.
189+
*
190+
* A single task may emit this more than once (e.g. an "idle" or "shutdown"
191+
* installment followed by a final "attempt_completion" one). toolsUsed and
192+
* messageCount are always deltas since the previous emission for that task,
193+
* not running totals -- summing installments for a taskId reconstructs the
194+
* full-task counts without double-counting.
195+
*
196+
* Note: "attempt_completion" means the model called that tool, not that the
197+
* user accepted the result.
198+
*/
199+
public captureTaskCompleted(
200+
taskId: string,
201+
toolsUsed?: ToolUsage,
202+
messageCount?: { user: number; assistant: number },
203+
completionReason: "attempt_completion" | "idle" | "shutdown" = "attempt_completion",
204+
): void {
205+
this.captureEvent(TelemetryEventName.TASK_COMPLETED, {
206+
taskId,
207+
completionReason,
208+
...(toolsUsed !== undefined && { toolsUsed }),
209+
...(messageCount !== undefined && { messageCount }),
210+
})
187211
}
188212

189213
public captureConversationMessage(taskId: string, source: "user" | "assistant"): void {
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// pnpm --filter @roo-code/telemetry test src/__tests__/TelemetryService.task-completed.test.ts
2+
3+
import { TelemetryEventName, type TelemetryClient } from "@roo-code/types"
4+
5+
import { TelemetryService } from "../TelemetryService"
6+
7+
describe("TelemetryService.captureTaskCompleted", () => {
8+
let mockClient: TelemetryClient
9+
10+
beforeEach(() => {
11+
mockClient = {
12+
setProvider: vi.fn(),
13+
capture: vi.fn().mockResolvedValue(undefined),
14+
captureException: vi.fn().mockResolvedValue(undefined),
15+
updateTelemetryState: vi.fn(),
16+
isTelemetryEnabled: vi.fn().mockReturnValue(true),
17+
shutdown: vi.fn().mockResolvedValue(undefined),
18+
}
19+
})
20+
21+
it("captures Task Completed with the taskId and a default 'attempt_completion' completionReason when no summary is provided", () => {
22+
const service = new TelemetryService([mockClient])
23+
24+
service.captureTaskCompleted("task_1")
25+
26+
expect(mockClient.capture).toHaveBeenCalledWith({
27+
event: TelemetryEventName.TASK_COMPLETED,
28+
properties: { taskId: "task_1", completionReason: "attempt_completion" },
29+
})
30+
})
31+
32+
it("includes toolsUsed and messageCount summaries when provided", () => {
33+
const service = new TelemetryService([mockClient])
34+
35+
service.captureTaskCompleted(
36+
"task_1",
37+
{ read_file: { attempts: 3, failures: 0 }, apply_diff: { attempts: 1, failures: 1 } },
38+
{ user: 4, assistant: 5 },
39+
)
40+
41+
expect(mockClient.capture).toHaveBeenCalledWith({
42+
event: TelemetryEventName.TASK_COMPLETED,
43+
properties: {
44+
taskId: "task_1",
45+
completionReason: "attempt_completion",
46+
toolsUsed: { read_file: { attempts: 3, failures: 0 }, apply_diff: { attempts: 1, failures: 1 } },
47+
messageCount: { user: 4, assistant: 5 },
48+
},
49+
})
50+
})
51+
52+
it("includes the given completionReason for idle/shutdown installments", () => {
53+
const service = new TelemetryService([mockClient])
54+
55+
service.captureTaskCompleted("task_1", { read_file: { attempts: 1, failures: 0 } }, undefined, "idle")
56+
57+
expect(mockClient.capture).toHaveBeenCalledWith({
58+
event: TelemetryEventName.TASK_COMPLETED,
59+
properties: {
60+
taskId: "task_1",
61+
completionReason: "idle",
62+
toolsUsed: { read_file: { attempts: 1, failures: 0 } },
63+
},
64+
})
65+
})
66+
})

‎src/__tests__/history-resume-delegation.spec.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ vi.mock("vscode", () => {
2626
return { window, workspace, env, Uri, commands, ExtensionMode, version }
2727
})
2828

29-
// Mock TelemetryService (needed by attemptCompletionTool's emitTaskCompleted)
29+
// Mock TelemetryService (needed by attemptCompletionTool's emitPublicTaskCompleted)
3030
vi.mock("@roo-code/telemetry", () => ({
3131
TelemetryService: {
3232
instance: {
@@ -1255,6 +1255,7 @@ describe("History resume delegation - parent metadata transitions", () => {
12551255
userMessageContent: [],
12561256
consecutiveMistakeCount: 0,
12571257
emitFinalTokenUsageUpdate: vi.fn(),
1258+
flushTelemetryInstallment: vi.fn(),
12581259
} as unknown as import("../core/task/Task").Task
12591260

12601261
const block = {

‎src/__tests__/nested-delegation-resume.spec.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ describe("Nested delegation resume (A → B → C)", () => {
203203
userMessageContent: [],
204204
consecutiveMistakeCount: 0,
205205
emitFinalTokenUsageUpdate: vi.fn(),
206+
flushTelemetryInstallment: vi.fn(),
206207
} as unknown as Task
207208

208209
const blockC = {
@@ -250,6 +251,7 @@ describe("Nested delegation resume (A → B → C)", () => {
250251
userMessageContent: [],
251252
consecutiveMistakeCount: 0,
252253
emitFinalTokenUsageUpdate: vi.fn(),
254+
flushTelemetryInstallment: vi.fn(),
253255
} as unknown as Task
254256

255257
const blockB = {

‎src/__tests__/task-run-dispatch.spec.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ type Runnable = {
1717
metadata: { task?: string | null; images?: string[] | null }
1818
resumeTaskFromHistory: () => Promise<void>
1919
startTask: (task?: string, images?: string[]) => Promise<void>
20+
startIdleTelemetryCheck: () => void
2021
}
2122

2223
function makeRunnable(overrides: Partial<Runnable> = {}): Runnable & { run(): Promise<void> } {
@@ -27,6 +28,7 @@ function makeRunnable(overrides: Partial<Runnable> = {}): Runnable & { run(): Pr
2728
metadata: { task: undefined, images: undefined },
2829
resumeTaskFromHistory: vi.fn().mockResolvedValue(undefined),
2930
startTask: vi.fn().mockResolvedValue(undefined),
31+
startIdleTelemetryCheck: vi.fn(),
3032
...overrides,
3133
}
3234
// Bind the real run() implementation from Task.prototype to our stand-in.

‎src/core/assistant-message/__tests__/presentAssistantMessage-custom-tool.spec.ts‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,6 @@ vi.mock("@roo-code/core", () => ({
2323
},
2424
}))
2525

26-
vi.mock("@roo-code/telemetry", () => ({
27-
TelemetryService: {
28-
instance: {
29-
captureToolUsage: vi.fn(),
30-
captureConsecutiveMistakeError: vi.fn(),
31-
},
32-
},
33-
}))
34-
35-
import { TelemetryService } from "@roo-code/telemetry"
3626
import { customToolRegistry } from "@roo-code/core"
3727

3828
describe("presentAssistantMessage - Custom Tool Recording", () => {
@@ -118,7 +108,6 @@ describe("presentAssistantMessage - Custom Tool Recording", () => {
118108

119109
// Should record as "custom_tool", not "my_custom_tool"
120110
expect(mockTask.recordToolUsage).toHaveBeenCalledWith("custom_tool")
121-
expect(TelemetryService.instance.captureToolUsage).toHaveBeenCalledWith(mockTask.taskId, "custom_tool")
122111
})
123112
})
124113

@@ -171,7 +160,6 @@ describe("presentAssistantMessage - Custom Tool Recording", () => {
171160

172161
// Should record as "read_file", not "custom_tool"
173162
expect(mockTask.recordToolUsage).toHaveBeenCalledWith("read_file")
174-
expect(TelemetryService.instance.captureToolUsage).toHaveBeenCalledWith(mockTask.taskId, "read_file")
175163
})
176164

177165
it("should record MCP tool usage as 'use_mcp_tool' (not custom_tool)", async () => {
@@ -213,7 +201,6 @@ describe("presentAssistantMessage - Custom Tool Recording", () => {
213201

214202
// Should record as "use_mcp_tool", not "custom_tool"
215203
expect(mockTask.recordToolUsage).toHaveBeenCalledWith("use_mcp_tool")
216-
expect(TelemetryService.instance.captureToolUsage).toHaveBeenCalledWith(mockTask.taskId, "use_mcp_tool")
217204
})
218205
})
219206

@@ -355,7 +342,6 @@ describe("presentAssistantMessage - Custom Tool Recording", () => {
355342

356343
// Should not record usage for partial blocks
357344
expect(mockTask.recordToolUsage).not.toHaveBeenCalled()
358-
expect(TelemetryService.instance.captureToolUsage).not.toHaveBeenCalled()
359345
})
360346
})
361347
})

‎src/core/assistant-message/presentAssistantMessage.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,6 @@ export async function presentAssistantMessage(cline: Task) {
235235

236236
if (!mcpBlock.partial) {
237237
cline.recordToolUsage("use_mcp_tool") // Record as use_mcp_tool for analytics
238-
TelemetryService.instance.captureToolUsage(cline.taskId, "use_mcp_tool")
239238
}
240239

241240
// Resolve sanitized server name back to original server name
@@ -558,7 +557,6 @@ export async function presentAssistantMessage(cline: Task) {
558557
const isCustomTool = stateExperiments?.customTools && customToolRegistry.has(block.name)
559558
const recordName = isCustomTool ? "custom_tool" : block.name
560559
cline.recordToolUsage(recordName)
561-
TelemetryService.instance.captureToolUsage(cline.taskId, recordName)
562560

563561
// Track legacy format usage for read_file tool (for migration monitoring)
564562
if (block.name === "read_file" && block.usedLegacyFormat) {

0 commit comments

Comments
 (0)