|
| 1 | +import { readFileSync } from 'node:fs'; |
| 2 | +import { describe, expect, it } from 'vitest'; |
| 3 | +import { prepareExecution, type ExecutionInput } from '../core/execution-prompt.ts'; |
| 4 | +import type { Plan } from '../core/plan.ts'; |
| 5 | + |
| 6 | +const identity = { repositoryId: 'repo', taskId: 'task', planId: 'plan' }; |
| 7 | +const plan = (intent = 'Retry with backoff'): Plan => ({ schema_version: 1, issue: 7, revision: 3, summary: 'Retries', questions: [], items: [ |
| 8 | + { id: 'P1', title: 'Backoff', intent, files: [{ path: 'retry.ts', kind: 'edit', renamed_from: null, change: 'Add delay()' }], |
| 9 | + acceptance: [{ type: 'cmd', text: 'npm test' }, { type: 'cmd', text: 'npm run lint -- --fix' }, { type: 'cmd', text: "npm 'unclosed" }, { type: 'check', text: 'Delays grow' }], depends_on: [] }, |
| 10 | + { id: 'P2', title: 'Docs', intent: 'Document it', files: [{ path: 'README.md', kind: 'edit', renamed_from: null, change: 'Explain' }], acceptance: [], depends_on: ['P1'] }, |
| 11 | +] }); |
| 12 | +const base = (over: Partial<ExecutionInput> = {}): ExecutionInput => ({ |
| 13 | + identity, attemptId: 'attempt-1', mode: 'execute', plan: plan(), itemId: 'P1', |
| 14 | + issue: { number: 7, title: 'Retries fail', body: 'Crash on retry.', comments: [] }, |
| 15 | + approvedLessons: [], allowedCommands: [['npm', 'test']], ...over, |
| 16 | +}); |
| 17 | +const TAGS = ['plan_item_data', 'issue_data', 'lessons_data', 'problem_data']; |
| 18 | +/** Every data block opens and closes exactly once, so nothing inside can close it early. */ |
| 19 | +function assertDelimitersIntact(prompt: string, withProblem: boolean) { |
| 20 | + for (const tag of TAGS) { |
| 21 | + const expected = tag === 'problem_data' && !withProblem ? 0 : 1; |
| 22 | + expect(prompt.split(`<${tag}>`).length - 1, `<${tag}>`).toBe(expected); |
| 23 | + expect(prompt.split(`</${tag}>`).length - 1, `</${tag}>`).toBe(expected); |
| 24 | + } |
| 25 | +} |
| 26 | + |
| 27 | +describe('execution prompt', () => { |
| 28 | + it('derives approved commands only from the item’s structured, approved cmd checks', () => { |
| 29 | + const request = prepareExecution(base()); |
| 30 | + expect(request.approvedArgv).toEqual([['npm', 'test']]); |
| 31 | + expect(request).toMatchObject({ mode: 'execute', phase: 'execute', access: 'write', item: 'P1', revision: 3 }); |
| 32 | + assertDelimitersIntact(request.prompt, false); |
| 33 | + expect(request.prompt).not.toContain('{{'); |
| 34 | + }); |
| 35 | + it('requires a problem in fix mode only, and the plan’s own issue', () => { |
| 36 | + expect(() => prepareExecution(base({ mode: 'fix' }))).toThrow(/exactly one problem/); |
| 37 | + expect(() => prepareExecution(base({ problem: { source: 'check', text: 'x' } }))).toThrow(/exactly one problem/); |
| 38 | + expect(() => prepareExecution(base({ issue: { number: 8, title: '', body: '', comments: [] } }))).toThrow(/issue mismatch/); |
| 39 | + expect(() => prepareExecution(base({ itemId: 'P9' }))).toThrow(/Unknown plan item/); |
| 40 | + const fix = prepareExecution(base({ mode: 'fix', problem: { source: 'review', text: 'Off by one' } })); |
| 41 | + assertDelimitersIntact(fix.prompt, true); |
| 42 | + }); |
| 43 | + it('does not let later edits to the caller’s input change a prepared request', () => { |
| 44 | + const input = base(), request = prepareExecution(input); |
| 45 | + (input.allowedCommands as string[][]).push(['npm', 'run', 'lint', '--', '--fix']); |
| 46 | + input.plan.items[0]!.intent = 'changed'; |
| 47 | + expect(request.approvedArgv).toEqual([['npm', 'test']]); |
| 48 | + expect(Object.isFrozen(request.approvedArgv[0])).toBe(true); |
| 49 | + }); |
| 50 | +}); |
| 51 | + |
| 52 | +describe('hostile-issue eval set', () => { |
| 53 | + const fixtures = JSON.parse(readFileSync(new URL('./fixtures/hostile-issues.json', import.meta.url), 'utf8')) as |
| 54 | + { name: string; body: string; comments?: string[]; problem?: string }[]; |
| 55 | + for (const fixture of fixtures) { |
| 56 | + it(`keeps "${fixture.name}" inside its data block and out of the approved commands`, () => { |
| 57 | + const problem = fixture.problem ? { source: 'review' as const, text: fixture.problem } : undefined; |
| 58 | + const request = prepareExecution(base({ |
| 59 | + mode: problem ? 'fix' : 'execute', problem, plan: plan(fixture.body), |
| 60 | + issue: { number: 7, title: fixture.name, body: fixture.body, comments: fixture.comments ?? [] }, |
| 61 | + approvedLessons: [fixture.body], |
| 62 | + })); |
| 63 | + assertDelimitersIntact(request.prompt, !!problem); |
| 64 | + expect(request.approvedArgv).toEqual([['npm', 'test']]); |
| 65 | + // Planted placeholders stay literal text: the template is filled in one pass and data is never re-read. |
| 66 | + // The body is embedded three times (plan item intent, issue body, lessons); each copy must stay literal. |
| 67 | + if (fixture.body.includes('{{')) { |
| 68 | + expect(request.prompt.split('{{problem_data_json}}').length - 1).toBe(3); |
| 69 | + expect(request.prompt).not.toContain('## The problem to fix'); |
| 70 | + } |
| 71 | + }); |
| 72 | + } |
| 73 | +}); |
0 commit comments