diff --git a/packages/harness-testing/src/exec/interactive.ts b/packages/harness-testing/src/exec/interactive.ts index aaae39e..ce42764 100644 --- a/packages/harness-testing/src/exec/interactive.ts +++ b/packages/harness-testing/src/exec/interactive.ts @@ -5,8 +5,8 @@ export {settleOn} from './pageReady.js'; // Unlike a case run, these are *streamed* — the UI watches the page while the model // thinks. They emit frames instead of returning one result, and screenshots leave as file // refs: the frames also land in lineage, and lineage must not fill up with base64 JPEGs. -import { mkdirSync, writeFileSync } from "node:fs"; -import { resolve } from "node:path"; +import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import { resolveText, redact, withModel, type ResolveContext } from "@testpilot/harness-core"; import { launchSession, type LaunchOpts, type Session } from "./session.js"; import { isInfraError } from "../failure.js"; @@ -379,6 +379,18 @@ export interface DebugSpec { launch: LaunchOpts; } +/** + * 探索半成品的落点。**两侧共用这一个函数**：探索器往这里写，服务端从这里捡 + * （`workflowOps.readPartialObservation`）。 + * + * 抽出来是因为这是个典型的会悄悄错开的接缝：两处各写一遍模板字符串， + * 哪天改了目录名，写的一侧和读的一侧不会有任何一层报错——只是永远捡不到东西， + * 而「捡不到」和「本来就没有」长得一模一样。 + */ +export function partialObservationPath(artifactDir: string, execId: string): string { + return resolve(artifactDir, "observe", `${execId}.partial.json`); +} + function shooter(spec: { execId: string; artifactDir: string }) { const dir = resolve(spec.artifactDir, "live"); mkdirSync(dir, { recursive: true }); @@ -1228,6 +1240,38 @@ export async function runObserve( } const screens: string[] = [first.text]; + /** + * **每采到一屏就把已有的材料落一次盘。** + * + * 2026-09-14 调研出来的：探索是九个节点里唯一一个「中途挂 = 全丢」的，而它同时是最长的 + * 一个——这次的材料 187,669 字，跑满 20 屏要几十分钟加一次钱包会话。整份结果只在函数 + * 返回时才组装、才交给服务端落账本，所以第 18 屏上崩掉，前 17 屏花掉的十几次模型调用 + * 全部作废，resume 从第 1 屏重来。（2026-09-12 还因为收尾时 runner 被心跳看门狗 SIGKILL， + * 一晚上丢过四次完整探索。） + * + * 只落**材料**，不落状态图：`graph` 要到循环跑完才建得出来，而材料就是钱花在的地方。 + * 写法是「先写临时文件再 rename」——崩在写一半上会留下半个 JSON，那比没有更糟。 + * + * 这不是断点续跑：探索不会从第 18 屏接着走（那要动状态机，是另一件事）。 + * 它只保证**已经花掉的钱不白花**，以及下游拿到的材料上写着它只到第几屏。 + */ + const partialPath = partialObservationPath(spec.artifactDir, spec.execId); + const snapshotPartial = (): void => { + try { + mkdirSync(dirname(partialPath), { recursive: true }); + const body = JSON.stringify({ + partial: true, + at: new Date().toISOString(), + url: spec.url, + screens: screens.length, + notes: budgeted(screens, "", `===== 这次探索停在第 ${screens.length} 屏 =====`), + }); + writeFileSync(`${partialPath}.tmp`, body); + renameSync(`${partialPath}.tmp`, partialPath); + } catch { /* 落盘失败不能带垮探索本身——它是附加物，不是判决 */ } + }; + snapshotPartial(); + const seen = new Set([signatureOf(first)]); const visited: string[] = [first.url]; const missed: string[] = []; @@ -2329,6 +2373,7 @@ export async function runObserve( */ if (next.kind === "probe") { screens.push(`（实验：${PROBE_WORDS[next.variant]} ${next.label}）\n${after.text}`); + snapshotPartial(); note(`实验结果记入材料（状态未变，但页面文字变了）`); } /** @@ -2341,6 +2386,7 @@ export async function runObserve( */ if (next.kind === "click" && next.group && effect.changed) { screens.push(describeEffect(next.label, next.group, effect)); + snapshotPartial(); note(`页内切换有效果，记入材料（签名未变）`); } // 代表没走出去 → 它的结构同类一并跳过。这一条直接把「12 张商品卡片吃掉 @@ -2367,6 +2413,7 @@ export async function runObserve( triedGoto.add(pathOf(after.url)); visited.push(after.url); screens.push(after.text); + snapshotPartial(); dry = 0; consecutiveFailures = 0; note(`第 ${screens.length} 屏：${after.url}，${after.controls.length} 个控件`); @@ -2522,6 +2569,9 @@ export async function runObserve( const shotRef = await shot(session); note(`收尾用时：回执摘要 ${t1 - t0}ms / 材料 ${t2 - t1}ms / 截图 ${Date.now() - t2}ms`); + // 跑完了就把半成品删掉：留着它，下一次失败会捡到上一次的材料，而那比没有更糟。 + try { rmSync(partialPath, { force: true }); } catch { /* 删不掉就算了，里面带着时间戳 */ } + return { // 图的摘要跟着材料一起走：下游整理规格时**先看结构再看正文**—— // 实证研究的结论是「精简的功能级上下文」对 LLM 最有效，原始屏幕转储不是。 diff --git a/server/src/workflowOps.ts b/server/src/workflowOps.ts index 2d83fd7..d87cc5e 100644 --- a/server/src/workflowOps.ts +++ b/server/src/workflowOps.ts @@ -1,8 +1,10 @@ import { captureWebModels } from './modelSnapshots.js'; import { observeProduct } from './procs.js'; +import { partialObservationPath } from "@testpilot/harness-testing/exec"; +import { ARTIFACT_DIR } from "./db.js"; import { controls, beginStage, stageEvent, resumeControls } from './workflowControls.js'; import { cancelRun as cancelCodex } from "./codex.js"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, basename } from "node:path"; import { randomUUID } from "node:crypto"; import { z } from "zod"; @@ -116,6 +118,26 @@ export function sourceKnowledge(runId:string,projectId:string,entryUrl:string,ma truncation:{omittedOptionalRefs:packs.slice(1).map(p=>p.revision),missingRequiredRefs:[]},isolationEvidence:'service-scoped'}); return {charter,pack,packRevision:first?.revision,manifest}; } +/** + * 捡起探索器落下的半成品（`exec/interactive.ts` 的 `snapshotPartial`）。 + * + * 路径按约定拼：观察的 execId 是 `observe-`（见 index.ts 的 setAgentObserver）， + * 一个项目同时只探索一份，所以这个名字够用。读不到、读坏了都当作没有—— + * 捡不回来是可以接受的，捡回来一份半个 JSON 不行。 + */ +export function readPartialObservation(projectId:string):{notes:string;url:string;screens:number;graph:unknown;stoppedBecause:unknown}|undefined{ + try{ + // 落点由探索器那一侧的函数算，两处共用——见 `partialObservationPath` 的注释。 + const path=partialObservationPath(ARTIFACT_DIR,`observe-${projectId}`); + const raw=JSON.parse(readFileSync(path,'utf8')) as {notes?:string;url?:string;screens?:number;graph?:unknown}; + if(!raw?.notes?.trim())return undefined; + // 半成品里没有状态图（`graph` 要循环跑完才建得出来）。给 undefined 而不是编一个空图： + // 下游读到「没有图」是真的没有，读到一个空图会以为这个产品只有一屏。 + return {notes:raw.notes,url:raw.url??'',screens:raw.screens??0,graph:undefined, + stoppedBecause:`探索中途失败，这份材料只到第 ${raw.screens ?? 0} 屏`}; + }catch{return undefined;} +} + async function launchSource(runId:string,projectId:string,directory:string,params:{sourceKind:string;sourceUrl?:string;limit:number;stageControlVersion:number;outputLanguage?:string;maxScreens?:number;envRef?:string;exploreActions?:string;exploreWallet?:boolean},envRef?:string){ const ledger=runLedger(); try { @@ -124,7 +146,29 @@ async function launchSource(runId:string,projectId:string,directory:string,param const interact=params.exploreActions==='interact'; const bound=sourceKnowledge(runId,projectId,params.sourceUrl!,params.maxScreens??8,envRef??params.envRef,interact); const manifestRevision=ledger.putRevision({runId,projectId,name:'context/source',kind:'report',content:bound.manifest,sourceRefs:bound.manifest.knowledge.map(k=>k.revision)},{kind:'system',id:'stage-validator'}); - const result=await observeProduct({url:params.sourceUrl,projectId,envRef:envRef??params.envRef,deep:true,maxScreens:params.maxScreens??8,settleMs:interact?3000:1800,scenarioFirst:true,inPageFirst:'on',groupCap:6,...(params.exploreWallet?{wallet:true}:{}),...(bound.charter?{charter:bound.charter}:{})}) as {notes:string;url:string;screens:unknown;stoppedBecause:unknown;graph:unknown;report?:unknown}; + /** + * **探索崩了，也要把已经采到的屏捡回来。** + * + * 2026-09-14 调研（docs/v3 的三项顾虑）：九个节点里只有探索是「中途挂 = 全丢」。 + * 它同时是最长的一个——这次的材料是 187,669 字，跑满 20 屏要几十分钟加一次钱包会话。 + * 探索器现在每采到一屏就落一次半成品（`exec/interactive.ts` 的 `snapshotPartial`）， + * 这里在失败路径上把它捡起来：有材料就带着已采到的屏继续走，没有才如实抛。 + * + * 这**不是**断点续跑：不会从第 18 屏接着探。它保证的是已经花掉的钱不白白作废， + * 而且这件事要在材料里写明白——下游读到的是一份 18 屏的材料，不是 20 屏的。 + */ + let result:{notes:string;url:string;screens:unknown;stoppedBecause:unknown;graph:unknown;report?:unknown;partial?:boolean}; + let partialReason:string|undefined; + try { + result=await observeProduct({url:params.sourceUrl,projectId,envRef:envRef??params.envRef,deep:true,maxScreens:params.maxScreens??8,settleMs:interact?3000:1800,scenarioFirst:true,inPageFirst:'on',groupCap:6,...(params.exploreWallet?{wallet:true}:{}),...(bound.charter?{charter:bound.charter}:{})}) as typeof result; + } catch(error) { + const salvaged=readPartialObservation(projectId); + if(!salvaged?.notes?.trim())throw error; + partialReason=String((error as Error).message??error).slice(0,300); + result={...salvaged,partial:true}; + ledger.putRevision({runId,projectId,name:'report/exploration-partial',kind:'report', + content:{screens:salvaged.screens,reason:partialReason,at:new Date().toISOString()},sourceRefs:[manifestRevision.id]},{kind:'system',id:'explorer'}); + } if(ledger.getRun(runId,projectId).status==='cancelled')return; if(!result.notes?.trim())throw new Error('exploration_returned_no_observations'); const observation=ledger.putRevision({runId,projectId,name:'exploration/observations',kind:'report',content:result,sourceRefs:[manifestRevision.id]},{kind:'system',id:'explorer'}); @@ -159,8 +203,26 @@ export function workflowCheckpoint(runId: string, projectId: string) { const verified = registeredStageProducts(runId); const finalized = verified.protected && verified.finalized; const states = ledger.nodeStates(runId); - const stages = ["instructions", "stories", "cases", "gate", "finalize"]; - const next = finalized ? "review" : stages.find(stage => !states.some(s => s.node === stage && s.phase === "done")) ?? "finalize"; + /** + * **`source` 与 `modules` 也要在这张清单里。** + * + * 2026-09-14 调研发现它们不在：一次在 `modules` 上失败的运行，`next` 会指向 + * `stories`——resume 于是把模块节点整个跳过去，而下游所有单元都按模块树切。 + * 清单要和 `workflowControls` 的 `nodes` 对齐（少了 g2/execution：那两个不由 + * resume 驱动，各自有自己的入口和幂等键）。 + * + * 但这两个是**有条件的**：宿主注册的运行（`registerHostRun`）材料在注册时就交了， + * 根本没有 `source` 节点，也不走模块规划。所以判据不能是「没 done 就回到它」—— + * 那会让每一个宿主运行 resume 到一个它从来没有过的节点上（测试当场红了，对的）。 + * 规则是：**走过、而且没走完**，才回到它；从没走过就不属于这条路径。 + * `instructions` 往后是必经的，仍然按「没 done 就回到它」。 + */ + const conditional = new Set(["source", "modules"]); + const stages = ["source", "modules", "instructions", "stories", "cases", "gate", "finalize"]; + const done = (stage: string) => states.some(s => s.node === stage && s.phase === "done"); + const touched = (stage: string) => states.some(s => s.node === stage); + const next = finalized ? "review" + : stages.find(stage => (conditional.has(stage) ? touched(stage) && !done(stage) : !done(stage))) ?? "finalize"; return { runId, inputHash: run.binding.inputHash, next, finalized, stages: states, materialRevisions: run.binding.materialRevisions, source: run.binding.models.entry, runtime: run.binding.models.runtime }; } diff --git a/server/test/exploration-partial.test.ts b/server/test/exploration-partial.test.ts new file mode 100644 index 0000000..5a63ee7 --- /dev/null +++ b/server/test/exploration-partial.test.ts @@ -0,0 +1,92 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** + * 探索是九个节点里唯一一个「中途挂 = 全丢」的，而它同时是最长的一个 + * （一次 187,669 字的材料要几十分钟加一次钱包会话）。探索器现在每采一屏落一次半成品， + * 服务端在失败路径上把它捡回来——已经花掉的钱不白花。 + * + * 这不是断点续跑：不会从第 18 屏接着探。测的是「捡得回来」和「捡不回来时如实抛」。 + */ +describe("探索中途失败时捡回已采到的屏", () => { + let dir: string; + let ops: typeof import("../src/workflowOps.js"); + let db: typeof import("../src/db.js"); + + beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), "tp-partial-")); + vi.stubEnv("TP_DATA_DIR", dir); + db = await import("../src/db.js"); + ops = await import("../src/workflowOps.js"); + }); + afterEach(() => { vi.unstubAllEnvs(); rmSync(dir, { recursive: true, force: true }); }); + + // 写的时候用**探索器那一侧的函数**算落点：两侧共用同一个函数，这个接缝就错不开。 + const writePartial = async (projectId: string, body: unknown) => { + const { partialObservationPath } = await import("@testpilot/harness-testing/exec"); + const path = partialObservationPath(db.ARTIFACT_DIR, `observe-${projectId}`); + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, JSON.stringify(body)); + }; + + it("有材料就捡回来，并说清楚它只到第几屏", async () => { + await writePartial("p1", { partial: true, url: "https://x.test/", screens: 17, notes: "===== 第 1 屏 =====\n看到了东西" }); + const got = ops.readPartialObservation("p1"); + expect(got).toMatchObject({ screens: 17, url: "https://x.test/" }); + expect(got!.notes).toContain("看到了东西"); + // 停止原因要说人话：下游读到的是一份 17 屏的材料，不是 20 屏的。 + expect(String(got!.stoppedBecause)).toContain("17"); + // 半成品里没有状态图——给 undefined 而不是编一个空图，否则下游以为这产品只有一屏。 + expect(got!.graph).toBeUndefined(); + }); + + it("材料是空的就当作没有——宁可如实抛出原来的错", async () => { + await writePartial("p2", { partial: true, screens: 3, notes: " " }); + expect(ops.readPartialObservation("p2")).toBeUndefined(); + }); + + it("文件不存在、或者是半个 JSON，都当作没有", async () => { + expect(ops.readPartialObservation("never-explored")).toBeUndefined(); + const { partialObservationPath } = await import("@testpilot/harness-testing/exec"); + const path = partialObservationPath(db.ARTIFACT_DIR, "observe-p3"); + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, '{"notes":"半个'); + expect(ops.readPartialObservation("p3")).toBeUndefined(); + }); +}); + +/** + * `workflowCheckpoint` 的节点清单原来漏了 `source` 与 `modules`：一次在 modules 上失败的 + * 运行，`next` 会指向 `stories`——resume 把模块节点整个跳过去，而下游所有单元都按模块树切。 + * + * 但这两个是有条件的：宿主注册的运行材料在注册时就交了，根本没有 source 节点。 + * 所以判据是「走过、而且没走完」，不是「没 done 就回到它」——第一版写成后者， + * `run-routes` 的那条测试当场红了，对的。 + */ +describe("checkpoint 的下一个节点", () => { + const pick = (states: Array<{ node: string; phase: string }>) => { + const conditional = new Set(["source", "modules"]); + const stages = ["source", "modules", "instructions", "stories", "cases", "gate", "finalize"]; + const done = (s: string) => states.some((x) => x.node === s && x.phase === "done"); + const touched = (s: string) => states.some((x) => x.node === s); + return stages.find((s) => (conditional.has(s) ? touched(s) && !done(s) : !done(s))) ?? "finalize"; + }; + + it("走过 modules 但没走完 → 回到 modules", () => { + expect(pick([{ node: "source", phase: "done" }, { node: "modules", phase: "failed" }])).toBe("modules"); + }); + + it("从没走过 source 的宿主运行 → 不会被送回一个它没有的节点", () => { + expect(pick([{ node: "instructions", phase: "done" }, { node: "stories", phase: "done" }])).toBe("cases"); + }); + + it("探索没跑完 → 回到 source", () => { + expect(pick([{ node: "source", phase: "running" }])).toBe("source"); + }); + + it("必经节点没走过就是没走完", () => { + expect(pick([{ node: "source", phase: "done" }, { node: "modules", phase: "done" }])).toBe("instructions"); + }); +});