diff --git a/examples/adversarial-implementation/README.md b/examples/adversarial-implementation/README.md index eaf55140d..fb54fba8e 100644 --- a/examples/adversarial-implementation/README.md +++ b/examples/adversarial-implementation/README.md @@ -248,8 +248,8 @@ xmd run --allow-absolute /absolute/path/to/workflow/examples/adversarial-impleme --include /absolute/path/to/workflow/examples/adversarial-implementation/components --raw ``` -`status` starts no model. It reads the saved joined review, the current PR and -GitHub's CI workflow runs. Its `nextAction` is computed from the same facts as +`status` starts no model. It reads the saved joined review, its published comment, +the current PR and GitHub's CI workflow runs. Its `nextAction` is computed from the same facts as its `status` and `remaining` fields. When it requests `review`, select `--props-action review` on that command to perform the independent review with the provider, model, effort and session recorded at launch. The review uses @@ -295,10 +295,11 @@ feedback ready -> independent review pending -> joined exact-commit verdict | Fable unavailable/refused -> one Astra review (review) | no reviewer completed -> review_held: answer via review | incomplete/uncertain -> reconcile original call - | blocking finding -> request approved repair - | pass + required CI + main -> finish PR metadata - | all passed + PR ready -> await authorized merge - | confirmed merge -> delivered + | comment pending -> publish-review + | published changes required -> request approved repair + | published pass + checks -> finish PR metadata + | all passed + PR ready -> await authorized merge + | confirmed merge -> delivered ``` Independent review summaries allow 1500 characters; detailed observations belong @@ -340,6 +341,28 @@ and saves a raw reply the execution joined but never wrote. It never shortens te a model call or overwrites a contradictory receipt. Missing or uncertain execution continues to hold. Status uses this same validation boundary to read receipts. +`review` and `reconcile` publish each validated verdict as a comment on the +current pull request, including findings that require repair. The comment +names the exact reviewed commit and the reviewer who supplied the result. +Delivery uses `` to reconcile and verify the comment. With +the native GitHub provider, include the repository URL in the host's +`XMD_WORKFLOW_GITHUB_PULL_REQUESTS` allowlist. For a host that reads one repository: + +```sh +export XMD_WORKFLOW_GITHUB_PULL_REQUESTS='{"allowed":["https://github.com/owner/repository"]}' +``` + +Once GitHub returns the matching comment, the workflow saves its URL with the +reviewed SHA. A new head has its own comment. + +If `nextAction` is `publish_review`, use `--props-action publish-review` with +the same record and PR number. It checks for the matching comment before any +write and adopts it if GitHub already accepted it. Concurrent calls with the +same delivery record share one write reservation per PR and reviewed SHA. +If a write could have reached GitHub and no matching comment is visible, the +reservation stays in place for reconciliation. `status` and `closeout` only +read GitHub. A pending comment holds readiness and the repair handoff. + Reconciliation is limited to independent review from a work item launched under the same response contract. Its assignment retains the exact response schema; a missing or different schema refuses rather than reinterpreting an old reply. It validates only corrections recorded under the original review assignment. @@ -377,8 +400,9 @@ Main health reuses the repository's authoritative selection rule for a push to main's exact current head. Pending, cancelled, skipped and failed runs cannot pass. The maintainer-applied `ci-main-red-fix` exception keeps the repository's CI contract: it substitutes composability for the main lookup -inside CI. A draft PR remains `publication_pending`; only a confirmed merge with -a merge commit reports `delivered`. Refresh status before acting on readiness. +inside CI. A draft with a verified passing review and passing checks reports +`publication_pending` with `finish_pr_metadata`. A confirmed merge with a +merge commit reports `delivered`. Refresh status before acting on readiness. The saved `status.json` is the last observation, not a second execution state. Use the command for a current answer. Local records are trusted workflow diff --git a/examples/adversarial-implementation/components/Delivery/Closeout.md b/examples/adversarial-implementation/components/Delivery/Closeout.md index 7afb51765..54adb6f4b 100644 --- a/examples/adversarial-implementation/components/Delivery/Closeout.md +++ b/examples/adversarial-implementation/components/Delivery/Closeout.md @@ -30,6 +30,10 @@ Next action: {status.nextAction} Independent review: {status.review.result.verdict} + +Published review comment {status.reviewPublication.url} + + {status.review.result.summary} diff --git a/examples/adversarial-implementation/components/Delivery/Reconcile.md b/examples/adversarial-implementation/components/Delivery/Reconcile.md index bf5daaf8a..3a8ae46a9 100644 --- a/examples/adversarial-implementation/components/Delivery/Reconcile.md +++ b/examples/adversarial-implementation/components/Delivery/Reconcile.md @@ -54,4 +54,4 @@ completion turn can supply the review. - + diff --git a/examples/adversarial-implementation/components/Delivery/ReviewComment.ts b/examples/adversarial-implementation/components/Delivery/ReviewComment.ts new file mode 100644 index 000000000..ac9898bb3 --- /dev/null +++ b/examples/adversarial-implementation/components/Delivery/ReviewComment.ts @@ -0,0 +1,256 @@ +import { rm } from "@effectionx/fs"; +import { API, cwd, exec, parseFilesFailure } from "@executablemd/runtime"; +import { ensure, scoped, type Operation } from "effection"; +import { z } from "zod"; +import { createDirectory, digest } from "../Ticket/native-node.ts"; +import { clear, write } from "../Ticket/text-storage.ts"; +import { reviewerSchema } from "./records.ts"; + +export const props = { + type: "object", + properties: { + target: { type: "object" }, + review: { type: "object" }, + publish: { type: "boolean" }, + allowPost: { type: "boolean" }, + comments: { type: "array" }, + }, + required: ["target", "review", "publish", "allowPost", "comments"], + additionalProperties: false, +}; + +const sha = z.string().regex(/^[0-9a-f]{40}$/); +const targetSchema = z.strictObject({ + directory: z.string().startsWith("/"), + repository: z.string().regex(/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/), + number: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + branch: z.string().min(1), + revision: sha, +}); +const input = z.strictObject({ + target: targetSchema, + review: z.object({ + reviewer: reviewerSchema, + model: z.string().min(1).nullable(), + result: z.object({ + revision: sha, + verdict: z.enum(["pass", "changes_required", "blocked"]), + summary: z.string().min(1).max(1500), + findings: z.array(z.object({ blocking: z.boolean(), description: z.string().min(1) })), + }), + }), + publish: z.boolean(), + allowPost: z.boolean(), + comments: z.array( + z.object({ + kind: z.enum(["conversation", "review"]), + id: z.string(), + url: z.string(), + body: z.string(), + }), + ), +}); +const pulls = z.object({ + number: targetSchema.shape.number, + base: z.object({ repo: z.object({ full_name: z.string() }) }), + head: z.object({ ref: z.string(), sha }), +}); +const saved = z.strictObject({ + revision: sha, + bodySha256: z.string().regex(/^[0-9a-f]{64}$/), + id: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + url: z.string(), +}); + +type Target = z.infer; +type Review = z.infer["review"]; +type Verified = z.infer; + +function pending(reason: string) { + return { state: "pending", reason }; +} + +function quote(text: string) { + return text + .replace(/\r\n?/g, "\n") + .split("\n") + .map((line) => " " + line) + .join("\n"); +} + +function render(marker: string, review: Review, model: string) { + const { revision, verdict, summary, findings } = review.result; + const titles = { pass: "PASS", changes_required: "CHANGES REQUIRED", blocked: "BLOCKED" }; + return [ + marker, + "## Independent review", + "Reviewed commit `" + revision + "`", + "Verdict **" + titles[verdict] + "**", + "Reviewer\n\n" + quote(review.reviewer.provider + " / " + model), + "### Summary\n\n" + quote(summary), + "### Findings", + ...(findings.length === 0 + ? ["None."] + : findings.map( + ({ blocking, description: findingText }) => + (blocking ? "**Blocking**" : "**Non-blocking**") + "\n\n" + quote(findingText), + )), + ].join("\n\n"); +} + +function* github(...args: string[]) { + return yield* exec({ + command: ["gh", "api", "--hostname", "github.com", ...args], + cwd: yield* cwd(), + timeout: 30_000, + }); +} + +function* readTarget(target: Target) { + const { exitCode, stdout } = yield* github(`repos/${target.repository}/pulls/${target.number}`); + if (exitCode !== 0) { + return false; + } + const pr = pulls.parse(JSON.parse(stdout)); + return ( + pr.number === target.number && + pr.base.repo.full_name.toLowerCase() === target.repository.toLowerCase() && + pr.head.ref === target.branch && + pr.head.sha === target.revision + ); +} + +function findComment( + target: Target, + comments: z.infer["comments"], + marker: string, + body: string, +): Verified | null { + const matches = comments.filter( + (item) => item.kind === "conversation" && item.body.split("\n", 1)[0] === marker, + ); + if (matches.length > 1) { + throw new Error("Multiple GitHub comments claim the same review revision."); + } + const [value] = matches; + if (value === undefined) { + return null; + } + const { repository, number, revision } = target; + const id = Number(value.id); + const urls = ["issues", "pull"].map((kind) => + `https://github.com/${repository}/${kind}/${number}#issuecomment-${id}`.toLowerCase(), + ); + if ( + !Number.isSafeInteger(id) || + id <= 0 || + value.id !== String(id) || + value.body !== body || + !urls.includes(value.url.toLowerCase()) + ) { + throw new Error("The GitHub comment does not match the saved review."); + } + return { revision, bodySha256: digest(new TextEncoder().encode(body)), id, url: value.url }; +} + +function* readSaved(directory: string) { + const result = yield* API.Files.operations.readTextFile({ cwd: directory, path: "comment.json" }); + if (result.ok) { + return saved.parse(JSON.parse(result.value)); + } + if (parseFilesFailure(result.error)?.reason === "missing") { + return null; + } + throw result.error; +} + +function* post(target: Target, reservation: string, body: string): Operation { + try { + // A reservation survives interruption so a retry can adopt the remote comment. + yield* createDirectory(reservation); + } catch (error) { + if (z.object({ code: z.literal("EEXIST") }).safeParse(error).success) { + return; + } + throw error; + } + yield* scoped(function* () { + let release = true; + yield* ensure(function* () { + if (release) { + yield* rm(reservation, { recursive: true, force: true }); + } + }); + const payload = reservation + "/request.json"; + yield* write(target.directory, payload, JSON.stringify({ body })); + if (!(yield* readTarget(target))) { + return; + } + release = false; + const { exitCode, stdout } = yield* github( + "--include", + "--method", + "POST", + `repos/${target.repository}/issues/${target.number}/comments`, + "--input", + payload, + ); + // An explicit rejection permits another attempt. Transport and server errors leave the reservation in place. + release = exitCode !== 0 && /^HTTP\/\d(?:\.\d)? (400|401|403|404|422|429)\b/.test(stdout); + }); +} + +export default function* ReviewComment(value: unknown) { + const { target, review, publish, allowPost, comments } = input.parse(value); + if (review.result.revision !== target.revision) { + return pending("The retained review names a different revision."); + } + const marker = ``; + const model = review.model ?? review.reviewer.model; + const body = render(marker, review, model); + const record = `${target.directory}/review-${target.revision}/publication-${target.number}`; + try { + const previous = yield* readSaved(record); + if (!publish && previous === null) { + return pending("Publish the joined review and verify the GitHub comment."); + } + if (body.length > 65_536) { + return pending("The rendered review exceeds the GitHub comment limit."); + } + yield* clear( + [ + model, + review.result.summary, + ...review.result.findings.map((finding) => finding.description), + ].join("\n"), + ); + yield* clear(body); + if (!(yield* readTarget(target))) { + return pending("The current pull request does not match the reviewed revision."); + } + + const observed = findComment(target, comments, marker, body); + if (observed === null && publish && allowPost && previous === null) { + yield* post(target, record, body); + return { state: "readback" }; + } + if (observed === null) { + return pending( + "The current review comment could not be verified. Retry publication from the saved receipt.", + ); + } + if (previous !== null && JSON.stringify(previous) !== JSON.stringify(observed)) { + return pending("The GitHub comment differs from its recorded identity."); + } + if (publish && previous === null) { + // A matching comment may already exist even if no local request survived. + yield* API.Fs.operations.ensureDir(record); + yield* write(target.directory, record + "/comment.json", JSON.stringify(observed)); + } + return { state: "verified", ...observed }; + } catch { + return pending( + "The current review comment could not be verified. Check publication access and reconcile the saved receipt.", + ); + } +} diff --git a/examples/adversarial-implementation/components/Delivery/ReviewPublication.md b/examples/adversarial-implementation/components/Delivery/ReviewPublication.md new file mode 100644 index 000000000..29e5e522b --- /dev/null +++ b/examples/adversarial-implementation/components/Delivery/ReviewPublication.md @@ -0,0 +1,26 @@ +--- +description: 'Verify a saved review comment or publish it from its validated receipt. Use .' +props: + type: object + properties: + target: {type: object} + review: {type: object} + publish: {type: boolean} + required: [target, review, publish] + additionalProperties: false +returns: {type: object} +--- + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/components/Delivery/Status.md b/examples/adversarial-implementation/components/Delivery/Status.md index 5db424e04..636061438 100644 --- a/examples/adversarial-implementation/components/Delivery/Status.md +++ b/examples/adversarial-implementation/components/Delivery/Status.md @@ -34,7 +34,8 @@ returns: {type: object} -{reading.output} +{reading.output} + @@ -43,7 +44,6 @@ returns: {type: object} - ```bash exec as="branchRead" git symbolic-ref --quiet --short HEAD @@ -51,13 +51,20 @@ git symbolic-ref --quiet --short HEAD + + + + + + + + - + gate.state !== "passed")} /> - - gate.state === "failed") ? "checks_failed" : remaining.length !== 0 ? "checks_pending" : remote.merged && /^[0-9a-f]{40}$/.test(remote.mergeRevision) ? "delivered" : remote.state !== "open" ? "closed_unmerged" : remote.draft ? "publication_pending" : "ready_for_merge"} /> - - + gate.state === "failed") ? "checks_failed" : remaining.length !== 0 ? "checks_pending" : remote.merged && /^[0-9a-f]{40}$/.test(remote.mergeRevision) ? "delivered" : remote.state !== "open" ? "closed_unmerged" : remote.draft ? "publication_pending" : "ready_for_merge"} /> + + diff --git a/examples/adversarial-implementation/delivery.md b/examples/adversarial-implementation/delivery.md index ea366dadc..4b51b01c1 100644 --- a/examples/adversarial-implementation/delivery.md +++ b/examples/adversarial-implementation/delivery.md @@ -4,7 +4,7 @@ props: properties: record: {type: string, minLength: 1} number: {type: integer, minimum: 1} - action: {enum: [status, review, reconcile, closeout], default: status} + action: {enum: [status, review, reconcile, publish-review, closeout], default: status} required: [record, number] additionalProperties: false --- @@ -13,20 +13,35 @@ Read the original delivery record and perform only its computed next action. A status read starts no agent. An explicit progress reply gets one continuation in the same conversation. An unavailable Fable reviewer is replaced once by Codex Astra. The workflow preserves each attempt and asks about a review it -cannot complete. Readiness permits no merge. +cannot complete. Delivery publishes joined verdicts as pull request comments. +`publish-review` retries publication from the saved receipt. Readiness permits no merge. {recordText} + - + - + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/tests/Delivery.test.md b/examples/adversarial-implementation/tests/Delivery.test.md index 364cbd7b9..47a481a39 100644 --- a/examples/adversarial-implementation/tests/Delivery.test.md +++ b/examples/adversarial-implementation/tests/Delivery.test.md @@ -35,28 +35,6 @@ printf '%s' "$XMD_EXAMPLE_COMMAND" - - - - -```bash exec as="identity" -git rev-parse HEAD -git symbolic-ref --short HEAD -``` - - - - - - - - - - - - - - diff --git a/examples/adversarial-implementation/tests/DeliveryPublication.test.md b/examples/adversarial-implementation/tests/DeliveryPublication.test.md new file mode 100644 index 000000000..75adfb206 --- /dev/null +++ b/examples/adversarial-implementation/tests/DeliveryPublication.test.md @@ -0,0 +1,32 @@ +```bash exec as="location" +pwd -P +``` + +```bash exec as="commandSetting" +printf '%s' "$XMD_EXAMPLE_COMMAND" +``` + + + + + + + +```bash exec as="identity" +git rev-parse HEAD +git symbolic-ref --short HEAD +``` + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts new file mode 100644 index 000000000..712f1e5f4 --- /dev/null +++ b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts @@ -0,0 +1,97 @@ +import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { PullRequestAPI } from "@executablemd/git/api"; +import { API } from "@executablemd/runtime"; +import { z } from "zod"; +import { waitForFile } from "./receipts.ts"; + +const repository = "taras/executable.md"; +const pr = `repos/${repository}/pulls/7`; +const comments = `repos/${repository}/issues/7/comments`; +const url = `https://github.com/${repository}/pull/7`; +const evidence = z.strictObject({ + kind: z.literal("conversation"), + id: z.string(), + url: z.string(), + author: z.string(), + body: z.string(), + createdAt: z.string(), + updatedAt: z.string(), +}); +const reply = (stdout: string, exitCode = 0) => ({ stdout, stderr: "", exitCode }); + +export function* installGitHub(directory: string, checkout: string, actor: string) { + let lost = false; + const remote = z.object({ revision: z.string(), branch: z.string() }); + const payload = z.strictObject({ body: z.string() }); + const tracked = `${directory}/comment.json`; + + yield* PullRequestAPI.around({ + *read([target, options]) { + if (target !== url || options.kind !== "comments") { + throw new Error("Unexpected pull request evidence read."); + } + if (lost || (yield* exists(`${directory}/comments-unavailable`))) { + throw new Error("The pull request comments are unavailable."); + } + const items = (yield* exists(tracked)) + ? [evidence.parse(JSON.parse(yield* readTextFile(tracked)))] + : []; + return { kind: "comments", items }; + }, + }); + yield* API.Process.around({ + *exec([options], next) { + const [tool, ...args] = options.command; + if (tool !== "gh") { + return yield* next(options); + } + if (args.slice(0, 3).join(" ") !== "api --hostname github.com") { + throw new Error("Unexpected GitHub CLI boundary."); + } + const rest = args.slice(3); + if (rest.join(" ") === pr) { + const { revision, branch } = remote.parse( + JSON.parse(yield* readTextFile(`${checkout}/remote.json`)), + ); + const stale = yield* exists(`${directory}/stale`); + return reply( + JSON.stringify({ + number: 7, + base: { repo: { full_name: repository } }, + head: { ref: branch, sha: stale ? "0".repeat(40) : revision }, + }), + ); + } + if ( + rest.length !== 6 || + rest.slice(0, 5).join(" ") !== `--include --method POST ${comments} --input` + ) { + throw new Error("Unexpected GitHub comment command."); + } + yield* writeTextFile(`${directory}/${actor}-posted`, ""); + const body = payload.parse(JSON.parse(yield* readTextFile(rest[5]))).body; + const delayed = yield* exists(`${directory}/delay-post`); + if (delayed && actor === "winner") { + yield* writeTextFile(`${directory}/post-ready`, ""); + if ((yield* waitForFile(`${directory}/release-post`, 45_000)).timeout) { + throw new Error("Timed out waiting to release the GitHub write."); + } + } + if (yield* exists(tracked)) { + throw new Error("GitHub received a duplicate review write."); + } + const saved = evidence.parse({ + kind: "conversation", + id: "42", + url: `${url}#issuecomment-42`, + author: "workflow", + createdAt: "2026-11-15T00:00:00Z", + updatedAt: "2026-11-15T00:00:00Z", + body, + }); + yield* writeTextFile(tracked, JSON.stringify(saved)); + lost = delayed; + return delayed ? reply("", 1) : reply("HTTP/2.0 201 Created\r\n\r\n" + JSON.stringify(saved)); + }, + }); +} diff --git a/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts b/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts index 2b826cf95..2361ec425 100644 --- a/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts +++ b/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts @@ -1,17 +1,18 @@ import { createAcpRuntime, createAcpxProvider, createAcpxSessionStore } from "@executablemd/acp"; import { AgentProviders, registerAgentProvider } from "@executablemd/core"; import { Component, Plugin } from "@executablemd/core/api"; -import { API, command, env, readTextFile } from "@executablemd/runtime"; +import { API, command, cwd, env, readTextFile } from "@executablemd/runtime"; import { useTestAgentController } from "@executablemd/test-agent"; import { writeTextFile } from "@effectionx/fs"; import { ensure } from "effection"; +import { installGitHub } from "./github-comments.ts"; import { actorEnv, actorSchema, fableFailure, overlapEnv, waitForFile } from "./receipts.ts"; import type { PromptReceipt } from "./receipts.ts"; export default Plugin({ name: "delivery-acp-overlap", - *install(request) { - if (request.command !== "run") { + *install(invocation) { + if (invocation.command !== "run") { throw new Error("Delivery ACP workers require a run invocation."); } const directory = yield* env(overlapEnv); @@ -19,6 +20,7 @@ export default Plugin({ if (!directory?.startsWith("/")) { throw new Error("Delivery ACP workers require an absolute fixture directory."); } + yield* installGitHub(directory, yield* cwd(), actor); const receipts: PromptReceipt[] = []; yield* ensure(() => writeTextFile(`${directory}/${actor}-prompts.json`, JSON.stringify(receipts)), @@ -89,10 +91,10 @@ export default Plugin({ return [...worker, "--connect", scenario.route]; }, }, - createRuntime(options) { + createRuntime(runtimeOptions) { const pending = new Map(); return createAcpRuntime({ - ...options, + ...runtimeOptions, onAcpMessage(direction, message) { if ( direction === "outbound" && diff --git a/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md b/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md new file mode 100644 index 000000000..5efa82382 --- /dev/null +++ b/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md @@ -0,0 +1,5 @@ +--- +props: {type: object, properties: {target: {type: object}, review: {type: object}, publish: {type: boolean}}, required: [target, review, publish], additionalProperties: false} +returns: {type: object} +--- + diff --git a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts index 988493a82..d583fffa8 100644 --- a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts +++ b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts @@ -1,4 +1,4 @@ -import { copyFile, ensureDir, exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { copyFile, ensureDir, exists, readTextFile, rm, writeTextFile } from "@effectionx/fs"; import { cwd } from "@executablemd/runtime"; import { runShell, shellQuote } from "@executablemd/test-support/launch"; import { ensure, race, scoped, spawn } from "effection"; @@ -20,8 +20,9 @@ export const props = { command: { type: "string" }, directory: { type: "string" }, revision: { type: "string" }, + scenario: { enum: ["pass", "changes_required", "lost"] }, }, - required: ["command", "directory", "revision"], + required: ["command", "directory", "revision", "scenario"], additionalProperties: false, }; @@ -29,6 +30,7 @@ const input = z.strictObject({ command: z.string(), directory: z.string().startsWith("/"), revision: z.string(), + scenario: z.enum(["pass", "changes_required", "lost"]), }); const production = new URL("../../../", import.meta.url); const fixtures = new URL("../../acp-overlap/", import.meta.url); @@ -45,6 +47,12 @@ function check(condition: boolean, message: string): asserts condition { const statusSchema = z.object({ status: z.string(), + nextAction: z.string(), + reviewPublication: z.object({ + state: z.string(), + revision: z.string().optional(), + url: z.string().optional(), + }), review: z.object({ attempt: z.string(), result: z.object({ @@ -66,16 +74,28 @@ const outcomeSchema = z.object({ }); export default function* ReviewOverlap(value: unknown) { - const { command, directory, revision } = input.parse(value); + const { command, directory, revision, scenario } = input.parse(value); const repository = yield* cwd(); const evidence = `${directory}/overlap`; const attempt = `${directory}/review-${revision}`; const replacement = `${attempt}/replacement`; const release = `${evidence}/release`; yield* ensureDir(evidence); - for (const file of ["Fable.md", "Astra.md", "Gate.md"]) { + for (const file of ["Fable.md", "Gate.md"]) { yield* copyFile(new URL(file, fixtures), `${evidence}/${file}`); } + const astra = yield* readTextFile(new URL("Astra.md", fixtures)); + yield* writeTextFile( + `${evidence}/Astra.md`, + scenario !== "changes_required" + ? astra + : astra + .replace('verdict:"pass"', 'verdict:"changes_required"') + .replace( + "findings:[]", + 'findings:[{blocking:true,description:"Queue drops a waiting message."}]', + ), + ); const args = [ fileURLToPath(new URL("delivery.md", production)), @@ -83,8 +103,6 @@ export default function* ReviewOverlap(value: unknown) { `${directory}/requirements.json`, "--props-number", "7", - "--props-action", - "review", "--include", fileURLToPath(new URL("components", fixtures)), "--include", @@ -98,8 +116,8 @@ export default function* ReviewOverlap(value: unknown) { .map(shellQuote) .join(" "); - const review = (actor: z.infer, timeout: number) => - runShell(`exec ${command} ${args}`, { + const review = (actor: z.infer, timeout: number, action = "review") => + runShell(`exec ${command} ${args} --props-action ${shellQuote(action)}`, { cwd: repository, env: { [overlapEnv]: evidence, [actorEnv]: actor }, inheritEnv: true, @@ -168,6 +186,9 @@ export default function* ReviewOverlap(value: unknown) { `Unexpected contender result (${contender.code}): ${contender.stderr} ${contender.stdout}`, ); + if (scenario === "lost") { + yield* writeTextFile(`${evidence}/stale`, ""); + } yield* writeTextFile(release, ""); released = true; return yield* first; @@ -185,15 +206,119 @@ export default function* ReviewOverlap(value: unknown) { check(winner.code === 0, `Winner exited ${winner.code}: ${winner.stderr} ${winner.stdout}`); const status = statusSchema.parse(JSON.parse(winner.stdout)); check( - status.status === "ready_for_merge" && + status.status === + (scenario === "pass" + ? "ready_for_merge" + : scenario === "changes_required" + ? "repair_required" + : "publication_pending") && status.review.attempt === replacement && status.review.result.revision === revision && - status.review.result.verdict === "pass" && + status.review.result.verdict === + (scenario === "changes_required" ? "changes_required" : "pass") && status.review.result.acceptance.length === 1 && status.review.result.acceptance[0] === "behavior", `Winner did not retain its exact-head Astra pass: ${JSON.stringify(status)}`, ); + if (scenario === "lost") { + check(!(yield* exists(`${evidence}/winner-posted`)), "A stale PR head received a POST."); + check( + status.nextAction === "publish_review", + "A missing publication must take precedence over merge readiness.", + ); + yield* rm(`${evidence}/stale`); + yield* writeTextFile(`${evidence}/comments-unavailable`, ""); + const unavailable = yield* review("later", 25_000, "publish-review"); + check( + unavailable.code === 0 && + statusSchema.parse(JSON.parse(unavailable.stdout)).nextAction === "publish_review", + "Unavailable native comments granted readiness.", + ); + check( + !(yield* exists(`${evidence}/later-posted`)), + "Unavailable native comments allowed a POST.", + ); + yield* rm(`${evidence}/comments-unavailable`); + yield* rm(`${replacement}/receipt.json`); + yield* writeTextFile(`${evidence}/delay-post`, ""); + yield* scoped(function* () { + const first = yield* spawn(() => review("winner", 75_000, "reconcile")); + yield* ensure(function* () { + yield* writeTextFile(`${evidence}/release-post`, ""); + yield* first; + }); + function* exitedEarly(): Operation { + const result = yield* first; + throw new Error( + `Reconcile exited before comment publication (${result.code}): ${result.stderr} ${result.stdout}`, + ); + } + const ready = yield* race([waitForFile(`${evidence}/post-ready`, 35_000), exitedEarly()]); + check(!ready.timeout, "Reconciliation did not attempt to publish the joined result."); + const overlapping = yield* review("contender", 25_000, "publish-review"); + check(overlapping.code === 0, `Concurrent publication failed: ${overlapping.stderr}`); + const waiting = statusSchema.parse(JSON.parse(overlapping.stdout)); + check( + waiting.nextAction === "publish_review", + "Concurrent publish did not wait for readback.", + ); + check( + !(yield* exists(`${evidence}/contender-posted`)), + "Concurrent publication sent a second POST.", + ); + yield* writeTextFile(`${evidence}/release-post`, ""); + const result = yield* first; + check( + result.code === 0, + `Reconciliation after lost acknowledgement failed: ${result.stderr}`, + ); + const unconfirmed = statusSchema.parse(JSON.parse(result.stdout)); + check(unconfirmed.nextAction === "publish_review", "Lost acknowledgement granted readiness."); + }); + const report = yield* review("later", 25_000, "closeout"); + check( + report.code === 0 && + statusSchema.parse(JSON.parse(report.stdout)).nextAction === "publish_review", + "Closeout must leave unconfirmed publication pending.", + ); + check(!(yield* exists(`${evidence}/later-posted`)), "Closeout attempted a GitHub write."); + } + const published = yield* review("later", 25_000, "publish-review"); + check(published.code === 0, `Publication retry failed: ${published.stderr} ${published.stdout}`); + const verified = statusSchema.parse(JSON.parse(published.stdout)); + check( + verified.status === (scenario === "changes_required" ? "repair_required" : "ready_for_merge") && + verified.reviewPublication.state === "verified" && + verified.reviewPublication.revision === revision && + verified.reviewPublication.url === + "https://github.com/taras/executable.md/pull/7#issuecomment-42", + `Publication was not verified: ${JSON.stringify(verified)}`, + ); + const comment = z + .object({ body: z.string() }) + .parse(JSON.parse(yield* readTextFile(`${evidence}/comment.json`))); + check( + comment.body.startsWith(``) && + comment.body.includes("codex / gpt-6-astra") && + comment.body.includes("Reviewed the assigned behavior.") && + comment.body.includes( + scenario === "changes_required" ? "**CHANGES REQUIRED**" : "**PASS**", + ) && + (scenario !== "changes_required" || comment.body.includes("Queue drops a waiting message.")), + `The published comment does not describe the validated Astra review: ${comment.body}`, + ); + for (const actor of ["contender", "later"] as const) { + check(!(yield* exists(`${evidence}/${actor}-posted`)), `${actor} sent an extra POST.`); + check( + (yield* readPrompts(evidence, actor)).length === 0, + `${actor} contacted ACP during publication.`, + ); + } + if (scenario === "lost") { + check((yield* readPrompts(evidence, "winner")).length === 0, "Reconciliation contacted ACP."); + } + const later = yield* review("later", 25_000); const laterPrompts = yield* readPrompts(evidence, "later"); check( @@ -203,7 +328,7 @@ export default function* ReviewOverlap(value: unknown) { check( later.code === 1 && later.stderr.includes( - "The next action is await_authorized_merge; do not start another review.", + `The next action is ${scenario === "changes_required" ? "request_approved_repair" : "await_authorized_merge"}; do not start another review.`, ), `Unexpected later result (${later.code}): ${later.stderr} ${later.stdout}`, ); diff --git a/scripts/tests/runtime-exclusions.test.ts b/scripts/tests/runtime-exclusions.test.ts index 125123dd9..360dddc84 100644 --- a/scripts/tests/runtime-exclusions.test.ts +++ b/scripts/tests/runtime-exclusions.test.ts @@ -179,6 +179,7 @@ describe("the applicable corpus", () => { "ArtifactReturnRecovery", "DecisionResolution", "Delivery", + "DeliveryPublication", "Friction", "Implementation", "LaunchPolicy",