From b331068460a98dd6e9adfd856282479b19e0afee Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sun, 15 Nov 2026 04:35:46 -0500 Subject: [PATCH 1/3] Publish validated Delivery reviews as PR comments --- examples/adversarial-implementation/README.md | 32 ++- .../components/Delivery/Closeout.md | 4 + .../components/Delivery/Reconcile.md | 2 +- .../components/Delivery/ReviewPublication.ts | 262 ++++++++++++++++++ .../components/Delivery/Status.md | 21 +- .../adversarial-implementation/delivery.md | 23 +- .../tests/Delivery.test.md | 22 -- .../tests/DeliveryPublication.test.md | 32 +++ .../tests/acp-overlap/github-comments.ts | 76 +++++ .../tests/acp-overlap/local-workers.ts | 12 +- .../components/Delivery/ReviewPublication.md | 5 + .../tests/components/Fixture/ReviewOverlap.ts | 135 ++++++++- 12 files changed, 568 insertions(+), 58 deletions(-) create mode 100644 examples/adversarial-implementation/components/Delivery/ReviewPublication.ts create mode 100644 examples/adversarial-implementation/tests/DeliveryPublication.test.md create mode 100644 examples/adversarial-implementation/tests/acp-overlap/github-comments.ts create mode 100644 examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md diff --git a/examples/adversarial-implementation/README.md b/examples/adversarial-implementation/README.md index eaf55140d..1733ff151 100644 --- a/examples/adversarial-implementation/README.md +++ b/examples/adversarial-implementation/README.md @@ -248,8 +248,8 @@ xmd run --allow-absolute /absolute/path/to/workflow/examples/adversarial-impleme --include /absolute/path/to/workflow/examples/adversarial-implementation/components --raw ``` -`status` starts no model. It reads the saved joined review, the current PR and -GitHub's CI workflow runs. Its `nextAction` is computed from the same facts as +`status` starts no model. It reads the saved joined review, its published comment, +the current PR and GitHub's CI workflow runs. Its `nextAction` is computed from the same facts as its `status` and `remaining` fields. When it requests `review`, select `--props-action review` on that command to perform the independent review with the provider, model, effort and session recorded at launch. The review uses @@ -295,10 +295,11 @@ feedback ready -> independent review pending -> joined exact-commit verdict | Fable unavailable/refused -> one Astra review (review) | no reviewer completed -> review_held: answer via review | incomplete/uncertain -> reconcile original call - | blocking finding -> request approved repair - | pass + required CI + main -> finish PR metadata - | all passed + PR ready -> await authorized merge - | confirmed merge -> delivered + | comment pending -> publish-review + | published changes required -> request approved repair + | published pass + checks -> finish PR metadata + | all passed + PR ready -> await authorized merge + | confirmed merge -> delivered ``` Independent review summaries allow 1500 characters; detailed observations belong @@ -340,6 +341,20 @@ and saves a raw reply the execution joined but never wrote. It never shortens te a model call or overwrites a contradictory receipt. Missing or uncertain execution continues to hold. Status uses this same validation boundary to read receipts. +`review` and `reconcile` publish each validated verdict as a comment on the +current pull request, including findings that require repair. The comment +names the exact reviewed commit and the reviewer who supplied the result. +Once GitHub returns the matching comment, the workflow saves its URL with the +reviewed SHA. A new head has its own comment. + +If `nextAction` is `publish_review`, use `--props-action publish-review` with +the same record and PR number. It checks for the matching comment before any +write and adopts it if GitHub already accepted it. Concurrent calls with the +same delivery record share one write reservation per PR and reviewed SHA. +If a write could have reached GitHub and no matching comment is visible, the +reservation stays in place for reconciliation. `status` and `closeout` only +read GitHub. A pending comment holds readiness and the repair handoff. + Reconciliation is limited to independent review from a work item launched under the same response contract. Its assignment retains the exact response schema; a missing or different schema refuses rather than reinterpreting an old reply. It validates only corrections recorded under the original review assignment. @@ -377,8 +392,9 @@ Main health reuses the repository's authoritative selection rule for a push to main's exact current head. Pending, cancelled, skipped and failed runs cannot pass. The maintainer-applied `ci-main-red-fix` exception keeps the repository's CI contract: it substitutes composability for the main lookup -inside CI. A draft PR remains `publication_pending`; only a confirmed merge with -a merge commit reports `delivered`. Refresh status before acting on readiness. +inside CI. A draft with a verified passing review and passing checks reports +`publication_pending` with `finish_pr_metadata`. A confirmed merge with a +merge commit reports `delivered`. Refresh status before acting on readiness. The saved `status.json` is the last observation, not a second execution state. Use the command for a current answer. Local records are trusted workflow diff --git a/examples/adversarial-implementation/components/Delivery/Closeout.md b/examples/adversarial-implementation/components/Delivery/Closeout.md index 7afb51765..54adb6f4b 100644 --- a/examples/adversarial-implementation/components/Delivery/Closeout.md +++ b/examples/adversarial-implementation/components/Delivery/Closeout.md @@ -30,6 +30,10 @@ Next action: {status.nextAction} Independent review: {status.review.result.verdict} + +Published review comment {status.reviewPublication.url} + + {status.review.result.summary} diff --git a/examples/adversarial-implementation/components/Delivery/Reconcile.md b/examples/adversarial-implementation/components/Delivery/Reconcile.md index bf5daaf8a..3a8ae46a9 100644 --- a/examples/adversarial-implementation/components/Delivery/Reconcile.md +++ b/examples/adversarial-implementation/components/Delivery/Reconcile.md @@ -54,4 +54,4 @@ completion turn can supply the review. - + diff --git a/examples/adversarial-implementation/components/Delivery/ReviewPublication.ts b/examples/adversarial-implementation/components/Delivery/ReviewPublication.ts new file mode 100644 index 000000000..a424dc04d --- /dev/null +++ b/examples/adversarial-implementation/components/Delivery/ReviewPublication.ts @@ -0,0 +1,262 @@ +import { rm } from "@effectionx/fs"; +import { API, cwd, exec, parseFilesFailure } from "@executablemd/runtime"; +import { ensure, scoped, type Operation } from "effection"; +import { z } from "zod"; +import { createDirectory, digest } from "../Ticket/native-node.ts"; +import { clear, write } from "../Ticket/text-storage.ts"; +import { reviewerSchema } from "./records.ts"; + +export const description = + 'Verify a saved review comment or publish it from its validated receipt. Use .'; + +export const props = { + type: "object", + properties: { + target: { type: "object" }, + review: { type: "object" }, + publish: { type: "boolean" }, + }, + required: ["target", "review", "publish"], + additionalProperties: false, +}; + +const sha = z.string().regex(/^[0-9a-f]{40}$/); +const targetSchema = z.strictObject({ + directory: z.string().startsWith("/"), + repository: z.string().regex(/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/), + number: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + branch: z.string().min(1), + revision: sha, +}); +const input = z.strictObject({ + target: targetSchema, + review: z.object({ + reviewer: reviewerSchema, + model: z.string().min(1).nullable(), + result: z.object({ + revision: sha, + verdict: z.enum(["pass", "changes_required", "blocked"]), + summary: z.string().min(1).max(1500), + findings: z.array(z.object({ blocking: z.boolean(), description: z.string().min(1) })), + }), + }), + publish: z.boolean(), +}); +const pulls = z.object({ + number: targetSchema.shape.number, + base: z.object({ repo: z.object({ full_name: z.string() }) }), + head: z.object({ ref: z.string(), sha }), +}); +const comment = z.object({ + id: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + issue_url: z.string(), + html_url: z.string(), + body: z.string(), +}); +const saved = z.strictObject({ + revision: sha, + bodySha256: z.string().regex(/^[0-9a-f]{64}$/), + id: comment.shape.id, + url: z.string(), +}); + +type Target = z.infer; +type Review = z.infer["review"]; +type Verified = z.infer; + +function pending(reason: string) { + return { state: "pending", reason }; +} + +function quote(text: string) { + return text + .replace(/\r\n?/g, "\n") + .split("\n") + .map((line) => " " + line) + .join("\n"); +} + +function render(marker: string, review: Review, model: string) { + const { revision, verdict, summary, findings } = review.result; + const titles = { pass: "PASS", changes_required: "CHANGES REQUIRED", blocked: "BLOCKED" }; + return [ + marker, + "## Independent review", + "Reviewed commit `" + revision + "`", + "Verdict **" + titles[verdict] + "**", + "Reviewer\n\n" + quote(review.reviewer.provider + " / " + model), + "### Summary\n\n" + quote(summary), + "### Findings", + ...(findings.length === 0 + ? ["None."] + : findings.map( + ({ blocking, description: findingText }) => + (blocking ? "**Blocking**" : "**Non-blocking**") + "\n\n" + quote(findingText), + )), + ].join("\n\n"); +} + +function* github(...args: string[]) { + return yield* exec({ + command: ["gh", "api", "--hostname", "github.com", ...args], + cwd: yield* cwd(), + timeout: 30_000, + }); +} + +function* readTarget(target: Target) { + const { exitCode, stdout } = yield* github(`repos/${target.repository}/pulls/${target.number}`); + if (exitCode !== 0) { + return false; + } + const pr = pulls.parse(JSON.parse(stdout)); + return ( + pr.number === target.number && + pr.base.repo.full_name.toLowerCase() === target.repository.toLowerCase() && + pr.head.ref === target.branch && + pr.head.sha === target.revision + ); +} + +function verifiedComment(target: Target, value: z.infer, body: string): Verified { + const { repository, number, revision } = target; + const validUrls = ["issues", "pull"].map((kind) => + `https://github.com/${repository}/${kind}/${number}#issuecomment-${value.id}`.toLowerCase(), + ); + if ( + value.body !== body || + value.issue_url.toLowerCase() !== + `https://api.github.com/repos/${repository}/issues/${number}`.toLowerCase() || + !validUrls.includes(value.html_url.toLowerCase()) + ) { + throw new Error("The GitHub comment does not match the saved review."); + } + return { + revision, + bodySha256: digest(new TextEncoder().encode(body)), + id: value.id, + url: value.html_url, + }; +} + +function* findComment(target: Target, marker: string, body: string) { + const { exitCode, stdout } = yield* github( + "--paginate", + "--slurp", + `repos/${target.repository}/issues/${target.number}/comments?per_page=100`, + ); + if (exitCode !== 0) { + throw new Error("Could not read GitHub review comments."); + } + const matches = z + .array(z.array(comment)) + .parse(JSON.parse(stdout)) + .flat() + .filter((item) => item.body.split("\n", 1)[0] === marker); + if (matches.length > 1) { + throw new Error("Multiple GitHub comments claim the same review revision."); + } + return matches[0] === undefined ? null : verifiedComment(target, matches[0], body); +} + +function* readSaved(directory: string) { + const result = yield* API.Files.operations.readTextFile({ cwd: directory, path: "comment.json" }); + if (result.ok) { + return saved.parse(JSON.parse(result.value)); + } + if (parseFilesFailure(result.error)?.reason === "missing") { + return null; + } + throw result.error; +} + +function* post(target: Target, reservation: string, body: string): Operation { + try { + // A reservation survives interruption so a retry can adopt the remote comment. + yield* createDirectory(reservation); + } catch (error) { + if (z.object({ code: z.literal("EEXIST") }).safeParse(error).success) { + return; + } + throw error; + } + yield* scoped(function* () { + let release = true; + yield* ensure(function* () { + if (release) { + yield* rm(reservation, { recursive: true, force: true }); + } + }); + const payload = reservation + "/request.json"; + yield* write(target.directory, payload, JSON.stringify({ body })); + if (!(yield* readTarget(target))) { + return; + } + release = false; + const { exitCode, stdout } = yield* github( + "--include", + "--method", + "POST", + `repos/${target.repository}/issues/${target.number}/comments`, + "--input", + payload, + ); + // An explicit rejection permits another attempt. Transport and server errors leave the reservation in place. + release = exitCode !== 0 && /^HTTP\/\d(?:\.\d)? (400|401|403|404|422|429)\b/.test(stdout); + }); +} + +export default function* ReviewPublication(value: unknown) { + const { target, review, publish } = input.parse(value); + if (review.result.revision !== target.revision) { + return pending("The retained review names a different revision."); + } + const marker = ``; + const model = review.model ?? review.reviewer.model; + const body = render(marker, review, model); + const record = `${target.directory}/review-${target.revision}/publication-${target.number}`; + try { + const previous = yield* readSaved(record); + if (!publish && previous === null) { + return pending("Publish the joined review and verify the GitHub comment."); + } + if (body.length > 65_536) { + return pending("The rendered review exceeds the GitHub comment limit."); + } + yield* clear( + [ + model, + review.result.summary, + ...review.result.findings.map((finding) => finding.description), + ].join("\n"), + ); + yield* clear(body); + if (!(yield* readTarget(target))) { + return pending("The current pull request does not match the reviewed revision."); + } + + let observed = yield* findComment(target, marker, body); + if (observed === null && publish && previous === null) { + yield* post(target, record, body); + observed = yield* findComment(target, marker, body); + } + if (observed === null || !(yield* readTarget(target))) { + return pending( + "The current review comment could not be verified. Retry publication from the saved receipt.", + ); + } + if (previous !== null && JSON.stringify(previous) !== JSON.stringify(observed)) { + return pending("The GitHub comment differs from its recorded identity."); + } + if (publish && previous === null) { + // A matching comment may already exist even if no local request survived. + yield* API.Fs.operations.ensureDir(record); + yield* write(target.directory, record + "/comment.json", JSON.stringify(observed)); + } + return { state: "verified", ...observed }; + } catch { + return pending( + "The current review comment could not be verified. Check publication access and reconcile the saved receipt.", + ); + } +} diff --git a/examples/adversarial-implementation/components/Delivery/Status.md b/examples/adversarial-implementation/components/Delivery/Status.md index 5db424e04..636061438 100644 --- a/examples/adversarial-implementation/components/Delivery/Status.md +++ b/examples/adversarial-implementation/components/Delivery/Status.md @@ -34,7 +34,8 @@ returns: {type: object} -{reading.output} +{reading.output} + @@ -43,7 +44,6 @@ returns: {type: object} - ```bash exec as="branchRead" git symbolic-ref --quiet --short HEAD @@ -51,13 +51,20 @@ git symbolic-ref --quiet --short HEAD + + + + + + + + - + gate.state !== "passed")} /> - - gate.state === "failed") ? "checks_failed" : remaining.length !== 0 ? "checks_pending" : remote.merged && /^[0-9a-f]{40}$/.test(remote.mergeRevision) ? "delivered" : remote.state !== "open" ? "closed_unmerged" : remote.draft ? "publication_pending" : "ready_for_merge"} /> - - + gate.state === "failed") ? "checks_failed" : remaining.length !== 0 ? "checks_pending" : remote.merged && /^[0-9a-f]{40}$/.test(remote.mergeRevision) ? "delivered" : remote.state !== "open" ? "closed_unmerged" : remote.draft ? "publication_pending" : "ready_for_merge"} /> + + diff --git a/examples/adversarial-implementation/delivery.md b/examples/adversarial-implementation/delivery.md index ea366dadc..4b51b01c1 100644 --- a/examples/adversarial-implementation/delivery.md +++ b/examples/adversarial-implementation/delivery.md @@ -4,7 +4,7 @@ props: properties: record: {type: string, minLength: 1} number: {type: integer, minimum: 1} - action: {enum: [status, review, reconcile, closeout], default: status} + action: {enum: [status, review, reconcile, publish-review, closeout], default: status} required: [record, number] additionalProperties: false --- @@ -13,20 +13,35 @@ Read the original delivery record and perform only its computed next action. A status read starts no agent. An explicit progress reply gets one continuation in the same conversation. An unavailable Fable reviewer is replaced once by Codex Astra. The workflow preserves each attempt and asks about a review it -cannot complete. Readiness permits no merge. +cannot complete. Delivery publishes joined verdicts as pull request comments. +`publish-review` retries publication from the saved receipt. Readiness permits no merge. {recordText} + - + - + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/tests/Delivery.test.md b/examples/adversarial-implementation/tests/Delivery.test.md index 364cbd7b9..47a481a39 100644 --- a/examples/adversarial-implementation/tests/Delivery.test.md +++ b/examples/adversarial-implementation/tests/Delivery.test.md @@ -35,28 +35,6 @@ printf '%s' "$XMD_EXAMPLE_COMMAND" - - - - -```bash exec as="identity" -git rev-parse HEAD -git symbolic-ref --short HEAD -``` - - - - - - - - - - - - - - diff --git a/examples/adversarial-implementation/tests/DeliveryPublication.test.md b/examples/adversarial-implementation/tests/DeliveryPublication.test.md new file mode 100644 index 000000000..75adfb206 --- /dev/null +++ b/examples/adversarial-implementation/tests/DeliveryPublication.test.md @@ -0,0 +1,32 @@ +```bash exec as="location" +pwd -P +``` + +```bash exec as="commandSetting" +printf '%s' "$XMD_EXAMPLE_COMMAND" +``` + + + + + + + +```bash exec as="identity" +git rev-parse HEAD +git symbolic-ref --short HEAD +``` + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts new file mode 100644 index 000000000..e3d769114 --- /dev/null +++ b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts @@ -0,0 +1,76 @@ +import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { API } from "@executablemd/runtime"; +import { z } from "zod"; +import { waitForFile } from "./receipts.ts"; + +const repository = "taras/executable.md"; +const pr = `repos/${repository}/pulls/7`; +const comments = `repos/${repository}/issues/7/comments`; +const reply = (stdout: string, exitCode = 0) => ({ stdout, stderr: "", exitCode }); + +export function* installGitHub(directory: string, checkout: string, actor: string) { + let lost = false; + const remote = z.object({ revision: z.string(), branch: z.string() }); + const payload = z.strictObject({ body: z.string() }); + const tracked = `${directory}/comment.json`; + + yield* API.Process.around({ + *exec([options], next) { + const [tool, ...args] = options.command; + if (tool !== "gh") { + return yield* next(options); + } + if (args.slice(0, 3).join(" ") !== "api --hostname github.com") { + throw new Error("Unexpected GitHub CLI boundary."); + } + const rest = args.slice(3); + if (rest.join(" ") === pr) { + const { revision, branch } = remote.parse( + JSON.parse(yield* readTextFile(`${checkout}/remote.json`)), + ); + const stale = yield* exists(`${directory}/stale`); + return reply( + JSON.stringify({ + number: 7, + base: { repo: { full_name: repository } }, + head: { ref: branch, sha: stale ? "0".repeat(40) : revision }, + }), + ); + } + if (rest.join(" ") === `--paginate --slurp ${comments}?per_page=100`) { + if (lost) { + return reply("", 1); + } + const page = (yield* exists(tracked)) ? [JSON.parse(yield* readTextFile(tracked))] : []; + return reply(JSON.stringify([[], page])); + } + if ( + rest.length !== 6 || + rest.slice(0, 5).join(" ") !== `--include --method POST ${comments} --input` + ) { + throw new Error("Unexpected GitHub comment command."); + } + yield* writeTextFile(`${directory}/${actor}-posted`, ""); + const body = payload.parse(JSON.parse(yield* readTextFile(rest[5]))).body; + const delayed = yield* exists(`${directory}/delay-post`); + if (delayed && actor === "winner") { + yield* writeTextFile(`${directory}/post-ready`, ""); + if ((yield* waitForFile(`${directory}/release-post`, 45_000)).timeout) { + throw new Error("Timed out waiting to release the GitHub write."); + } + } + if (yield* exists(tracked)) { + throw new Error("GitHub received a duplicate review write."); + } + const saved = { + id: 42, + html_url: `https://github.com/${repository}/pull/7#issuecomment-42`, + issue_url: `https://api.github.com/repos/${repository}/issues/7`, + body, + }; + yield* writeTextFile(tracked, JSON.stringify(saved)); + lost = delayed; + return delayed ? reply("", 1) : reply("HTTP/2.0 201 Created\r\n\r\n" + JSON.stringify(saved)); + }, + }); +} diff --git a/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts b/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts index 2b826cf95..2361ec425 100644 --- a/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts +++ b/examples/adversarial-implementation/tests/acp-overlap/local-workers.ts @@ -1,17 +1,18 @@ import { createAcpRuntime, createAcpxProvider, createAcpxSessionStore } from "@executablemd/acp"; import { AgentProviders, registerAgentProvider } from "@executablemd/core"; import { Component, Plugin } from "@executablemd/core/api"; -import { API, command, env, readTextFile } from "@executablemd/runtime"; +import { API, command, cwd, env, readTextFile } from "@executablemd/runtime"; import { useTestAgentController } from "@executablemd/test-agent"; import { writeTextFile } from "@effectionx/fs"; import { ensure } from "effection"; +import { installGitHub } from "./github-comments.ts"; import { actorEnv, actorSchema, fableFailure, overlapEnv, waitForFile } from "./receipts.ts"; import type { PromptReceipt } from "./receipts.ts"; export default Plugin({ name: "delivery-acp-overlap", - *install(request) { - if (request.command !== "run") { + *install(invocation) { + if (invocation.command !== "run") { throw new Error("Delivery ACP workers require a run invocation."); } const directory = yield* env(overlapEnv); @@ -19,6 +20,7 @@ export default Plugin({ if (!directory?.startsWith("/")) { throw new Error("Delivery ACP workers require an absolute fixture directory."); } + yield* installGitHub(directory, yield* cwd(), actor); const receipts: PromptReceipt[] = []; yield* ensure(() => writeTextFile(`${directory}/${actor}-prompts.json`, JSON.stringify(receipts)), @@ -89,10 +91,10 @@ export default Plugin({ return [...worker, "--connect", scenario.route]; }, }, - createRuntime(options) { + createRuntime(runtimeOptions) { const pending = new Map(); return createAcpRuntime({ - ...options, + ...runtimeOptions, onAcpMessage(direction, message) { if ( direction === "outbound" && diff --git a/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md b/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md new file mode 100644 index 000000000..5efa82382 --- /dev/null +++ b/examples/adversarial-implementation/tests/components/Delivery/ReviewPublication.md @@ -0,0 +1,5 @@ +--- +props: {type: object, properties: {target: {type: object}, review: {type: object}, publish: {type: boolean}}, required: [target, review, publish], additionalProperties: false} +returns: {type: object} +--- + diff --git a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts index 988493a82..45525849f 100644 --- a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts +++ b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts @@ -1,4 +1,4 @@ -import { copyFile, ensureDir, exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { copyFile, ensureDir, exists, readTextFile, rm, writeTextFile } from "@effectionx/fs"; import { cwd } from "@executablemd/runtime"; import { runShell, shellQuote } from "@executablemd/test-support/launch"; import { ensure, race, scoped, spawn } from "effection"; @@ -20,8 +20,9 @@ export const props = { command: { type: "string" }, directory: { type: "string" }, revision: { type: "string" }, + scenario: { enum: ["pass", "changes_required", "lost"] }, }, - required: ["command", "directory", "revision"], + required: ["command", "directory", "revision", "scenario"], additionalProperties: false, }; @@ -29,6 +30,7 @@ const input = z.strictObject({ command: z.string(), directory: z.string().startsWith("/"), revision: z.string(), + scenario: z.enum(["pass", "changes_required", "lost"]), }); const production = new URL("../../../", import.meta.url); const fixtures = new URL("../../acp-overlap/", import.meta.url); @@ -45,6 +47,12 @@ function check(condition: boolean, message: string): asserts condition { const statusSchema = z.object({ status: z.string(), + nextAction: z.string(), + reviewPublication: z.object({ + state: z.string(), + revision: z.string().optional(), + url: z.string().optional(), + }), review: z.object({ attempt: z.string(), result: z.object({ @@ -66,16 +74,28 @@ const outcomeSchema = z.object({ }); export default function* ReviewOverlap(value: unknown) { - const { command, directory, revision } = input.parse(value); + const { command, directory, revision, scenario } = input.parse(value); const repository = yield* cwd(); const evidence = `${directory}/overlap`; const attempt = `${directory}/review-${revision}`; const replacement = `${attempt}/replacement`; const release = `${evidence}/release`; yield* ensureDir(evidence); - for (const file of ["Fable.md", "Astra.md", "Gate.md"]) { + for (const file of ["Fable.md", "Gate.md"]) { yield* copyFile(new URL(file, fixtures), `${evidence}/${file}`); } + const astra = yield* readTextFile(new URL("Astra.md", fixtures)); + yield* writeTextFile( + `${evidence}/Astra.md`, + scenario !== "changes_required" + ? astra + : astra + .replace('verdict:"pass"', 'verdict:"changes_required"') + .replace( + "findings:[]", + 'findings:[{blocking:true,description:"Queue drops a waiting message."}]', + ), + ); const args = [ fileURLToPath(new URL("delivery.md", production)), @@ -83,8 +103,6 @@ export default function* ReviewOverlap(value: unknown) { `${directory}/requirements.json`, "--props-number", "7", - "--props-action", - "review", "--include", fileURLToPath(new URL("components", fixtures)), "--include", @@ -98,8 +116,8 @@ export default function* ReviewOverlap(value: unknown) { .map(shellQuote) .join(" "); - const review = (actor: z.infer, timeout: number) => - runShell(`exec ${command} ${args}`, { + const review = (actor: z.infer, timeout: number, action = "review") => + runShell(`exec ${command} ${args} --props-action ${shellQuote(action)}`, { cwd: repository, env: { [overlapEnv]: evidence, [actorEnv]: actor }, inheritEnv: true, @@ -168,6 +186,9 @@ export default function* ReviewOverlap(value: unknown) { `Unexpected contender result (${contender.code}): ${contender.stderr} ${contender.stdout}`, ); + if (scenario === "lost") { + yield* writeTextFile(`${evidence}/stale`, ""); + } yield* writeTextFile(release, ""); released = true; return yield* first; @@ -185,15 +206,107 @@ export default function* ReviewOverlap(value: unknown) { check(winner.code === 0, `Winner exited ${winner.code}: ${winner.stderr} ${winner.stdout}`); const status = statusSchema.parse(JSON.parse(winner.stdout)); check( - status.status === "ready_for_merge" && + status.status === + (scenario === "pass" + ? "ready_for_merge" + : scenario === "changes_required" + ? "repair_required" + : "publication_pending") && status.review.attempt === replacement && status.review.result.revision === revision && - status.review.result.verdict === "pass" && + status.review.result.verdict === + (scenario === "changes_required" ? "changes_required" : "pass") && status.review.result.acceptance.length === 1 && status.review.result.acceptance[0] === "behavior", `Winner did not retain its exact-head Astra pass: ${JSON.stringify(status)}`, ); + if (scenario === "lost") { + check(!(yield* exists(`${evidence}/winner-posted`)), "A stale PR head received a POST."); + check( + status.nextAction === "publish_review", + "A missing publication must take precedence over merge readiness.", + ); + yield* rm(`${evidence}/stale`); + yield* rm(`${replacement}/receipt.json`); + yield* writeTextFile(`${evidence}/delay-post`, ""); + yield* scoped(function* () { + const first = yield* spawn(() => review("winner", 75_000, "reconcile")); + yield* ensure(function* () { + yield* writeTextFile(`${evidence}/release-post`, ""); + yield* first; + }); + function* exitedEarly(): Operation { + const result = yield* first; + throw new Error( + `Reconcile exited before comment publication (${result.code}): ${result.stderr} ${result.stdout}`, + ); + } + const ready = yield* race([waitForFile(`${evidence}/post-ready`, 35_000), exitedEarly()]); + check(!ready.timeout, "Reconciliation did not attempt to publish the joined result."); + const overlapping = yield* review("contender", 25_000, "publish-review"); + check(overlapping.code === 0, `Concurrent publication failed: ${overlapping.stderr}`); + const waiting = statusSchema.parse(JSON.parse(overlapping.stdout)); + check( + waiting.nextAction === "publish_review", + "Concurrent publish did not wait for readback.", + ); + check( + !(yield* exists(`${evidence}/contender-posted`)), + "Concurrent publication sent a second POST.", + ); + yield* writeTextFile(`${evidence}/release-post`, ""); + const result = yield* first; + check( + result.code === 0, + `Reconciliation after lost acknowledgement failed: ${result.stderr}`, + ); + const unconfirmed = statusSchema.parse(JSON.parse(result.stdout)); + check(unconfirmed.nextAction === "publish_review", "Lost acknowledgement granted readiness."); + }); + const report = yield* review("later", 25_000, "closeout"); + check( + report.code === 0 && + statusSchema.parse(JSON.parse(report.stdout)).nextAction === "publish_review", + "Closeout must leave unconfirmed publication pending.", + ); + check(!(yield* exists(`${evidence}/later-posted`)), "Closeout attempted a GitHub write."); + } + const published = yield* review("later", 25_000, "publish-review"); + check(published.code === 0, `Publication retry failed: ${published.stderr} ${published.stdout}`); + const verified = statusSchema.parse(JSON.parse(published.stdout)); + check( + verified.status === (scenario === "changes_required" ? "repair_required" : "ready_for_merge") && + verified.reviewPublication.state === "verified" && + verified.reviewPublication.revision === revision && + verified.reviewPublication.url === + "https://github.com/taras/executable.md/pull/7#issuecomment-42", + `Publication was not verified: ${JSON.stringify(verified)}`, + ); + const comment = z + .object({ body: z.string() }) + .parse(JSON.parse(yield* readTextFile(`${evidence}/comment.json`))); + check( + comment.body.startsWith(``) && + comment.body.includes("codex / gpt-6-astra") && + comment.body.includes("Reviewed the assigned behavior.") && + comment.body.includes( + scenario === "changes_required" ? "**CHANGES REQUIRED**" : "**PASS**", + ) && + (scenario !== "changes_required" || comment.body.includes("Queue drops a waiting message.")), + `The published comment does not describe the validated Astra review: ${comment.body}`, + ); + for (const actor of ["contender", "later"] as const) { + check(!(yield* exists(`${evidence}/${actor}-posted`)), `${actor} sent an extra POST.`); + check( + (yield* readPrompts(evidence, actor)).length === 0, + `${actor} contacted ACP during publication.`, + ); + } + if (scenario === "lost") { + check((yield* readPrompts(evidence, "winner")).length === 0, "Reconciliation contacted ACP."); + } + const later = yield* review("later", 25_000); const laterPrompts = yield* readPrompts(evidence, "later"); check( @@ -203,7 +316,7 @@ export default function* ReviewOverlap(value: unknown) { check( later.code === 1 && later.stderr.includes( - "The next action is await_authorized_merge; do not start another review.", + `The next action is ${scenario === "changes_required" ? "request_approved_repair" : "await_authorized_merge"}; do not start another review.`, ), `Unexpected later result (${later.code}): ${later.stderr} ${later.stdout}`, ); From f33ffc9ab5482df468922ef5143cf1eb70589a92 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sun, 15 Nov 2026 04:53:00 -0500 Subject: [PATCH 2/3] Read Delivery review comments through PullRequest.Comments --- examples/adversarial-implementation/README.md | 8 ++ ...{ReviewPublication.ts => ReviewComment.ts} | 92 +++++++++---------- .../components/Delivery/ReviewPublication.md | 26 ++++++ .../tests/acp-overlap/github-comments.ts | 45 ++++++--- .../tests/components/Fixture/ReviewOverlap.ts | 12 +++ 5 files changed, 122 insertions(+), 61 deletions(-) rename examples/adversarial-implementation/components/Delivery/{ReviewPublication.ts => ReviewComment.ts} (79%) create mode 100644 examples/adversarial-implementation/components/Delivery/ReviewPublication.md diff --git a/examples/adversarial-implementation/README.md b/examples/adversarial-implementation/README.md index 1733ff151..fb54fba8e 100644 --- a/examples/adversarial-implementation/README.md +++ b/examples/adversarial-implementation/README.md @@ -344,6 +344,14 @@ continues to hold. Status uses this same validation boundary to read receipts. `review` and `reconcile` publish each validated verdict as a comment on the current pull request, including findings that require repair. The comment names the exact reviewed commit and the reviewer who supplied the result. +Delivery uses `` to reconcile and verify the comment. With +the native GitHub provider, include the repository URL in the host's +`XMD_WORKFLOW_GITHUB_PULL_REQUESTS` allowlist. For a host that reads one repository: + +```sh +export XMD_WORKFLOW_GITHUB_PULL_REQUESTS='{"allowed":["https://github.com/owner/repository"]}' +``` + Once GitHub returns the matching comment, the workflow saves its URL with the reviewed SHA. A new head has its own comment. diff --git a/examples/adversarial-implementation/components/Delivery/ReviewPublication.ts b/examples/adversarial-implementation/components/Delivery/ReviewComment.ts similarity index 79% rename from examples/adversarial-implementation/components/Delivery/ReviewPublication.ts rename to examples/adversarial-implementation/components/Delivery/ReviewComment.ts index a424dc04d..ac9898bb3 100644 --- a/examples/adversarial-implementation/components/Delivery/ReviewPublication.ts +++ b/examples/adversarial-implementation/components/Delivery/ReviewComment.ts @@ -6,17 +6,16 @@ import { createDirectory, digest } from "../Ticket/native-node.ts"; import { clear, write } from "../Ticket/text-storage.ts"; import { reviewerSchema } from "./records.ts"; -export const description = - 'Verify a saved review comment or publish it from its validated receipt. Use .'; - export const props = { type: "object", properties: { target: { type: "object" }, review: { type: "object" }, publish: { type: "boolean" }, + allowPost: { type: "boolean" }, + comments: { type: "array" }, }, - required: ["target", "review", "publish"], + required: ["target", "review", "publish", "allowPost", "comments"], additionalProperties: false, }; @@ -41,22 +40,25 @@ const input = z.strictObject({ }), }), publish: z.boolean(), + allowPost: z.boolean(), + comments: z.array( + z.object({ + kind: z.enum(["conversation", "review"]), + id: z.string(), + url: z.string(), + body: z.string(), + }), + ), }); const pulls = z.object({ number: targetSchema.shape.number, base: z.object({ repo: z.object({ full_name: z.string() }) }), head: z.object({ ref: z.string(), sha }), }); -const comment = z.object({ - id: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), - issue_url: z.string(), - html_url: z.string(), - body: z.string(), -}); const saved = z.strictObject({ revision: sha, bodySha256: z.string().regex(/^[0-9a-f]{64}$/), - id: comment.shape.id, + id: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), url: z.string(), }); @@ -118,45 +120,37 @@ function* readTarget(target: Target) { ); } -function verifiedComment(target: Target, value: z.infer, body: string): Verified { +function findComment( + target: Target, + comments: z.infer["comments"], + marker: string, + body: string, +): Verified | null { + const matches = comments.filter( + (item) => item.kind === "conversation" && item.body.split("\n", 1)[0] === marker, + ); + if (matches.length > 1) { + throw new Error("Multiple GitHub comments claim the same review revision."); + } + const [value] = matches; + if (value === undefined) { + return null; + } const { repository, number, revision } = target; - const validUrls = ["issues", "pull"].map((kind) => - `https://github.com/${repository}/${kind}/${number}#issuecomment-${value.id}`.toLowerCase(), + const id = Number(value.id); + const urls = ["issues", "pull"].map((kind) => + `https://github.com/${repository}/${kind}/${number}#issuecomment-${id}`.toLowerCase(), ); if ( + !Number.isSafeInteger(id) || + id <= 0 || + value.id !== String(id) || value.body !== body || - value.issue_url.toLowerCase() !== - `https://api.github.com/repos/${repository}/issues/${number}`.toLowerCase() || - !validUrls.includes(value.html_url.toLowerCase()) + !urls.includes(value.url.toLowerCase()) ) { throw new Error("The GitHub comment does not match the saved review."); } - return { - revision, - bodySha256: digest(new TextEncoder().encode(body)), - id: value.id, - url: value.html_url, - }; -} - -function* findComment(target: Target, marker: string, body: string) { - const { exitCode, stdout } = yield* github( - "--paginate", - "--slurp", - `repos/${target.repository}/issues/${target.number}/comments?per_page=100`, - ); - if (exitCode !== 0) { - throw new Error("Could not read GitHub review comments."); - } - const matches = z - .array(z.array(comment)) - .parse(JSON.parse(stdout)) - .flat() - .filter((item) => item.body.split("\n", 1)[0] === marker); - if (matches.length > 1) { - throw new Error("Multiple GitHub comments claim the same review revision."); - } - return matches[0] === undefined ? null : verifiedComment(target, matches[0], body); + return { revision, bodySha256: digest(new TextEncoder().encode(body)), id, url: value.url }; } function* readSaved(directory: string) { @@ -206,8 +200,8 @@ function* post(target: Target, reservation: string, body: string): Operation.' +props: + type: object + properties: + target: {type: object} + review: {type: object} + publish: {type: boolean} + required: [target, review, publish] + additionalProperties: false +returns: {type: object} +--- + + + + + + + + + + + + + + diff --git a/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts index e3d769114..712f1e5f4 100644 --- a/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts +++ b/examples/adversarial-implementation/tests/acp-overlap/github-comments.ts @@ -1,4 +1,5 @@ import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; +import { PullRequestAPI } from "@executablemd/git/api"; import { API } from "@executablemd/runtime"; import { z } from "zod"; import { waitForFile } from "./receipts.ts"; @@ -6,6 +7,16 @@ import { waitForFile } from "./receipts.ts"; const repository = "taras/executable.md"; const pr = `repos/${repository}/pulls/7`; const comments = `repos/${repository}/issues/7/comments`; +const url = `https://github.com/${repository}/pull/7`; +const evidence = z.strictObject({ + kind: z.literal("conversation"), + id: z.string(), + url: z.string(), + author: z.string(), + body: z.string(), + createdAt: z.string(), + updatedAt: z.string(), +}); const reply = (stdout: string, exitCode = 0) => ({ stdout, stderr: "", exitCode }); export function* installGitHub(directory: string, checkout: string, actor: string) { @@ -14,6 +25,20 @@ export function* installGitHub(directory: string, checkout: string, actor: strin const payload = z.strictObject({ body: z.string() }); const tracked = `${directory}/comment.json`; + yield* PullRequestAPI.around({ + *read([target, options]) { + if (target !== url || options.kind !== "comments") { + throw new Error("Unexpected pull request evidence read."); + } + if (lost || (yield* exists(`${directory}/comments-unavailable`))) { + throw new Error("The pull request comments are unavailable."); + } + const items = (yield* exists(tracked)) + ? [evidence.parse(JSON.parse(yield* readTextFile(tracked)))] + : []; + return { kind: "comments", items }; + }, + }); yield* API.Process.around({ *exec([options], next) { const [tool, ...args] = options.command; @@ -37,13 +62,6 @@ export function* installGitHub(directory: string, checkout: string, actor: strin }), ); } - if (rest.join(" ") === `--paginate --slurp ${comments}?per_page=100`) { - if (lost) { - return reply("", 1); - } - const page = (yield* exists(tracked)) ? [JSON.parse(yield* readTextFile(tracked))] : []; - return reply(JSON.stringify([[], page])); - } if ( rest.length !== 6 || rest.slice(0, 5).join(" ") !== `--include --method POST ${comments} --input` @@ -62,12 +80,15 @@ export function* installGitHub(directory: string, checkout: string, actor: strin if (yield* exists(tracked)) { throw new Error("GitHub received a duplicate review write."); } - const saved = { - id: 42, - html_url: `https://github.com/${repository}/pull/7#issuecomment-42`, - issue_url: `https://api.github.com/repos/${repository}/issues/7`, + const saved = evidence.parse({ + kind: "conversation", + id: "42", + url: `${url}#issuecomment-42`, + author: "workflow", + createdAt: "2026-11-15T00:00:00Z", + updatedAt: "2026-11-15T00:00:00Z", body, - }; + }); yield* writeTextFile(tracked, JSON.stringify(saved)); lost = delayed; return delayed ? reply("", 1) : reply("HTTP/2.0 201 Created\r\n\r\n" + JSON.stringify(saved)); diff --git a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts index 45525849f..d583fffa8 100644 --- a/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts +++ b/examples/adversarial-implementation/tests/components/Fixture/ReviewOverlap.ts @@ -228,6 +228,18 @@ export default function* ReviewOverlap(value: unknown) { "A missing publication must take precedence over merge readiness.", ); yield* rm(`${evidence}/stale`); + yield* writeTextFile(`${evidence}/comments-unavailable`, ""); + const unavailable = yield* review("later", 25_000, "publish-review"); + check( + unavailable.code === 0 && + statusSchema.parse(JSON.parse(unavailable.stdout)).nextAction === "publish_review", + "Unavailable native comments granted readiness.", + ); + check( + !(yield* exists(`${evidence}/later-posted`)), + "Unavailable native comments allowed a POST.", + ); + yield* rm(`${evidence}/comments-unavailable`); yield* rm(`${replacement}/receipt.json`); yield* writeTextFile(`${evidence}/delay-post`, ""); yield* scoped(function* () { From c4a4fad216b44d4fa3901f97172ffde6fdaf1139 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sun, 15 Nov 2026 21:18:27 -0500 Subject: [PATCH 3/3] Include review publication in the expected runtime corpus --- scripts/tests/runtime-exclusions.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/runtime-exclusions.test.ts b/scripts/tests/runtime-exclusions.test.ts index 125123dd9..360dddc84 100644 --- a/scripts/tests/runtime-exclusions.test.ts +++ b/scripts/tests/runtime-exclusions.test.ts @@ -179,6 +179,7 @@ describe("the applicable corpus", () => { "ArtifactReturnRecovery", "DecisionResolution", "Delivery", + "DeliveryPublication", "Friction", "Implementation", "LaunchPolicy",