diff --git a/packages/agent/host.ts b/packages/agent/host.ts index a31cff18e..d7dbf53e4 100644 --- a/packages/agent/host.ts +++ b/packages/agent/host.ts @@ -21,8 +21,8 @@ * * Kept apart from `./api` for the reason core keeps its own `./host`: nothing a * document, a component or a middleware package reaches by importing the - * consumer API can install a provider for a region it did not author or decide - * where a prompt is journaled. + * consumer API can install a provider for a region it did not author, decide + * where a prompt is journaled, or answer a withheld reply. */ // Agent components — // over a provider-factory seam. @@ -105,3 +105,31 @@ export type { */ export { AGENT_PROMPT, parsePromptRecord } from "./src/journal.ts"; export type { PromptRecord } from "./src/journal.ts"; + +/** + * A withheld reply a local person accepted, and the one rule that joins it to + * its turn. + * + * Readers of a held conversation's history — replay, a Sidekick's chat and + * its passive projections — read a withheld turn followed by its acceptance as + * that turn's successful reply. They read it through this reconciler rather + * than their own, so they agree on which acceptances count. + */ +export { acceptedReplies, AGENT_PROMPT_REPLY, parsePromptReply } from "./src/reply-acceptance.ts"; +export type { PromptReplyRecord } from "./src/reply-acceptance.ts"; + +/** + * The trusted foreground question about a withheld reply. + * + * Installed by a host with a person in front of it, beside its prompt + * publisher, as one of a held conversation's installations. A host that + * installs none keeps every withheld reply withheld. See + * `src/reply-review.ts`. + */ +export { useAgentReplyReview } from "./src/reply-review.ts"; +export type { + AgentReplyDecision, + AgentReplyFinding, + AgentReplyHold, + AgentReplyReviewer, +} from "./src/reply-review.ts"; diff --git a/packages/agent/src/conversation.ts b/packages/agent/src/conversation.ts index dac6999b7..07e442180 100644 --- a/packages/agent/src/conversation.ts +++ b/packages/agent/src/conversation.ts @@ -90,13 +90,29 @@ import { secretPolicy, } from "@executablemd/core"; import type { ComponentInvocation, Json } from "@executablemd/core"; -import { durabilityFailure, executeInstalled } from "@executablemd/core/host"; +import { + acceptanceGrant, + approvedMatches, + captureReplyTextCheck, + carriedCopies, + durabilityFailure, + evaluationGrant, + executeInstalledBound, + feedbackGrant, + grantReplyCopies, + locateReplySecrets, +} from "@executablemd/core/host"; import type { + AcceptedRootReply, + ApprovedMatch, + BoundRootEvaluator, ExecutionInstallation, - HostExecution, + FeedbackSlice, GeneratedXmdRequest, GeneratedXmdRootResult, IdentityClaimant, + ReplyTextCheck, + SecretSpan, } from "@executablemd/core/host"; import { AGENT_ORIGIN } from "./origin.ts"; import { AgentConversationProvider } from "./conversation-provider.ts"; @@ -118,6 +134,14 @@ import { promptFailureFromRecord, } from "./journal.ts"; import type { PromptRecord } from "./journal.ts"; +import { + acceptedReplies, + AGENT_PROMPT_REPLY, + persistPromptReply, + replayPromptReply, +} from "./reply-acceptance.ts"; +import { askReplyReview } from "./reply-review.ts"; +import type { AgentReplyHold } from "./reply-review.ts"; import { freezeJson, parseProviderConfiguration } from "./provider-configuration.ts"; import type { AdmitProviderConfiguration, @@ -151,7 +175,10 @@ export interface AgentConversation { input: string, configuration?: SessionConfiguration, images?: readonly PromptImage[], + automaticFeedback?: boolean, ): Operation>; + /** Test retained feedback with this conversation's live disclosure policy. */ + checkSidekickText(text: string): Operation>; /** Inspect the selected agent's advertised model and effort choices. */ options(request?: AgentOptionsRequest): Operation>; /** @@ -170,7 +197,10 @@ export interface AgentConversation { * conversation's turns and its evaluations are separate durable sequences on * one history. */ - evaluateGeneratedXmdRoot(request: GeneratedXmdRequest): Operation>; + evaluateGeneratedXmdRoot( + request: GeneratedXmdRequest, + accepted?: AgentReplyHold, + ): Operation>; } /** @@ -222,6 +252,7 @@ interface PromptJob { readonly input: string; readonly configuration?: SessionConfiguration; readonly images?: readonly PromptImage[]; + readonly automaticFeedback: boolean; readonly answer: Resolvers>; readonly cancelled: Resolvers; readonly joined: Resolvers; @@ -238,6 +269,7 @@ interface OptionsJob { interface EvaluateJob { readonly kind: "evaluate"; readonly request: GeneratedXmdRequest; + readonly accepted?: AgentReplyHold; readonly answer: Resolvers; readonly cancelled: Resolvers; readonly joined: Resolvers; @@ -305,8 +337,36 @@ interface Live { * ending names one outcome. */ fatal?: Error; + checkReplyText?: ReplyTextCheck; + /** + * What an accepted reply may still carry, for the very next job only. + * + * Set when a person accepts a withheld reply, and taken by the next job the + * element serves whatever that job is: the evaluation of exactly that source + * uses it, and that evaluation's own selected answer may leave one for the + * next turn to carry. Any other job retires it. Never journaled and never + * rebuilt on reopening. + */ + accepted?: Accepted; } +/** The live provenance of one accepted reply, at the step it has reached. */ +type Accepted = + | { + readonly kind: "evaluation"; + readonly text: string; + readonly approved: readonly ApprovedMatch[]; + readonly hold: AgentReplyHold; + } + | { + readonly kind: "feedback"; + readonly select: ReturnType["select"]; + /** What the evaluation's own question selected, read when the turn starts. */ + readonly carried: ReturnType["carried"]; + readonly hold: AgentReplyHold; + registered?: { readonly text: string; readonly slices: readonly FeedbackSlice[] }; + }; + /** The native conversation identity one turn saw the provider name. */ interface Observed { used?: boolean; @@ -318,8 +378,13 @@ interface Observed { /** What a history already holds about the conversation it belongs to. */ interface ConversationHistory { readonly used: boolean; - /** The durable names of its turns, in the order the history holds them. */ - readonly turns: readonly string[]; + /** + * Its turns and their acceptances, in the order the history holds them. + * + * An acceptance is offered again directly after the withheld turn it + * amends, which is where it was written. + */ + readonly replays: readonly { readonly type: string; readonly name: string }[]; /** * What this conversation's sessions are built under, once anything says. * @@ -458,10 +523,22 @@ function handle( ); return { + *checkSidekickText(text): Operation> { + if (!live.open || live.checkReplyText === undefined) { + return Err(ended()); + } + try { + yield* live.checkReplyText(text, []); + return live.open ? Ok(undefined) : Err(ended()); + } catch (error) { + return Err(error instanceof Error ? error : new AgentConversationError(String(error))); + } + }, prompt( input: string, configuration?: SessionConfiguration, images?: readonly PromptImage[], + automaticFeedback = false, ): Operation> { return scoped(function* () { const admitted = admit(); @@ -499,6 +576,7 @@ function handle( jobs.add({ kind: "prompt", input, + automaticFeedback, ...(configuration === undefined ? {} : { configuration }), ...(images === undefined ? {} @@ -508,7 +586,7 @@ function handle( joined, }); sent = true; - const settled = yield* awaiting(answer.operation, () => Err(ended())); + const settled = yield* awaiting(answer.operation, () => Err(live.fatal ?? ended())); answered = true; return settled; }); @@ -552,6 +630,7 @@ function handle( }, evaluateGeneratedXmdRoot( request: GeneratedXmdRequest, + accepted?: AgentReplyHold, ): Operation> { return scoped(function* () { // The same admission a turn passes, because it is the same mailbox: an @@ -579,7 +658,7 @@ function handle( live.busy = false; } }); - jobs.add({ kind: "evaluate", request, answer, cancelled, joined }); + jobs.add({ kind: "evaluate", request, accepted, answer, cancelled, joined }); sent = true; const settled = yield* awaiting(answer.operation, () => ({ value: Err(ended()), @@ -664,18 +743,22 @@ function* serve( ], }; try { - const execution = yield* executeInstalled( + const execution = yield* executeInstalledBound( { ...retainedSource(CONVERSATION_ROOT_PATH, CONVERSATION_ROOT_SOURCE), stream: request.history, }, [...request.installations, installation], + undefined, + (evaluate) => bridge.resolve(evaluate), ); // Before the completion is waited on, and after the handle exists: the // element is suspended on this, and nothing it does decides whether this // resolves. - bridge.resolve(evaluatorOf(execution)); const settled = yield* execution; + if (!settled.ok && durabilityFailure(settled.error) !== undefined) { + live.fatal ??= settled.error; + } // Reached only when the execution ended on its own terms. For a // conversation that means it ended before the element could serve // anything: the ordinary ending is a halt, which never arrives here. @@ -749,8 +832,12 @@ function* held( // Every turn the history already holds, offered again in its own order so // this run is aligned with the journal it is continuing. A retained turn // replays from its own record and reaches no provider. - for (const name of history.turns) { - yield* replayTurn(name); + for (const retained of history.replays) { + if (retained.type === AGENT_PROMPT_REPLY) { + yield* replayPromptReply(retained.name); + } else { + yield* replayTurn(retained.name); + } } let order = history.next; // What the history already established, carried forward so a turn in this @@ -762,6 +849,8 @@ function* held( // conversation is announced: a handle that reported itself ready and then // refused to evaluate would be ready for two of the three things it offers. const evaluate = yield* bridge.operation; + const checkReplyText = yield* captureReplyTextCheck(); + live.checkReplyText = checkReplyText; announce(Ok(undefined)); while (true) { const next = yield* jobs.next(); @@ -769,14 +858,32 @@ function* held( break; } const job = next.value; + // Taken by this job whatever it is: an accepted reply's provenance belongs + // to the next step of that same exchange, and anything else in between + // retires it. + const offered = live.accepted; + live.accepted = undefined; if (job.kind === "options") { yield* inspect(provider, request, job); continue; } if (job.kind === "evaluate") { - yield* evaluated(evaluate, live, job); + if ( + offered?.kind === "evaluation" && + job.accepted === offered.hold && + job.request.source === offered.text + ) { + yield* evaluatedAccepted(evaluate, live, job, offered); + } else { + yield* evaluated(evaluate, live, job); + } continue; } + const carried = offered?.kind === "feedback" ? offered.carried() : []; + const carrying = + offered?.kind === "feedback" && offered.registered?.text === job.input && carried.length > 0 + ? { carried, slices: offered.registered.slices } + : undefined; const observed: Observed = {}; yield* deliver( provider, @@ -789,7 +896,13 @@ function* held( admitting(active), used, sessionFacts(active, established, placement), + live, + carrying, + checkReplyText, ); + if (live.fatal !== undefined) { + yield* suspend(); + } // Established by the first turn the provider reported one for, and never // replaced afterwards: a turn naming a different one was refused before it // could record anything. @@ -1024,11 +1137,29 @@ function deliver( admit: AdmitProviderConfiguration, used: boolean, facts: SessionFacts, + live: Live, + carrying: Carrying | undefined, + checkReplyText: ReplyTextCheck, ): Operation { // Scoped, so the turn cannot outlive the call that asked for it however this // returns: a turn still running would hold its durable operation open, and // the next turn would be a second one open on the same sequence. return scoped(function* () { + const copies = + carrying === undefined ? [] : carriedCopies(job.input, carrying.slices, carrying.carried); + const prepare = (delegated: string) => (delegated.startsWith(job.input) ? copies : []); + // The one turn an accepted answer may be carried by, for exactly as long as + // this turn runs: retired when it commits, fails or is cancelled. + if (carrying !== undefined) { + const retire = yield* grantReplyCopies( + feedbackGrant({ + type: AGENT_PROMPT, + name: turnName(order), + prepare, + }), + ); + yield* ensure(retire); + } // Registered *before* the turn is spawned, which is what makes it the last // thing to unwind: cleanups registered later run first, so the turn — and // every cleanup the provider registered inside it — has finished by the @@ -1049,6 +1180,10 @@ function deliver( admit, used, facts, + live, + job.automaticFeedback || carrying !== undefined + ? (delegated) => checkReplyText(delegated, prepare(delegated)) + : undefined, ), ); const outcome = yield* race([ @@ -1111,6 +1246,8 @@ function conversationTurn( admit: AdmitProviderConfiguration, used: boolean, facts: SessionFacts, + live: Live, + checkInput: ((text: string) => Operation) | undefined, ): Operation { return (function* (): Operation { try { @@ -1180,22 +1317,32 @@ function conversationTurn( // by this conversation directly. const sequence = yield* AgentInternal.operations.nextPromptSequence(); const carried: Carried = {}; + // The exact completed reply a withheld record left out, and where the + // scanner found what it found — held in this frame and nowhere else, + // and only when every finding can be placed exactly. + let reviewable: + | { readonly text: string; readonly spans: readonly SecretSpan[] } + | undefined; const record = yield* persistPrompt( - { name: turnName(order), input: job.input }, + { name: turnName(order), input: checkInput === undefined ? job.input : "" }, // The composition this conversation installed runs above the turn's // own terminal, so the text it finally delegates is settled inside // the live runner. `retain` is the journal's own; nothing the // composition can reach carries it. function* (retain): Operation { - return yield* withholdFlaggedReply( - yield* runPrompt(job.input, options, sequence, false, carried, { - provider, - agent, - session, - retain, - } satisfies DirectPromptTurn), - established, - ); + const ran = yield* runPrompt(job.input, options, sequence, false, carried, { + provider, + agent, + session, + retain, + checkInput, + } satisfies DirectPromptTurn); + const kept = yield* withholdFlaggedReply(ran, established); + if (kept.withheld === true) { + const spans = yield* locateReplySecrets(ran.text); + reviewable = spans === undefined ? undefined : { text: ran.text, spans }; + } + return kept; }, () => carried.association, () => carried.begun, @@ -1228,7 +1375,19 @@ function conversationTurn( observed.native = record.agentSessionId; } if (record.withheld === true) { - job.answer.resolve(Err(new AgentReplyWithheldError())); + // Already a completion the history holds, and the publication that + // committed it has been released. Only now may anybody be asked, and + // whatever happens to the asking — no reviewer, a close, a Stop, a + // process that ends — this record is the answer that stands. + const pending = reviewable; + reviewable = undefined; + const accepted = + pending === undefined + ? undefined + : yield* reviewWithheld(pending, carried.begun, turnName(order), live); + job.answer.resolve( + accepted === undefined ? Err(new AgentReplyWithheldError()) : Ok(accepted), + ); return; } const failure = promptFailureFromRecord(record); @@ -1240,8 +1399,12 @@ function conversationTurn( // nobody asked for — is this turn's answer rather than the end of the // conversation: the host may ask again. A durability failure is not, and // it is the one thing that leaves here as itself. - if (durabilityFailure(error) !== undefined) { - throw error; + const fatal = durabilityFailure(error); + if (fatal !== undefined) { + live.fatal ??= fatal; + live.open = false; + job.answer.resolve(Err(live.fatal)); + return; } job.answer.resolve( Err(error instanceof Error ? error : new AgentConversationError(String(error))), @@ -1250,6 +1413,115 @@ function conversationTurn( })(); } +/** What one turn may carry of an accepted reply's selected answer. */ +interface Carrying { + readonly carried: ReturnType["carried"]>; + readonly slices: readonly FeedbackSlice[]; +} + +/** + * Ask the host's local reviewer about one withheld reply, and keep it if the + * person accepts. + * + * Asked from the turn's own frame, after the withheld completion committed, + * so the reply has never left this frame and the turn's caller is still + * waiting on this one continuation. An acceptance is saved before it is + * answered: the exact text, under its own record directly after the turn, + * through a grant that speaks for that one record. A failed append answers + * nothing accepted — it leaves this frame as the failure it is, and the + * withheld record before it stands. + */ +function* reviewWithheld( + pending: { readonly text: string; readonly spans: readonly SecretSpan[] }, + begun: unknown, + name: string, + live: Live, +): Operation { + const hold: { current?: AgentReplyHold } = {}; + const reviewed = yield* askReplyReview({ + text: pending.text, + spans: pending.spans, + begun, + selectedElicitation(selection) { + const current = live.accepted; + if (current?.kind === "feedback" && current.hold === hold.current) { + current.select(selection); + } + }, + carry(text, slices) { + const current = live.accepted; + if ( + current?.kind !== "feedback" || + current.hold !== hold.current || + current.registered !== undefined || + new Set(slices.map((slice) => `${slice.coroutineId}\0${slice.index}`)).size !== + slices.length || + !slices.every( + (slice) => + Number.isInteger(slice.start) && + Number.isInteger(slice.end) && + slice.start >= 0 && + slice.end > slice.start && + slice.end <= text.length, + ) + ) { + return false; + } + current.registered = Object.freeze({ + text, + slices: Object.freeze(slices.map((slice) => Object.freeze({ ...slice }))), + }); + return true; + }, + }); + hold.current = reviewed.hold; + if (reviewed.decision !== "accept") { + return undefined; + } + const approved = approvedMatches(pending.text, pending.spans); + const retire = yield* grantReplyCopies( + acceptanceGrant({ type: AGENT_PROMPT_REPLY, name, text: pending.text, approved }), + ); + try { + yield* persistPromptReply(name, pending.text, approved); + } finally { + retire(); + } + live.accepted = { kind: "evaluation", text: pending.text, approved, hold: reviewed.hold }; + return pending.text; +} + +/** + * Evaluate exactly the accepted source, with its copies granted for this one + * evaluation. + * + * The grant binds to the evaluation's own admission and coroutine and is + * retired when the evaluation joins. What its question selected from the + * accepted choices is left for the very next turn, and only that. + */ +function* evaluatedAccepted( + evaluate: BoundRootEvaluator, + live: Live, + job: EvaluateJob, + offered: Extract, +): Operation { + const grant = evaluationGrant({ + id: job.request.id, + source: offered.text, + approved: offered.approved, + }); + // Left for the next job before this one runs, so a host that registers its + // feedback the moment its evaluation answers finds it. What it can carry is + // read when that next turn starts, after this evaluation has joined. + live.accepted = { + kind: "feedback", + carried: grant.carried, + select: grant.select, + hold: offered.hold, + }; + yield* evaluated(evaluate, live, job, { grant, approved: offered.approved }); +} + /** * A completed turn's reply was withheld in full by secret detection. * @@ -1310,20 +1582,6 @@ function* withholdFlaggedReply( return { ...record, text: "", withheld: true }; } -/** The operation a trusted host handle exposes to evaluate one generated root. */ -type BoundRootEvaluator = HostExecution["evaluateGeneratedXmdRoot"]; - -/** - * The bound evaluator on one execution's handle, read once. - * - * Read off here rather than kept as the handle, so what crosses the bridge is - * the operation alone: the element never holds the execution it is running - * inside, and cannot observe its output or wait on its completion. - */ -function evaluatorOf(execution: HostExecution): BoundRootEvaluator { - return (request) => execution.evaluateGeneratedXmdRoot(request); -} - /** * Evaluate one generated root, and let the caller cancel it. * @@ -1333,7 +1591,12 @@ function evaluatorOf(execution: HostExecution): BoundRootEvaluator { * for has. The evaluation's own sequence belongs to the execution rather than * to this element, so what this adds is admission and joining and nothing else. */ -function evaluated(evaluate: BoundRootEvaluator, live: Live, job: EvaluateJob): Operation { +function evaluated( + evaluate: BoundRootEvaluator, + live: Live, + job: EvaluateJob, + accepted?: AcceptedRootReply, +): Operation { return scoped(function* () { yield* ensure(() => { job.joined.resolve(); @@ -1341,7 +1604,7 @@ function evaluated(evaluate: BoundRootEvaluator, live: Live, job: EvaluateJob): const served = yield* race([ (function* (): Operation { try { - return { kind: "settled", settled: { value: yield* evaluate(job.request) } }; + return { kind: "settled", settled: { value: yield* evaluate(job.request, accepted) } }; } catch (error) { // Answered *here*, at the throw, rather than after this race has // unwound. A failure of the run raised by this execution's own @@ -1437,8 +1700,14 @@ function* openable(request: AgentConversationRequest): Operation(); + // An acceptance is joined to its turn by the one rule every reader shares, + // and a history that rule cannot read is refused before anything opens. + const accepted = acceptedReplies(events); + if (!accepted.ok) { + return Err(new AgentConversationError(accepted.error.message)); + } let highest = 0; let configuration: DurableJson | undefined; let used = false; @@ -1465,6 +1734,10 @@ function* openable(request: AgentConversationRequest): Operation Operation; } /** What a held turn's own terminal refuses a rerouted call with. */ @@ -673,6 +674,19 @@ function* installHeldPromptTerminal( // The text this turn actually delegates, fixed in the journal's // own pending description and then announced to the host under // exactly that text, both before the provider is reached. + if (direct.checkInput !== undefined) { + try { + yield* direct.checkInput(content); + } catch (error) { + checked.failure = new HeldPromptInputRefusal( + "Secret detection withheld automatic feedback.", + { + cause: error, + }, + ); + throw checked.failure; + } + } direct.retain(content); const publisher = yield* AgentInternal.operations.promptPublisher; if (publisher?.begin !== undefined) { @@ -1000,7 +1014,7 @@ export function* runPrompt( cause: initiating, }); } - if (error instanceof HostObservationFailure) { + if (error instanceof HostObservationFailure || error instanceof HeldPromptInputRefusal) { // Not this turn's failure to describe. The provider's own cleanup has // already run — the scope above owns it — and this travels on to the // durability failure a host that cannot be trusted with its turn gets, @@ -1011,6 +1025,9 @@ export function* runPrompt( consumed.failure = serializePromptFailure(error); } + if (held?.failure instanceof HeldPromptInputRefusal) { + throw held.failure; + } const status = consumed.status ?? "failed"; let failure = consumed.failure; if (status !== "completed" && failure === undefined) { diff --git a/packages/agent/src/journal.ts b/packages/agent/src/journal.ts index 72b8c0e08..676a02d0f 100644 --- a/packages/agent/src/journal.ts +++ b/packages/agent/src/journal.ts @@ -78,7 +78,7 @@ import type { AgentPromptCheckpoint } from "./checkpoint.ts"; import { AgentPromptError, parsePromptFailure } from "./errors.ts"; import type { SerializedPromptFailure } from "./errors.ts"; import { AgentInternal } from "./internal.ts"; -import { HostObservationFailure } from "./publication.ts"; +import { HeldPromptInputRefusal, HostObservationFailure } from "./publication.ts"; import type { AgentPromptAssociation, AgentPromptHandle } from "./publication.ts"; import { canonicalUsage, @@ -593,6 +593,9 @@ function promptPublication( try { result = { status: "ok", value: yield* execute() }; } catch (error) { + if (error instanceof HeldPromptInputRefusal) { + throw error; + } const fatal = durabilityFailure(error) ?? filesFatalFailure(error); if (fatal !== undefined || hasInvocationTeardown(error)) { throw activateFailure(fatal ?? error); diff --git a/packages/agent/src/publication.ts b/packages/agent/src/publication.ts index 4c9d9edaf..30fba9147 100644 --- a/packages/agent/src/publication.ts +++ b/packages/agent/src/publication.ts @@ -204,3 +204,6 @@ export class HostObservationFailure extends Error { super("an installed prompt publisher failed while observing this turn"); } } + +/** The composed input was refused before publication and before delegation. */ +export class HeldPromptInputRefusal extends Error {} diff --git a/packages/agent/src/reply-acceptance.ts b/packages/agent/src/reply-acceptance.ts new file mode 100644 index 000000000..08b756352 --- /dev/null +++ b/packages/agent/src/reply-acceptance.ts @@ -0,0 +1,200 @@ +/** + * The canonical record that a withheld reply was accepted, and how every + * reader joins it to its turn. + * + * A held conversation that withheld a completed reply first journals that + * withheld completion, unchanged. When the person in front of the host + * accepts the reply, a second, distinct record follows it in the same root + * sequence: `agent_prompt_reply`, under the same durable name as the turn it + * amends, holding the exact original text and an inert audit of the rule, + * message and range each approved match had. The first record is never + * rewritten. It consumes no turn ordinal and no feedback slot: it is the same + * turn, now with its reply. + * + * One reconciler reads the pair for every consumer, so replay, the Sidekick's + * readers and its projections agree on one rule. An acceptance counts only + * directly after the withheld completion of the turn it names, once. Anything + * else — one with no turn before it, a second one, one after another turn, + * one after a turn that was not withheld, one whose record does not read — is + * a history that cannot say what its turn answered, and it is refused rather + * than resolved in favour of a later candidate. + */ + +import { Err, Ok } from "effection"; +import type { Operation, Result } from "effection"; +import { createDurableOperation } from "@executablemd/durable-streams"; +import type { DurableEvent, Json } from "@executablemd/durable-streams"; +import { isJsonObject } from "@executablemd/core"; +import { AGENT_PROMPT, parsePromptRecord } from "./journal.ts"; +import type { ApprovedMatch } from "@executablemd/core/host"; + +/** The durable effect an accepted reply is retained under. */ +export const AGENT_PROMPT_REPLY = "agent_prompt_reply"; + +const VERSION = 1; + +/** One accepted reply, as its record holds it. */ +export interface PromptReplyRecord { + /** The exact text the provider completed the turn with. */ + readonly text: string; + /** Where each approved match was, by rule and range. Never the matched text. */ + readonly approved: readonly { + readonly ruleId: string; + readonly messageId: string; + readonly start: number; + readonly end: number; + }[]; +} + +/** Why an acceptance in a history cannot be joined to its turn. */ +export class PromptReplyHistoryError extends Error { + override name = "PromptReplyHistoryError"; +} + +/** Read one acceptance record, or nothing when it is not exactly one. */ +export function parsePromptReply(value: Json | undefined): PromptReplyRecord | undefined { + if (!isJsonObject(value) || value.version !== VERSION || typeof value.text !== "string") { + return undefined; + } + const keys = Object.keys(value).sort().join(","); + if (keys !== "approved,text,version" || !Array.isArray(value.approved)) { + return undefined; + } + const approved: PromptReplyRecord["approved"][number][] = []; + for (const one of value.approved) { + if ( + !isJsonObject(one) || + Object.keys(one).sort().join(",") !== "end,messageId,ruleId,start" || + typeof one.ruleId !== "string" || + typeof one.messageId !== "string" || + typeof one.start !== "number" || + typeof one.end !== "number" || + !Number.isInteger(one.start) || + !Number.isInteger(one.end) || + one.start < 0 || + one.end <= one.start || + one.end > value.text.length + ) { + return undefined; + } + approved.push( + Object.freeze({ + ruleId: one.ruleId, + messageId: one.messageId, + start: one.start, + end: one.end, + }), + ); + } + if (approved.length === 0) { + return undefined; + } + return Object.freeze({ text: value.text, approved: Object.freeze(approved) }); +} + +/** + * Append the acceptance of one withheld turn. + * + * Its own durable operation, in the turn's own frame, so it lands in the root + * sequence directly after the turn it amends. The text is saved before + * anything can show it as accepted or run it. + */ +export function* persistPromptReply( + name: string, + text: string, + approved: readonly ApprovedMatch[], +): Operation { + const record: Json = { + version: VERSION, + text, + approved: approved.map((one) => ({ + ruleId: one.ruleId, + messageId: one.messageId, + start: one.start, + end: one.end, + })), + }; + yield createDurableOperation( + { type: AGENT_PROMPT_REPLY, name, input: name }, + // deno-lint-ignore require-yield + function* (): Operation { + return record; + }, + ); +} + +/** + * One retained acceptance, offered again so replay stays aligned. + * + * Its live body refuses: reaching it means the record is not there after all, + * and an acceptance is never made again without the person who made it. + */ +export function* replayPromptReply(name: string): Operation { + yield createDurableOperation( + { type: AGENT_PROMPT_REPLY, name, input: name }, + // deno-lint-ignore require-yield + function* (): Operation { + throw new PromptReplyHistoryError( + `the conversation's retained acceptance for "${name}" has no recorded outcome, so it ` + + "cannot be read back and is not made again", + ); + }, + ); +} + +/** + * The accepted text of each turn in one conversation's root sequence. + * + * `events` are that conversation's canonical events in history order. Only + * root yields take part; a generated evaluation's sequence is its own. + */ +export function acceptedReplies( + events: readonly DurableEvent[], +): Result> { + const accepted = new Map(); + let previous: DurableEvent | undefined; + for (const event of events) { + if (event.type !== "yield") { + continue; + } + if (event.description.type === AGENT_PROMPT_REPLY) { + const name = event.description.name; + const refused = (why: string) => + Err>( + new PromptReplyHistoryError( + `this conversation's acceptance for "${name}" ${why}, so what that turn answered ` + + "cannot be read from its history", + ), + ); + if (event.coroutineId !== "root" || event.description.input !== name) { + return refused("has a different canonical association"); + } + if (event.result.status !== "ok") { + return refused("recorded no outcome"); + } + const reply = parsePromptReply(event.result.value); + if (reply === undefined) { + return refused("does not read as an acceptance"); + } + if (accepted.has(name)) { + return refused("is recorded more than once"); + } + if ( + previous?.type !== "yield" || + previous.description.type !== AGENT_PROMPT || + previous.description.name !== name || + previous.result.status !== "ok" + ) { + return refused("does not directly follow that turn"); + } + if (parsePromptRecord(previous.result.value)?.withheld !== true) { + return refused("follows a turn whose reply was not withheld"); + } + accepted.set(name, reply); + } + if (event.coroutineId === "root") { + previous = event; + } + } + return Ok(accepted); +} diff --git a/packages/agent/src/reply-review.ts b/packages/agent/src/reply-review.ts new file mode 100644 index 000000000..8c3cd500e --- /dev/null +++ b/packages/agent/src/reply-review.ts @@ -0,0 +1,232 @@ +/** + * Asking a trusted foreground host whether to keep one withheld reply. + * + * A held conversation whose completed reply secret detection withheld has + * already done the safe thing: the turn is journaled as a withheld completion + * and that record stands whatever happens next. Only then, outside any + * publication transaction, does the turn's own task ask this operation. A host + * with nobody in front of it installs nothing, and the built-in terminal + * answers for it: withhold. + * + * The question is a contextual operation, not a callback on a request. A + * trusted host installs middleware for it beside its publisher; the turn calls + * the chain with a hold Core made for this one invocation; and a decision + * counts only when the middleware hands it to the terminal it was given, + * together with that same hold, while the turn is still waiting on it. What a + * handler returns is never read, a hold kept past its turn settles nothing, a + * second settlement is ignored, and no request property, attached client, + * authored `` or replayed record reaches this terminal at all. + * + * What the hold shows is for the local screen and nothing else: the rule, + * where in the reply it matched, a masked excerpt, and — only while the turn + * is waiting — the matched text itself, for a person who explicitly asks to + * see it. None of it is journaled, rendered by Core or attached to an error. + */ + +import { createApi } from "@effectionx/context-api"; +import type { Api } from "@effectionx/context-api"; +import { ensure } from "effection"; +import type { Operation } from "effection"; +import type { AgentPromptHandle } from "./publication.ts"; +import type { ElicitChoiceSelection, FeedbackSlice, SecretSpan } from "@executablemd/core/host"; + +/** One finding in a withheld reply, as a local reviewer may show it. */ +export interface AgentReplyFinding { + /** The rule that matched. */ + readonly ruleId: string; + /** The rule's message identifier. */ + readonly messageId: string; + /** Where in the reply the match begins: one-based line and column. */ + readonly line: number; + readonly column: number; + /** + * A masked excerpt: the text around the match on its own line, with the + * match itself replaced. Control characters are already replaced, so the + * excerpt is safe to draw as text — and it still carries reply text nobody + * has reviewed, which is why it is shown on the local screen only. + */ + readonly excerpt: { readonly before: string; readonly masked: string; readonly after: string }; +} + +/** What a host decides about one withheld reply. */ +export type AgentReplyDecision = "accept" | "withhold"; + +/** + * One withheld reply waiting for a local decision. + * + * Made by Core for one invocation of the review and good for that invocation + * only. Holding one grants nothing: it is settled through the terminal the + * reviewer was handed, and only while the turn is still waiting. + */ +export interface AgentReplyHold { + /** What this host's own publisher `begin()` returned for the turn. */ + readonly begun: AgentPromptHandle; + readonly findings: readonly AgentReplyFinding[]; + /** + * The matched text of one finding, for an explicit local Reveal. + * + * Answers nothing once the turn has stopped waiting. A reveal changes what + * the local screen draws and nothing else: it is not a decision. + */ + reveal(finding: number): string | undefined; + /** + * After an accepted reply's evaluation, say where the next turn's text + * carries the answer that evaluation's own question selected. + * + * `text` is exactly what the host hands that turn, and `slices` are where + * in it the answer sits. Core keeps this only for the very next turn of the + * same conversation, only when the evaluation of exactly the accepted + * source selected an approved choice, and only while nothing else has run; + * otherwise it answers false and grants nothing. + */ + carry(text: string, slices: readonly FeedbackSlice[]): boolean; + selectedElicitation(selection: ElicitChoiceSelection): void; +} + +/** A trusted host's local reviewer. */ +export interface AgentReplyReviewer { + /** + * Present one withheld reply, and hand the decision to `decide` once. + * + * `decide` is the terminal for this invocation. Returning without calling it + * leaves the reply withheld, which is also what closing the question means. + */ + review( + hold: AgentReplyHold, + decide: (decision: AgentReplyDecision) => Operation, + ): Operation; +} + +interface ReplyReviewApi { + review(hold: AgentReplyHold, decision: AgentReplyDecision | undefined): Operation; +} + +/** What Core knows about each hold it issued, and whether it was decided. */ +const REVIEW = "agent.reply-review"; +const ReplyReview: Api = createApi(REVIEW, { + // deno-lint-ignore require-yield + *review(): Operation {}, +}); + +/** + * Install this host's local reviewer for the conversations it opens. + * + * A host act, reached through `@executablemd/core/host` and installed as one + * of the conversation's own installations, beside its prompt publisher. + */ +export function* useAgentReplyReview(reviewer: AgentReplyReviewer): Operation { + yield* ReplyReview.around({ + *review([hold, _decision], next) { + let delegated = false; + yield* reviewer.review(hold, function* (decision) { + if (delegated) { + return; + } + delegated = true; + yield* next(hold, decision); + }); + }, + }); +} + +/** What the turn hands the review: the reply, where it matched, and its turn. */ +export interface ReviewRequest { + readonly text: string; + readonly spans: readonly SecretSpan[]; + readonly begun: AgentPromptHandle; + readonly carry: AgentReplyHold["carry"]; + readonly selectedElicitation: AgentReplyHold["selectedElicitation"]; +} + +/** + * Ask the installed reviewer about one withheld reply, and answer only what + * reached the terminal for this invocation. + * + * The hold is live for exactly this call. However it ends — decided, returned + * without a decision, failed or halted — it is dead afterwards, and a reveal + * or settlement through it answers nothing. + */ +export function* askReplyReview( + request: ReviewRequest, +): Operation<{ readonly decision: AgentReplyDecision; readonly hold: AgentReplyHold }> { + let live = true; + let decided: AgentReplyDecision | undefined; + const findings = Object.freeze( + request.spans.map((span) => Object.freeze(findingOf(request.text, span, request.spans))), + ); + const hold: AgentReplyHold = Object.freeze({ + begun: request.begun, + findings, + reveal(finding: number): string | undefined { + const span = request.spans[finding]; + return live && span !== undefined ? request.text.slice(span.start, span.end) : undefined; + }, + carry: request.carry, + selectedElicitation: request.selectedElicitation, + }); + yield* ensure(() => { + live = false; + }); + const terminal = createApi(REVIEW, { + // deno-lint-ignore require-yield + *review(answered, decision): Operation { + if ( + live && + answered === hold && + decided === undefined && + (decision === "accept" || decision === "withhold") + ) { + decided = decision; + } + }, + }); + yield* terminal.operations.review(hold, undefined); + live = false; + return { decision: decided ?? "withhold", hold }; +} + +/** How much of the line a masked excerpt keeps on each side of the match. */ +const EXCERPT = 24; + +/** One span, as the safe description a local reviewer draws. */ +function findingOf( + text: string, + span: SecretSpan, + spans: readonly SecretSpan[], +): AgentReplyFinding { + const before = text.slice(0, span.start); + const lineStart = before.lastIndexOf("\n") + 1; + const line = before.split("\n").length; + const nextLine = text.indexOf("\n", span.end); + const lineEnd = nextLine < 0 ? text.length : nextLine; + return { + ruleId: span.ruleId, + messageId: span.messageId, + line, + column: span.start - lineStart + 1, + excerpt: { + before: maskedExcerpt(text, spans, Math.max(lineStart, span.start - EXCERPT), span.start), + masked: "•".repeat(8), + after: maskedExcerpt(text, spans, span.end, Math.min(lineEnd, span.end + EXCERPT)), + }, + }; +} + +function maskedExcerpt( + text: string, + spans: readonly SecretSpan[], + start: number, + end: number, +): string { + let excerpt = ""; + for (let at = start; at < end; at += 1) { + excerpt += spans.some((span) => span.start <= at && at < span.end) ? "•" : text[at]; + } + return printable(excerpt); +} + +/** Text with every control character replaced, so it can only be drawn as text. */ +function printable(text: string): string { + // deno-lint-ignore no-control-regex + return text.replace(/[\u0000-\u001f\u007f-\u009f]/g, "�"); +} diff --git a/packages/agent/tests/agent-conversation.test.ts b/packages/agent/tests/agent-conversation.test.ts index da315b5a7..ab2eeb6e7 100644 --- a/packages/agent/tests/agent-conversation.test.ts +++ b/packages/agent/tests/agent-conversation.test.ts @@ -17,7 +17,7 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { InMemoryStream } from "@executablemd/durable-streams"; +import { durablePosition, InMemoryStream } from "@executablemd/durable-streams"; import type { DurableEvent } from "@executablemd/durable-streams"; import { createContext, @@ -68,7 +68,14 @@ import type { AgentSessionSetup } from "../src/session-setup.ts"; import { sessionOf } from "../src/session-use.ts"; import { useAgentPromptPublisher } from "../src/publication.ts"; import { secretPolicy, useSecretScannerFactory } from "../../core/src/secrets/policy.ts"; +import { createScannerWith } from "../../core/src/secrets/scanner.ts"; import type { SecretScanner } from "../../core/src/secrets/scanner.ts"; +import { Elicitation } from "../../core/src/elicitation-api.ts"; +import { getExpansion } from "../../core/src/expansion.ts"; +import { elicitationEffectName, questionFingerprint } from "../../core/src/elicit-journal.ts"; +import type { ElicitationRequest } from "../../core/src/elicitation-api.ts"; +import { acceptedReplies, AGENT_PROMPT_REPLY, useAgentReplyReview } from "../host.ts"; +import type { AgentReplyDecision, AgentReplyHold } from "../host.ts"; import type { SecretFinding } from "../../core/src/secrets/findings.ts"; import type { AgentLaunchCoordinator } from "../src/launch-coordinator.ts"; import type { ProviderSessionConfiguration } from "../src/provider-configuration.ts"; @@ -1487,6 +1494,905 @@ describe("Tier CW — a completed reply secret detection withholds", () => { expect(JSON.stringify(events).includes("native-late")).toBe(false); expect(turnNames(events)).toEqual(["turn:1"]); }); + + /** + * The real scanner over a detector that finds the marker wherever it is, + * with its range — the only kind of scanner whose findings can be reviewed. + */ + function useRangedScanner( + matchEnd = (_content: string, start: number) => start + marker.length, + onScan?: () => void, + ): Operation { + return useSecretScannerFactory(() => + createScannerWith((content) => { + onScan?.(); + const messages: { + ruleId: string; + messageId: string; + range: readonly [number, number]; + loc: { start: { line: number; column: number } }; + }[] = []; + for (let at = content.indexOf(marker); at >= 0; at = content.indexOf(marker, at + 1)) { + messages.push({ + ruleId: "test-rule", + messageId: "TEST_SECRET", + range: [at, matchEnd(content, at)], + loc: { start: { line: 1, column: at } }, + }); + } + return Promise.resolve({ messages }); + }), + ); + } + + /** A local reviewer that decides each hold as `decide` says, and keeps it. */ + function reviewing( + decide: (hold: AgentReplyHold, count: number) => AgentReplyDecision | "return", + seen: AgentReplyHold[], + settlers: ((decision: AgentReplyDecision) => Operation)[] = [], + ): ExecutionInstallation { + return { + *install(): Operation { + yield* useAgentReplyReview({ + *review(hold, settle) { + seen.push(hold); + settlers.push(settle); + const decision = decide(hold, seen.length); + if (decision !== "return") { + yield* settle(decision); + } + }, + }); + }, + }; + } + + /** The conversation's own root records, by type and name, in history order. */ + function rootRecords(events: readonly DurableEvent[]): string[] { + return events.flatMap((event) => + event.type === "yield" && + (event.description.type === "agent_prompt" || + event.description.type === AGENT_PROMPT_REPLY || + event.description.type === "generated_xmd") + ? [`${event.description.type} ${event.description.name}`] + : [], + ); + } + + it("CW5: an accepted reply is saved exactly after its withheld turn, answered once and run", function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const seen: AgentReplyHold[] = []; + const reply = `Lead ${marker}\n\n\nshown ${marker}\n\n`; + const answered = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + stream, + stub(traffic, { respond: (content) => (content === "first" ? { deltas: [reply] } : {}) }), + function* (chat) { + const first = yield* chat.prompt("first"); + const ran = first.ok + ? yield* chat.evaluateGeneratedXmdRoot(generated(first.value), seen[0]) + : undefined; + return { first, ran, second: yield* chat.prompt("second") }; + }, + "chat-1", + [reviewing(() => "accept", seen)], + ); + }); + if (!answered.ok) { + throw new Error(`the conversation did not open: ${answered.error.message}`); + } + const { first, ran, second } = answered.value; + // The same turn, answered with exactly what the provider completed it + // with — not a retry, not a new turn. + expect(first.ok && first.value).toBe(reply); + expect(traffic.prompts.map((call) => call.content)).toEqual(["first", "second"]); + expect(second.ok && second.value).toBe("[second]"); + expect(seen).toHaveLength(1); + const hold = seen[0]!; + expect(hold.findings.map((one) => [one.ruleId, one.messageId, one.line, one.column])).toEqual([ + ["test-rule", "TEST_SECRET", 1, 6], + ["test-rule", "TEST_SECRET", 4, 7], + ]); + for (const finding of hold.findings) { + expect(JSON.stringify(finding).includes(marker)).toBe(false); + } + // The hold is dead once its turn stopped waiting. + expect(hold.reveal(0)).toBeUndefined(); + expect(ran?.ok && ran.value.output).toContain(`shown ${marker}`); + + const events = yield* stream.readAll(); + expect(rootRecords(events)).toEqual([ + "agent_prompt turn:1", + `${AGENT_PROMPT_REPLY} turn:1`, + "generated_xmd generated:reply-1", + "agent_prompt turn:2", + ]); + expect(record(events, "turn:1")).toMatchObject({ + status: "completed", + text: "", + withheld: true, + }); + const joined = acceptedReplies(events); + expect(joined.ok && joined.value.get("turn:1")?.text).toBe(reply); + + // Reopened: replayed in order, with no provider, no question and no grant. + const again = fresh(); + const reopened: AgentReplyHold[] = []; + const continued = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + stream, + stub(again), + function* (chat) { + return yield* chat.prompt("third"); + }, + "chat-1", + [reviewing(() => "accept", reopened)], + ); + }); + expect(continued.ok && continued.value.ok && continued.value.value).toBe("[third]"); + expect(again.prompts.map((call) => call.content)).toEqual(["third"]); + expect(reopened).toEqual([]); + expect(turnNames(yield* stream.readAll())).toEqual(["turn:1", "turn:2", "turn:3"]); + }); + + for (const [label, decision] of [ + ["returns without deciding", "return"], + ["withholds", "withhold"], + ] as const) { + it(`CW6: a reviewer that ${label} leaves the withheld completion standing`, function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const seen: AgentReplyHold[] = []; + const answered = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + stream, + stub(traffic, { respond: () => ({ deltas: [`${marker}`] }) }), + function* (chat) { + return yield* chat.prompt("first"); + }, + "chat-1", + [reviewing(() => decision, seen)], + ); + }); + expect(seen).toHaveLength(1); + const first = answered.ok ? answered.value : undefined; + expect(first?.ok).toBe(false); + expect(first !== undefined && !first.ok && first.error.name).toBe("AgentReplyWithheldError"); + const events = yield* stream.readAll(); + expect(JSON.stringify(events).includes(marker)).toBe(false); + expect(rootRecords(events)).toEqual(["agent_prompt turn:1"]); + expect(traffic.prompts).toHaveLength(1); + }); + } + + it("CW7: a settlement kept from an earlier question decides nothing about a later one", function* () { + const stream = new InMemoryStream(); + const seen: AgentReplyHold[] = []; + const settlers: ((decision: AgentReplyDecision) => Operation)[] = []; + let revealed: string | undefined; + const answered = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + stream, + stub(fresh(), { respond: () => ({ deltas: [`${marker}`] }) }), + function* (chat) { + return [yield* chat.prompt("first"), yield* chat.prompt("second")]; + }, + "chat-1", + [ + { + *install(): Operation { + yield* useAgentReplyReview({ + *review(hold, settle) { + seen.push(hold); + settlers.push(settle); + // While waiting, the hold reveals its own match locally. + revealed = hold.reveal(0); + if (seen.length === 1) { + yield* settle("withhold"); + return; + } + // The first question's terminal, and its dead hold, settle + // nothing for the second. + yield* settlers[0]!("accept"); + }, + }); + }, + }, + ], + ); + }); + expect(revealed).toBe(marker); + expect(seen).toHaveLength(2); + expect(seen[0]!.reveal(0)).toBeUndefined(); + const [first, second] = answered.ok ? answered.value : []; + expect(first?.ok).toBe(false); + expect(second?.ok).toBe(false); + const events = yield* stream.readAll(); + expect(JSON.stringify(events).includes(marker)).toBe(false); + expect(rootRecords(events)).toEqual(["agent_prompt turn:1", "agent_prompt turn:2"]); + }); + + it("CW7: cancelling a pending reply joins its review cleanup before returning", function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const pending = withResolvers(); + const cleanup = withResolvers(); + const release = withResolvers(); + yield* scoped(function* () { + yield* useRangedScanner(); + const result = yield* withConversation( + stream, + stub(traffic, { respond: () => ({ deltas: [`${marker}`] }) }), + function* (chat) { + yield* ensure(() => release.resolve()); + const caller = yield* spawn(() => chat.prompt("first")); + const hold = yield* pending.operation; + let joined = false; + const stopping = yield* spawn(function* () { + yield* caller.halt(); + joined = true; + }); + yield* cleanup.operation; + expect(joined).toBe(false); + expect(rootRecords(yield* stream.readAll())).toEqual(["agent_prompt turn:1"]); + release.resolve(); + yield* stopping; + expect(joined).toBe(true); + expect(hold.reveal(0)).toBeUndefined(); + }, + "chat-1", + [ + { + *install() { + yield* useAgentReplyReview({ + *review(hold) { + yield* ensure(function* () { + cleanup.resolve(); + yield* release.operation; + }); + pending.resolve(hold); + yield* suspend(); + }, + }); + }, + }, + ], + ); + expect(result.ok).toBe(true); + }); + expect(traffic.prompts.map((call) => call.content)).toEqual(["first"]); + expect(rootRecords(yield* stream.readAll())).toEqual(["agent_prompt turn:1"]); + }); + + it("CW8: approval covers exactly the accepted reply, once — a repeat asks again and other bytes refuse", function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const seen: AgentReplyHold[] = []; + const reply = `\n${marker}\n\n`; + let refused: unknown; + let released: unknown; + try { + yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + stream, + stub(traffic, { respond: () => ({ deltas: [reply] }) }), + function* (chat) { + const first = yield* chat.prompt("first"); + expect(first.ok).toBe(true); + // Other work in between retires what the acceptance could carry, + // and a later reply repeating the value is a new question. + yield* chat.options(); + const second = yield* chat.prompt("second"); + expect(second.ok).toBe(false); + try { + yield* chat.evaluateGeneratedXmdRoot(generated(reply)); + } catch (error) { + refused = error; + } + }, + "chat-1", + [reviewing((_hold, count) => (count === 1 ? "accept" : "withhold"), seen)], + ); + }); + } catch (error) { + released = error; + } + // The refusal is the run's own secret-policy failure, reported again when + // the conversation is released. + expect(released).toBe(refused); + expect(seen).toHaveLength(2); + expect(refused instanceof Error && refused.name).toBe("SecretDetectedError"); + const events = yield* stream.readAll(); + expect(rootRecords(events)).toEqual([ + "agent_prompt turn:1", + `${AGENT_PROMPT_REPLY} turn:1`, + "agent_prompt turn:2", + ]); + expect(traffic.prompts.map((call) => call.content)).toEqual(["first", "second"]); + }); + + /** A question whose choices the reply wrote, offered the way a host selects it. */ + const CHOICE = { + type: "object", + properties: { + choice: { type: "string", enum: [marker, "none"] }, + note: { type: "string" }, + }, + required: ["choice"], + additionalProperties: false, + } as const; + + /** Open a conversation that may evaluate roots asking ``. */ + function withQuestions( + stream: InMemoryStream, + installation: ExecutionInstallation, + reviewer: ExecutionInstallation, + answer: (request: ElicitationRequest) => unknown, + body: (conversation: AgentConversation) => Operation, + select?: () => AgentReplyHold | undefined, + ): Operation> { + return scoped(function* () { + yield* Elicitation.around( + { + *elicit([request]) { + const hold = select?.(); + if (hold !== undefined) { + hold.selectedElicitation({ + ...(yield* durablePosition()), + name: elicitationEffectName((yield* getExpansion()).position), + fingerprint: questionFingerprint(request), + field: "choice", + option: 0, + }); + } + return answer(request); + }, + }, + { at: "min" }, + ); + const opened = yield* useAgentConversation({ + history: stream, + id: "chat-1", + agent: AGENT, + installations: [ + installation, + reviewer, + { + componentAnswers: [answerProvider("Probe", probe([]).definition)], + *install(): Operation { + yield* useSelection({ Probe: ["self-closing"], Elicit: ["paired"] }); + }, + }, + ], + }); + if (!opened.ok) { + return opened; + } + return Ok(yield* body(opened.value)); + }); + } + + /** The reply these rows accept: a question whose first choice is the marker. */ + const asking = + `Choose.\n\nPick one\n\n` + + `\n${marker}\n\n`; + + it("CW9: acknowledges an ordinary Elicit failure with an approved authored choice", function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const seen: AgentReplyHold[] = []; + const ran = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withQuestions( + stream, + stub(traffic, { respond: () => ({ deltas: [asking] }) }), + reviewing(() => "accept", seen), + () => { + throw new Error("Question refused by the test presenter."); + }, + function* (chat) { + const first = yield* chat.prompt("first"); + if (!first.ok) { + throw first.error; + } + return yield* chat.evaluateGeneratedXmdRoot( + { id: "reply-1", source: first.value, allow: ["read", "write"] }, + seen[0], + ); + }, + ); + }); + expect(ran.ok && ran.value.ok).toBe(false); + const events = yield* stream.readAll(); + const failed = events.find( + (event) => event.type === "yield" && event.description.type === "elicit", + ); + if (failed?.type !== "yield") { + throw new Error("Expected an acknowledged elicitation refusal."); + } + expect(failed.result.status).toBe("err"); + expect(JSON.stringify(failed.description)).toContain(marker); + expect(JSON.stringify(failed.result)).not.toContain(marker); + expect(traffic.prompts).toHaveLength(1); + }); + + function* feedbackSlice(stream: InMemoryStream, text: string, start = 0) { + const events = yield* stream.readAll(); + const answer = events.findLast( + (event) => event.type === "yield" && event.description.type === "elicit", + ); + if (answer === undefined) { + throw new Error("Expected an acknowledged elicitation."); + } + return { + coroutineId: answer.coroutineId, + index: events + .filter((event) => event.type === "yield" && event.coroutineId === answer.coroutineId) + .indexOf(answer), + start, + end: start + text.length, + }; + } + + for (const kind of ["clean", "repeated", "crossing"] as const) { + it(`CW9: an accepted choice with a ${kind} TODO suffix`, function* () { + const stream = new InMemoryStream(); + const traffic = fresh(); + const seen: AgentReplyHold[] = []; + const begun: string[] = []; + const published: unknown[] = []; + const feedback = JSON.stringify({ choice: marker }, null, 2); + const suffix = + "\n\nCommitted TODO changes (in commit order):\n" + (kind === "repeated" ? marker : "[]"); + const composed = feedback + suffix; + const scanned = JSON.stringify(composed); + const join = JSON.stringify(feedback).length - 1; + const findings = new Map(); + const answered = yield* scoped(function* () { + yield* useRangedScanner((content, start) => { + const end = + kind === "crossing" && content === scanned ? scanned.length - 1 : start + marker.length; + if (content === scanned) { + findings.set(start, end); + } + return end; + }); + yield* useAgentPromptPublisher({ + *begin(input) { + begun.push(input); + return input; + }, + *publish(publication) { + yield* publication.append(); + published.push(publication.begun); + }, + }); + yield* Agent.around( + { + *prompt([text, options], next) { + return yield* next(text === feedback ? text + suffix : text, options); + }, + }, + { at: "max" }, + ); + return yield* withQuestions( + stream, + stub(traffic, { + respond: (content) => + content === "first" || content === "later" + ? { deltas: [asking] } + : { deltas: ["Feedback received."] }, + }), + reviewing((_hold, count) => (count === 1 ? "accept" : "withhold"), seen), + () => ({ choice: marker }), + function* (chat) { + const first = yield* chat.prompt("first"); + if (!first.ok) { + throw first.error; + } + const ran = yield* chat.evaluateGeneratedXmdRoot( + { id: "reply-1", source: first.value, allow: ["read", "write"] }, + seen[0], + ); + expect(ran.ok && ran.value.output).toContain(marker); + expect(seen[0]!.carry(feedback, [yield* feedbackSlice(stream, feedback)])).toBe(true); + const second = yield* chat.prompt(feedback); + if (kind === "clean") { + expect(second.ok).toBe(true); + const later = yield* chat.prompt("later"); + expect(later.ok).toBe(false); + expect(seen).toHaveLength(2); + expect(seen[1]).not.toBe(seen[0]); + } + return second; + }, + () => seen[0], + ); + }); + if (!answered.ok) { + throw answered.error; + } + const second = answered.value; + const events = yield* stream.readAll(); + const elicited = events.filter( + (event) => + event.type === "yield" && + event.description.type === "elicit" && + event.result.status === "ok", + ); + expect(elicited).toHaveLength(1); + expect(JSON.stringify(elicited[0])).toContain(marker); + const firstMatch = scanned.indexOf(marker); + expect(firstMatch).toBeLessThan(join); + if (kind === "clean") { + expect([...findings]).toEqual([[firstMatch, firstMatch + marker.length]]); + if (!second.ok) { + throw second.error; + } + const prompted = prompts(events).find( + (event) => event.type === "yield" && event.description.name === "turn:2", + ); + expect(prompted?.type === "yield" && prompted.description.input).toBe(composed); + } else { + if (kind === "repeated") { + const repeated = scanned.lastIndexOf(marker); + expect(repeated).toBeGreaterThanOrEqual(join); + expect([...findings]).toEqual([ + [firstMatch, firstMatch + marker.length], + [repeated, repeated + marker.length], + ]); + } else { + expect([...findings]).toEqual([[firstMatch, scanned.length - 1]]); + expect(scanned.length - 1).toBeGreaterThan(join); + } + expect(!second.ok && second.error.message).toBe( + "Secret detection withheld automatic feedback.", + ); + expect(rootRecords(events)).not.toContain("agent_prompt turn:2"); + } + const expected = kind === "clean" ? ["first", composed, "later"] : ["first"]; + expect(begun).toEqual(expected); + expect(published).toEqual(expected); + expect(traffic.prompts.map((call) => call.content)).toEqual(expected); + if (kind === "clean") { + expect(record(events, "turn:3")).toMatchObject({ status: "completed", withheld: true }); + const replayTraffic = fresh(); + const replayQuestions: AgentReplyHold[] = []; + let scans = 0; + let controls = 0; + let refusal: unknown; + let release: unknown; + try { + yield* scoped(function* () { + yield* useRangedScanner(undefined, () => { + scans += 1; + }); + yield* withQuestions( + stream, + stub(replayTraffic), + reviewing(() => "withhold", replayQuestions), + () => { + controls += 1; + return { choice: "none" }; + }, + function* (chat) { + expect(scans).toBe(0); + expect(replayTraffic).toEqual(fresh()); + expect(controls).toBe(0); + expect(replayQuestions).toEqual([]); + expect(yield* stream.readAll()).toEqual(events); + try { + yield* chat.evaluateGeneratedXmdRoot( + generated(`${marker}`, [], "independent-copy"), + ); + } catch (error) { + refusal = error; + } + }, + ); + }); + } catch (error) { + release = error; + } + expect(scans).toBeGreaterThan(0); + expect(refusal instanceof Error && refusal.name).toBe("SecretDetectedError"); + expect(release).toBe(refusal); + expect(controls).toBe(0); + expect(replayTraffic).toEqual(fresh()); + expect(rootRecords(yield* stream.readAll())).toEqual(rootRecords(events)); + } + }); + } + + for (const [label, answer, slice] of [ + ["the same bytes typed into a free field", { choice: "none", note: marker }, "stated"], + ["a slice the host did not state", { choice: marker }, "elsewhere"], + ] as const) { + it(`CW9: ${label} inherits nothing from the acceptance`, function* () { + const stream = new InMemoryStream(); + const seen: AgentReplyHold[] = []; + let failure: unknown; + let answered: Result> | undefined; + try { + answered = yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withQuestions( + stream, + stub(fresh(), { + respond: (content) => (content === "first" ? { deltas: [asking] } : {}), + }), + reviewing(() => "accept", seen), + () => answer, + function* (chat) { + const first = yield* chat.prompt("first"); + if (!first.ok) { + throw first.error; + } + yield* chat.evaluateGeneratedXmdRoot( + { + id: "reply-1", + source: first.value, + allow: ["read", "write"], + }, + seen[0], + ); + // The value appears twice; the host states only the second. + const result = JSON.stringify({ choice: marker }, null, 2); + const feedback = `${marker} and ${result}`; + seen[0]!.carry( + feedback, + slice === "elsewhere" + ? [yield* feedbackSlice(stream, result, marker.length + 5)] + : [], + ); + return yield* chat.prompt(feedback); + }, + slice === "elsewhere" ? () => seen[0] : undefined, + ); + }); + } catch (error) { + failure = error; + } + if (slice === "stated") { + // A free field's value is another origin: the evaluation itself is refused. + expect(failure instanceof Error && failure.name).toBe("SecretDetectedError"); + } else { + // An occurrence outside the stated slice: that turn's record is refused. + expect(failure).toBeUndefined(); + expect(answered?.ok && answered.value.ok).toBe(false); + } + const events = yield* stream.readAll(); + expect( + JSON.stringify( + events.filter((one) => one.type === "yield" && one.description.name === "turn:2"), + ), + ).not.toContain(marker); + }); + } + + it("CW10: an acceptance that does not directly follow its own withheld turn is refused on opening", function* () { + const withheld = { + sequence: 1, + agent: AGENT, + sessionKey: "k", + status: "completed", + text: "", + withheld: true, + }; + const completed = { + sequence: 1, + agent: AGENT, + sessionKey: "k", + status: "completed", + text: "kept", + }; + const accepted = { + version: 1, + text: "kept", + approved: [{ ruleId: "r", messageId: "m", start: 0, end: 4 }], + }; + const prompt = (name: string, value: Json): DurableEvent => ({ + type: "yield", + coroutineId: "root", + description: { type: "agent_prompt", name, input: "x" }, + result: { status: "ok", value }, + }); + const reply = (name: string, value: Json): DurableEvent => ({ + type: "yield", + coroutineId: "root", + description: { type: AGENT_PROMPT_REPLY, name, input: name }, + result: { status: "ok", value }, + }); + expect(acceptedReplies([prompt("turn:1", withheld), reply("turn:1", accepted)]).ok).toBe(true); + for (const history of [ + [reply("turn:1", accepted)], + [prompt("turn:1", withheld), reply("turn:1", accepted), reply("turn:1", accepted)], + [prompt("turn:1", withheld), prompt("turn:2", completed), reply("turn:1", accepted)], + [prompt("turn:1", completed), reply("turn:1", accepted)], + [prompt("turn:1", withheld), reply("turn:2", accepted)], + [prompt("turn:1", withheld), reply("turn:1", { ...accepted, extra: true })], + ]) { + expect(acceptedReplies(history).ok).toBe(false); + } + }); + + it("CW11: no question is asked where the reply was never the conversation's withheld completion", function* () { + const seen: AgentReplyHold[] = []; + // A scanner that fails, and a failed turn's partial text: refused by the + // journal's own guard and never offered. + for (const [mode, status] of [ + ["fail", "completed"], + ["flag", "failed"], + ] as const) { + const stream = new InMemoryStream(); + yield* scoped(function* () { + yield* useScanner(mode); + return yield* withConversation( + stream, + stub(fresh(), { respond: () => ({ status, deltas: [`${marker}`] }) }), + function* (chat) { + return yield* chat.prompt("first"); + }, + "chat-1", + [reviewing(() => "accept", seen)], + ); + }); + expect(JSON.stringify(yield* stream.readAll()).includes(marker)).toBe(false); + } + // A substituted scanner that cannot say where it matched: withheld, never offered. + const stream = new InMemoryStream(); + const answered = yield* scoped(function* () { + yield* useScanner("flag"); + return yield* withConversation( + stream, + stub(fresh(), { respond: () => ({ deltas: [`${marker}`] }) }), + function* (chat) { + return yield* chat.prompt("first"); + }, + "chat-1", + [reviewing(() => "accept", seen)], + ); + }); + expect(answered.ok && !answered.value.ok && answered.value.error.name).toBe( + "AgentReplyWithheldError", + ); + expect(seen).toEqual([]); + }); + + for (const failedType of ["agent_prompt", AGENT_PROMPT_REPLY]) { + it(`CW12: preserves the first storage failure when ${failedType} cannot be appended`, function* () { + const backend = new InMemoryStream(); + const storageFailure = new Error("the storage refused this append"); + const attempted: string[] = []; + backend.onAppend = (event) => { + attempted.push(...rootRecords([event])); + if (event.type === "yield" && event.description.type === failedType) { + throw storageFailure; + } + }; + const seen: AgentReplyHold[] = []; + const traffic = fresh(); + let failure: unknown; + let caller: Error | undefined; + try { + yield* scoped(function* () { + yield* useRangedScanner(); + return yield* withConversation( + backend, + stub(traffic, { respond: () => ({ deltas: [`${marker}`] }) }), + function* (chat) { + const settled = yield* chat.prompt("first"); + caller = settled.ok ? undefined : settled.error; + }, + "chat-1", + [reviewing(() => "accept", seen)], + ); + }); + } catch (error) { + failure = error; + } + expect(durabilityFailure(caller)?.cause).toBe(storageFailure); + expect(durabilityFailure(failure)?.cause).toBe(storageFailure); + const firstFailed = failedType === "agent_prompt"; + expect(seen).toHaveLength(firstFailed ? 0 : 1); + expect(attempted).toEqual( + firstFailed + ? ["agent_prompt turn:1"] + : ["agent_prompt turn:1", `${AGENT_PROMPT_REPLY} turn:1`], + ); + const events = yield* backend.readAll(); + expect(JSON.stringify(events).includes(marker)).toBe(false); + expect(rootRecords(events)).toEqual(firstFailed ? [] : ["agent_prompt turn:1"]); + if (!firstFailed) { + expect(record(events, "turn:1")).toMatchObject({ withheld: true }); + } + expect(traffic.prompts).toHaveLength(1); + }); + } + + for (const [description, partial, crossing] of [ + ["an independently produced match in completed", false, false], + ["an independently produced match in partial", true, false], + ["a match crossing the approved literal into independent", false, true], + ] as const) { + it(`CW13: refuses ${description} selected Output`, function* () { + const stream = new InMemoryStream(); + const seen: AgentReplyHold[] = []; + const traffic = fresh(); + const tail = "independent"; + const source = crossing + ? `${marker}` + : `Context ${marker}.\n\n\n${partial ? "" : ""}`; + let scannedCrossing = false; + const external = pinnedComponent( + "External", + definitionOf("External", "read", ["self-closing"], function* (): Operation { + return crossing ? tail : marker; + }), + "component-answer", + ); + const attempts: string[] = []; + let release: unknown; + try { + yield* scoped(function* () { + yield* useRangedScanner((content, start) => { + const end = start + marker.length; + if (crossing && content.startsWith(" ({ deltas: [source] }) }), + function* (chat) { + const prompt = yield* chat.prompt("first"); + if (!prompt.ok) { + throw prompt.error; + } + try { + yield* chat.evaluateGeneratedXmdRoot(generated(prompt.value), seen[0]); + } catch (error) { + attempts.push(error instanceof Error ? error.name : ""); + } + }, + "chat-1", + [reviewing(() => "accept", seen)], + AGENT, + [external, boom()], + ); + }); + } catch (error) { + release = error; + } + expect(attempts).toEqual(["SecretDetectedError"]); + expect(release instanceof Error && release.name).toBe("SecretDetectedError"); + expect(traffic.prompts).toHaveLength(1); + if (crossing) { + expect(scannedCrossing).toBe(true); + expect(release instanceof Error && release.message.includes(marker)).toBe(false); + const events = yield* stream.readAll(); + expect(rootRecords(events)).toEqual([ + "agent_prompt turn:1", + `${AGENT_PROMPT_REPLY} turn:1`, + "generated_xmd generated:reply-1", + ]); + expect(record(events, "turn:1")).toMatchObject({ withheld: true, text: "" }); + const saved = acceptedReplies(events); + expect(saved.ok && saved.value.get("turn:1")?.text).toBe(source); + } + }); + } }); describe("Tier CV — a conversation is an execution the host keeps open", () => { diff --git a/packages/cli/src/repl/application.ts b/packages/cli/src/repl/application.ts index 59a7b83b0..6b2b6f24f 100644 --- a/packages/cli/src/repl/application.ts +++ b/packages/cli/src/repl/application.ts @@ -463,6 +463,29 @@ export interface ReplSidekickSubmit { readonly admitted: boolean; } +/** + * A completed reply secret detection withheld, as the foreground screen asks + * about it. + * + * The findings are rule, position and a masked excerpt. The matched text is + * not a member: `reveal` answers it to the renderer, only while the question + * is waiting, and only when this screen's own state says the person asked to + * see it — so it is never part of a state, a location or a reading anything + * else is handed. + */ +export interface ReplSidekickApproval { + /** This screen's own name for the question, never reused. */ + readonly key: string; + /** The saved withheld turn. */ + readonly turn: number; + readonly findingCount: number; + /** Whether Accept was chosen and the acceptance is being saved. */ + readonly accepting: boolean; + /** Whether the person asked this screen to show the matched text. */ + readonly revealed: number | undefined; + finding(index: number): readonly [description: string, excerpt: string] | undefined; +} + export interface ReplSidekickReading { readonly feedback?: readonly SidekickFeedback[]; readonly usageTotal?: string; @@ -536,6 +559,13 @@ export interface ReplSidekickReading { * component holding it would be a second way to answer, answerable to nobody. */ readonly question: ReplSidekickQuestion | undefined; + /** + * The withheld reply waiting for the person's decision, or none. + * + * Local to the foreground screen that draws it: a headless or attached + * presentation is never given one. + */ + readonly approval?: ReplSidekickApproval; /** The conversation these facts are about, for a control that names it. */ readonly conversation: string | undefined; /** @@ -768,6 +798,7 @@ export interface ReplFormMessage { */ export interface ReplFormState { readonly carets?: Readonly>; + readonly choices?: readonly string[]; /** What has been typed into each field so far, by field name. */ readonly values: Readonly>; /** Which field text and Backspace act on, or none yet. */ @@ -1044,6 +1075,7 @@ export interface ReplViewports { readonly permission: number; /** Rows the Configure drawer is scrolled by. */ readonly configure: number; + readonly approval?: { readonly key: string; readonly offset: number }; /** * Rows each retained drawer reading is scrolled by, one per reading. * @@ -1221,6 +1253,16 @@ export interface ReplState { /** What the root must perform, because a component cannot. */ export type ReplIntent = | { readonly kind: "start-new-chat"; readonly conversation: string; readonly decision: string } + /** + * Show or hide the matched text of the withheld reply this key names, or + * decide it once, if it is still waiting. + */ + | { + readonly kind: "decide-sidekick-reply"; + readonly approval: string; + readonly decision: "reveal" | "withhold" | "accept"; + readonly finding?: number; + } | { readonly kind: "recovery-reference"; readonly accept: boolean; readonly reference: string } | { readonly kind: "todo-save"; @@ -1278,6 +1320,7 @@ export type ReplIntent = readonly kind: "answer-sidekick"; readonly question: string; readonly values: Readonly>; + readonly choices?: readonly string[]; } /** * Ask this conversation's agent what it offers, explicitly. @@ -2171,6 +2214,10 @@ export function reduceRepl( // drawer holding focus is where the keys a person presses are going — and // whether Escape is addressed to this drawer. const configuring = presented?.kind === "configure" && state.configuring.open; + const approval = + presented?.kind === "reply-approval" && presented.key === live.sidekick?.approval?.key + ? live.sidekick.approval + : undefined; switch (action.kind) { case "use-saved-recovery": { @@ -2619,6 +2666,20 @@ export function reduceRepl( : settled(state); } case "close-drawer": { + if (approval !== undefined && !approval.accepting) { + return reduceRepl( + state, + { + kind: "review-sidekick-reply", + approval: approval.key, + decision: "withhold", + }, + model, + live, + admission, + presented, + ); + } if (state.route.drawers.length === 0 && state.catalog) { return settled({ ...state, catalog: false }); } @@ -3068,6 +3129,23 @@ export function reduceRepl( refusal: undefined, }); } + case "review-sidekick-reply": { + const pending = approval; + if (pending === undefined || pending.key !== action.approval || pending.accepting) { + // Drawn for a question that has been decided, withdrawn or replaced: + // it reaches nothing, and the question now waiting is not this one. + return refuse(state, "that withheld reply is no longer waiting for a decision."); + } + return { + state: Object.freeze({ ...state, refusal: undefined }), + intent: { + kind: "decide-sidekick-reply", + approval: pending.key, + decision: action.decision, + finding: action.finding, + }, + }; + } case "close-sidekick-question": { if (state.asking === undefined || !state.asking.open) { return refuse(state, "the Sidekick's question form is not open."); @@ -3113,14 +3191,19 @@ export function reduceRepl( // The same meaning activation has in the entry's form: the value goes in // and the whole object is offered, because whether it is an answer is the // schema's to say. - const chosen = written(asking.held.form, action.field, action.option); + const chosen = written(asking.held.form, action.field, action.option, true); return { state: Object.freeze({ ...state, asking: Object.freeze({ ...asking.held, form: chosen }), refusal: undefined, }), - intent: { kind: "answer-sidekick", question: asking.key, values: chosen.values }, + intent: { + kind: "answer-sidekick", + question: asking.key, + values: chosen.values, + choices: chosen.choices, + }, }; } case "answer-sidekick": { @@ -3136,6 +3219,7 @@ export function reduceRepl( kind: "answer-sidekick", question: asking.key, values: asking.held.form.values, + choices: asking.held.form.choices, }, }; } @@ -3373,6 +3457,17 @@ export function reduceRepl( }); } case "scroll": { + const window = admission.windows.get(DRAWER_WINDOW); + if (approval !== undefined && window !== undefined) { + return settled({ + ...state, + viewports: { + ...state.viewports, + approval: { key: approval.key, offset: scrolled(window, action.delta) }, + }, + refusal: undefined, + }); + } const open = state.route.drawers[state.route.drawers.length - 1]; if (open === undefined) { const inspection = state.inspection; @@ -3388,7 +3483,6 @@ export function reduceRepl( // The window the frame measured for the drawer that is open. One viewport // moves at a time, because one drawer is read at a time; which offset the // new position is stored in is decided by what is being read. - const window = admission.windows.get(DRAWER_WINDOW); if (open.kind === "live-permission") { const pending = state.permission === undefined ? undefined : offered(state, live, state.permission); @@ -3839,12 +3933,15 @@ export function askingFor(state: ReplState, question: ReplSidekickQuestion | und } /** One field's value replaced, leaving every other field alone. */ -function written(form: ReplFormState, field: string, value: string): ReplFormState { +function written(form: ReplFormState, field: string, value: string, choice = false): ReplFormState { const carets = { ...form.carets }; delete carets[field]; return Object.freeze({ - carets, ...form, + carets, + choices: choice + ? Object.freeze([...(form.choices ?? []).filter((name) => name !== field), field]) + : form.choices?.filter((name) => name !== field), values: Object.freeze({ ...form.values, [field]: value }), // A value that has changed makes the last verdict about the old one stale, // so the messages go rather than sitting under a form they no longer @@ -4443,6 +4540,9 @@ function chatFacts(view: ReplView): readonly (string | number | boolean | null | view.state.showUsage === true ? reading?.inspectUsage : undefined, reading?.setup?.phase, reading?.setup?.failure, + reading?.approval?.key, + reading?.approval?.revealed, + reading?.approval?.accepting, staging !== undefined, staging?.inspection?.kind, staging?.inspection?.key, @@ -4971,6 +5071,7 @@ function chunked(text: string, width: number): readonly string[] { */ interface Described { readonly key: string; + readonly approvalLine?: string; readonly description: ReplDescription; readonly style: ReplRowStyle; /** @@ -5238,6 +5339,7 @@ export function focusSettled( if (claim.startsWith("drawer:")) { const from = admission.windows.get(DRAWER_WINDOW)?.from; if (from !== undefined) { + const approval = presentedApproval(view); const open = state.route.drawers.at(-1); const key = open === undefined ? undefined : readingKeyOf(state, open); return Object.freeze({ @@ -5247,6 +5349,8 @@ export function focusSettled( ...viewports, configure: presentedConfiguring(view) === undefined ? viewports.configure : from, permission: open?.kind === "live-permission" ? from : viewports.permission, + approval: + approval === undefined ? viewports.approval : { key: approval.key, offset: from }, readings: key === undefined ? viewports.readings : { ...viewports.readings, [key]: from }, }, @@ -6215,7 +6319,7 @@ function described(view: ReplView, context: ReplPresentationContext): DescribedS * needs are reparented into the form, exactly as `[history]` and `[exit]` * already are — described once, in the active focus root. */ - const covered = presentedAsking(view) !== undefined; + const covered = presentedAsking(view) !== undefined || presentedApproval(view) !== undefined; // The staged composition draws the Transcript of agent work on the left and // reaches the catalog through the selected work's breadcrumbs, so the two // surface headings and the catalog's own rows have no region in it. @@ -7965,9 +8069,14 @@ function currentChat(view: ReplView): boolean { export type ReplPresented = | { readonly kind: "entry-question"; readonly question: ReplQuestion } | { readonly kind: "question"; readonly question: string } - | { readonly kind: "configure" }; + | { readonly kind: "configure" } + | { readonly kind: "reply-approval"; readonly key: string }; export function presentedDrawer(view: ReplView): ReplPresented | undefined { + const approval = presentedApproval(view); + if (approval !== undefined) { + return { kind: "reply-approval", key: approval.key }; + } if ( !view.state.heldHistory && view.selection.drawers.at(-1)?.kind === "live-elicit" && @@ -8031,6 +8140,7 @@ function presentedInspection(view: ReplView): ReplInspection | undefined { view.selection.drawers.length > 0 || presentedCatalog(view) || presentedAsking(view) !== undefined || + presentedApproval(view) !== undefined || presentedConfiguring(view) !== undefined ? undefined : view.state.inspection; @@ -8058,11 +8168,18 @@ function inspectionRect(size: ReplTerminalSize): ReplBounds { * Sidekick's opener, so an open permission request is never answered or * dismissed by something underneath it. */ +function presentedApproval(view: ReplView): ReplSidekickApproval | undefined { + return view.selection.drawers.length === 0 && !presentedCatalog(view) && currentChat(view) + ? view.live.sidekick?.approval + : undefined; +} + export function modalOpen(view: ReplView): boolean { return ( view.selection.drawers.length > 0 || presentedCatalog(view) || presentedAsking(view) !== undefined || + presentedApproval(view) !== undefined || presentedConfiguring(view) !== undefined || (profileFor(view.size) === "narrow" && presentedInspection(view) !== undefined) ); @@ -8094,7 +8211,9 @@ function presentedRect(view: ReplView, context: ReplPresentationContext): ReplBo // a reading of the whole execution rather than of one pane, so a pane-local // rectangle would put the list of every entry inside the Sidekick's column. const rect = - view.selection.drawers.length > 0 || presentedCatalog(view) + view.selection.drawers.length > 0 || + presentedCatalog(view) || + presentedApproval(view) !== undefined ? (context.drawer ?? drawerRect(view.size, workspace)) : presentedAsking(view) !== undefined || presentedConfiguring(view) !== undefined ? sidekickDrawerRect(view.size, workspace) @@ -8212,7 +8331,11 @@ function drawerFor( ), ); } - if ((asking !== undefined || configuring) && view.live.sidekick?.busy === true) { + const approval = presentedApproval(view); + if ( + (approval !== undefined || asking !== undefined || configuring) && + view.live.sidekick?.busy === true + ) { rows.push( row("drawer:sidekick:stop", "[stop]", { select: "stop-sidekick" }, styleOf("action"), { here: view.focused, @@ -8229,21 +8352,23 @@ function drawerFor( row( "drawer:close", "[close]", - dismissing !== undefined - ? { select: "permission-dismiss", request: dismissing } - : catalog - ? // Putting the catalog away chooses nothing: the scope being read, - // the marker, the filter and both drafts stand. - { select: "catalog-close" } - : // Closing one of the Sidekick's own drawers is not what closing the - // entry's means. The question form keeps its question and its - // values; Configure discards what was chosen and not applied, and - // saves and changes nothing. - configuring - ? { select: "configure-close" } - : asking === undefined - ? { select: "close" } - : { select: "sidekick-question-close" }, + approval !== undefined + ? { select: "sidekick-approval", approval: approval.key, decision: "withhold" } + : dismissing !== undefined + ? { select: "permission-dismiss", request: dismissing } + : catalog + ? // Putting the catalog away chooses nothing: the scope being read, + // the marker, the filter and both drafts stand. + { select: "catalog-close" } + : // Closing one of the Sidekick's own drawers is not what closing the + // entry's means. The question form keeps its question and its + // values; Configure discards what was chosen and not applied, and + // saves and changes nothing. + configuring + ? { select: "configure-close" } + : asking === undefined + ? { select: "close" } + : { select: "sidekick-question-close" }, styleOf("action"), { here: view.focused }, ), @@ -8601,6 +8726,69 @@ function messageRows( return question.message.split("\n").map((text) => ({ text, runs: sourceRuns(text) })); } +function approvalContent( + view: ReplView, + approval: ReplSidekickApproval, + width: number, + preview: readonly ReplFittedLine[] | undefined, +): DrawerContent { + const content: DrawerRow[] = []; + const line = (key: string, label: string) => { + content.push({ described: drawerLine(key, label, width, styleOf("metadata")) }); + }; + const control = (decision: "reveal" | "accept" | "withhold", label: string, finding?: number) => { + content.push({ + described: row( + `drawer:reply-approval:${decision}${finding === undefined ? "" : `:${finding}`}`, + padControl(label, width), + { + select: "sidekick-approval", + approval: approval.key, + decision, + ...(finding === undefined ? {} : { finding }), + }, + styleOf("action"), + { here: view.focused }, + ), + }); + }; + let next = 0; + const pieces = preview ?? []; + for (let index = 0; index < approval.findingCount; index += 1) { + const rule = `drawer:reply-approval:rule:${index}`; + const excerpt = `drawer:reply-approval:excerpt:${index}`; + while (pieces[next]?.line === rule || pieces[next]?.line === excerpt) { + const piece = pieces[next++]; + content.push({ + described: { + ...drawerLine(piece.key, "", width, styleOf("metadata")), + approvalLine: piece.key, + }, + }); + } + if (!approval.accepting) { + const verb = approval.revealed === index ? "hide" : "reveal"; + const suffix = approval.findingCount === 1 ? "" : ` ${index + 1}`; + control("reveal", `[${verb} match${suffix}]`, index); + } + } + line("drawer:reply-approval:disclosure", "Accept saves the original and permits ACP Output"); + line("drawer:reply-approval:feedback", "and automatic feedback."); + if (approval.accepting) { + line("drawer:reply-approval:saving", "Saving the accepted reply…"); + } else { + control("withhold", "[withhold reply]"); + control("accept", "[accept reply]"); + } + return { + title: `Reply withheld · turn ${approval.turn}`, + titleStyle: styleOf("question-title"), + dismissing: undefined, + entering: false, + content, + }; +} + function drawerContent( view: ReplView, width: number, @@ -8613,6 +8801,10 @@ function drawerContent( return catalogContent(view, width); } if (open === undefined) { + const approval = presentedApproval(view); + if (approval !== undefined) { + return approvalContent(view, approval, width, preview); + } // No route drawer, so this is one of the Sidekick's own two drawers if // anything is. Configure first: opening it hides the question form's // presentation, so both are never open at once. @@ -10443,6 +10635,7 @@ export interface ReplPresentationContext { /** One reading's descriptions, paired with where the engine places them. */ export interface ReplPresentation { + readonly approvalRows?: ReadonlyMap; readonly descriptions: readonly ReplDescription[]; readonly manifest: ReplLayoutManifest; } @@ -11121,9 +11314,13 @@ export function presentationFor( requested !== undefined && containsRequest(placedDescriptions) ? placedDescriptions.map(requestedDescription) : placedDescriptions; + const approvalRows = new Map(); const styles = new Map(); const runs = new Map(); for (const one of screen.rows) { + if (one.approvalLine !== undefined) { + approvalRows.set(one.key, one.approvalLine); + } styles.set(one.key, one.style); const edited = editorRuns(one.description, context); if (edited !== undefined || one.runs !== undefined) { @@ -11546,18 +11743,21 @@ export function presentationFor( const workspace = staged(view); if (workspace !== undefined) { - return workspacePresentation({ - view, - context, - descriptions, - candidates, - rail, - listColumn, - chatColumn, - viewports, - regions, - contents, - }); + return { + ...workspacePresentation({ + view, + context, + descriptions, + candidates, + rail, + listColumn, + chatColumn, + viewports, + regions, + contents, + }), + approvalRows, + }; } const columns: ReplBox[] = []; @@ -11787,6 +11987,7 @@ export function presentationFor( return { descriptions, + approvalRows, manifest: Object.freeze({ profile, size: Object.freeze({ ...size }), @@ -12193,6 +12394,11 @@ function offsetOf(view: ReplView, window: string): number { if (window === ENTRIES_WINDOW) { return view.state.viewports.entries; } + const approval = presentedApproval(view); + if (approval !== undefined) { + const held = view.state.viewports.approval; + return held?.key === approval.key ? held.offset : 0; + } return ( presentedInspection(view)?.offset ?? drawerOffsetOf(view.state, view.state.route.drawers[view.state.route.drawers.length - 1]) diff --git a/packages/cli/src/repl/components/actions.ts b/packages/cli/src/repl/components/actions.ts index 05ea3de0f..d28085bb0 100644 --- a/packages/cli/src/repl/components/actions.ts +++ b/packages/cli/src/repl/components/actions.ts @@ -202,6 +202,19 @@ export type ReplAction = | { readonly kind: "open-sidekick-question"; readonly question: string } /** Hide the Sidekick's question form, leaving the question pending. */ | { readonly kind: "close-sidekick-question" } + /** + * Act on the withheld reply this key names: show or hide its matched text + * on this screen, or decide it. + * + * The key is the one question this was drawn for. A reveal changes only what + * this screen draws; only `withhold` and `accept` decide anything. + */ + | { + readonly kind: "review-sidekick-reply"; + readonly approval: string; + readonly decision: "reveal" | "withhold" | "accept"; + readonly finding?: number; + } /** Give one field of the Sidekick's own form the focus text acts on. */ | { readonly kind: "select-sidekick-field"; diff --git a/packages/cli/src/repl/components/rows.ts b/packages/cli/src/repl/components/rows.ts index a2b0d4174..eda148d82 100644 --- a/packages/cli/src/repl/components/rows.ts +++ b/packages/cli/src/repl/components/rows.ts @@ -405,6 +405,19 @@ function activation(input: ReplViewData): ReplAction | undefined { if (select === "sidekick-question-close") { return { kind: "close-sidekick-question" }; } + if (select === "sidekick-approval") { + const approval = named["approval"]; + const decision = named["decision"]; + return typeof approval === "string" && + (decision === "reveal" || decision === "withhold" || decision === "accept") + ? { + kind: "review-sidekick-reply", + approval, + decision, + finding: typeof named["finding"] === "number" ? named["finding"] : undefined, + } + : undefined; + } if (select === "sidekick-form-field") { const question = named["question"]; const field = named["field"]; diff --git a/packages/cli/src/repl/elicitation.ts b/packages/cli/src/repl/elicitation.ts index 8ef7e7ee4..52dede930 100644 --- a/packages/cli/src/repl/elicitation.ts +++ b/packages/cli/src/repl/elicitation.ts @@ -110,7 +110,7 @@ export interface ReplQuestion { * is still the question they are answering. */ refuse?(error: ElicitationProviderError): void; - submit(values: Readonly>): ReplFormOutcome; + submit(values: Readonly>, choices?: readonly string[]): ReplFormOutcome; } /** What the application reads to draw the question, and to know there is one. */ @@ -448,6 +448,7 @@ export function* useReplElicitation( * something a reading wants, not something answering a question needs. */ waits?: ReplWaits, + onSelection?: (answer: object, field: string, option: number) => void, ): Operation { const changes = createSignal(); let pending: ReplQuestion | undefined; @@ -489,7 +490,10 @@ export function* useReplElicitation( reject(error); } }, - submit(values: Readonly>): ReplFormOutcome { + submit( + values: Readonly>, + choices?: readonly string[], + ): ReplFormOutcome { if (settled) { return { kind: "invalid", issues: [] }; } @@ -525,6 +529,15 @@ export function* useReplElicitation( return { kind: "invalid", issues: [] }; } settled = true; + for (const field of form.fields) { + if (!choices?.includes(field.name) || field.secret === true) { + continue; + } + const option = field.choices?.indexOf(values[field.name]) ?? -1; + if (option >= 0) { + onSelection?.(assembled, field.name, option); + } + } resolve(assembled); return { kind: "answered", answer: projected.value }; }, diff --git a/packages/cli/src/repl/program.ts b/packages/cli/src/repl/program.ts index 945f610b8..b8cba6702 100644 --- a/packages/cli/src/repl/program.ts +++ b/packages/cli/src/repl/program.ts @@ -153,6 +153,7 @@ import type { ReplPresented, ReplRestorePhase, ReplSidekickChatRow, + ReplSidekickApproval, ReplSidekickQuestion, ReplSidekickReading, ReplSidekickSetup, @@ -216,6 +217,7 @@ import { } from "./layout.ts"; import { focusHintOps, focusRouteHint } from "./focus-hints.ts"; import { + ReplFitRefusal, createFitStore, createPlainFitStore, fitCells, @@ -1882,6 +1884,9 @@ function* drive( */ const sidekick: SidekickOwner = yield* useSidekickOwner({ follow: () => follow, + // Only a foreground terminal has a person who may decide about a reply + // secret detection withheld. A headless root keeps every one withheld. + reviewsReplies: options.headless === undefined, assembly: options.profile.sidekickAssembly, projectCwd: options.profile.projectCwd, workspace, @@ -2266,7 +2271,7 @@ function* drive( * a route is being validated at has to be the reading that route names. */ function sidekickReading(held: ReplState): ReplSidekickReading { - return sidekickReadingOf(sidekick, askingNow(), held.route.sidekick); + return sidekickReadingOf(sidekick, askingNow(), held.route.sidekick, approvalNow(sidekick)); } /** @@ -3870,6 +3875,42 @@ function setupOf( return Object.freeze({ phase: found.phase, failure: found.failure }); } +/** The withheld reply waiting on this screen's person, as the screen draws it. */ +function approvalNow(owner: SidekickOwner): ReplSidekickApproval | undefined { + const asked = owner.approval; + if (asked === undefined) { + return undefined; + } + return Object.freeze({ + key: asked.key, + turn: asked.turn, + findingCount: asked.findings.length, + accepting: asked.accepting, + revealed: asked.revealed, + finding(index: number): readonly [string, string] | undefined { + if (owner.approval !== asked) { + return undefined; + } + const finding = asked.findings[index]; + if (finding === undefined) { + return undefined; + } + const { ruleId, messageId, line, column, excerpt } = finding; + const revealed = asked.revealed === index; + const shown = revealed ? (asked.reveal(index) ?? excerpt.masked) : excerpt.masked; + return [ + approvalText(`${messageId} · line ${line}, column ${column} (${ruleId})`), + approvalText(revealed ? shown : excerpt.before + shown + excerpt.after), + ]; + }, + }); +} + +function approvalText(text: string): string { + // deno-lint-ignore no-control-regex + return text.replace(/[\u0000-\u001f\u007f-\u009f]/g, "�"); +} + /** * What the Sidekick says, as detached rows the screen can draw. * @@ -3896,6 +3937,8 @@ function sidekickReadingOf( * names a chat this history actually holds. */ selected: string | undefined, + /** The withheld reply waiting on this screen's person, as the screen names it. */ + approval?: ReplSidekickApproval, ): ReplSidekickReading { const read = owner.reading; if (!read.ok) { @@ -3970,6 +4013,8 @@ function sidekickReadingOf( agent: read.value.conversations.find((one) => one.id === reading)?.agent ?? owner.agent, refusal: undefined, question, + // Only for the current chat, which is the only one anybody is asked about. + ...(approval === undefined || reading !== active ? {} : { approval }), // What was asked for, read back out of the history rather than out of // anything this process remembered. requested: requestedBy(read.value, active), @@ -4121,7 +4166,9 @@ export function sidekickTurnRows( ? "asking" : live.phase === "generating" ? "generating" - : "evaluating the reply", + : live.phase === "reviewing" + ? "awaiting reply approval" + : "evaluating the reply", running: true, failed: false, sessionKey: live.sessionKey, @@ -4764,6 +4811,19 @@ function* perform( case "start-new-chat": chat.startNewChat(intent.conversation); return {}; + case "decide-sidekick-reply": { + // The question this key names, and only while it is still the one + // waiting: a decision drawn for an earlier question decides nothing. + const asked = chat.owner.approval; + if (asked === undefined || asked.key !== intent.approval) { + return { refusal: "that withheld reply is no longer waiting for a decision." }; + } + const done = + intent.decision === "reveal" + ? intent.finding !== undefined && asked.toggleReveal(intent.finding) + : asked.decide(intent.decision); + return done ? {} : { refusal: "that withheld reply is no longer waiting for a decision." }; + } case "recovery-reference": chat.reference(intent.accept, intent.reference); return {}; @@ -4943,7 +5003,7 @@ function* perform( // The schema decides, through the same compiled validator the request // carries, and the existing canonical operation writes the answer. This // reads no form and judges no value. - const outcome = asked.submit(intent.values); + const outcome = asked.submit(intent.values, intent.choices); return outcome.kind === "answered" ? { askingAnswered: true } : { askingMessages: reported(outcome) }; @@ -5451,6 +5511,13 @@ export function* commitReplFrame( for (const cell of tree.frame().cells) { cells.set(cell.node, cell.cell); } + for (const [key, line] of presentation.approvalRows ?? []) { + const node = nodeByKey.get(key); + const cell = prepared.value.approvalCell?.(line); + if (node !== undefined && cell !== undefined) { + cells.set(node, cell); + } + } // Read after the reconcile that settled it and handed on, never stored: the // tree owns focus, and decoration is the one thing that needs to know where it // ended up. A row's own `>` cue is the view's and is therefore a frame behind; @@ -5514,6 +5581,7 @@ function sidekickViewportOf(manifest: ReplLayoutManifest): string { /** What one bounded preparation measured, and the frame it decided on. */ export interface ReplPrepared { + readonly approvalCell?: (line: string) => string | undefined; readonly context: ReplPresentationContext; readonly presentation: ReplPresentation; readonly admission: ReplAdmission; @@ -5977,20 +6045,43 @@ export function* prepareFrame( } // A drawer row is one displayed row: fitting the presented question or // permission text before admission keeps wrapped content reachable. + const approval = + presentedDrawer(view)?.kind === "reply-approval" ? view.live.sidekick?.approval : undefined; + const findingLines: { key: string; text: string }[] = []; + for (let index = 0; index < (approval?.findingCount ?? 0); index += 1) { + const finding = approval?.finding(index); + if (finding !== undefined) { + findingLines.push( + { key: `drawer:reply-approval:rule:${index}`, text: finding[0] }, + { key: `drawer:reply-approval:excerpt:${index}`, text: finding[1] }, + ); + } + } + const fittedApproval = yield* fitPlain(renderer, view.size, findingLines, widths.drawer); + if (!fittedApproval.ok) { + return Err(new ReplFitRefusal("The reply approval text could not be fitted.")); + } + // Only the terminal paint receives the fitted text. The mounted tree and its + // layout get the same row identities and lengths with empty cells. + const approvalCells = new Map( + fittedApproval.value.map((row) => [row.key, row.text.padEnd(widths.drawer, " ")]), + ); const asked = previewedMessage(view); const preview = - asked === undefined || widths.drawer < 1 - ? Ok(undefined) - : yield* fitPlain( - renderer, - view.size, - asked.split("\n").map((text, offset) => ({ - key: `message:${offset}`, - text, - runs: sourceRuns(text), - })), - widths.drawer, - ); + approval !== undefined + ? Ok(fittedApproval.value.map((row) => ({ ...row, text: "", runs: [] }))) + : asked === undefined || widths.drawer < 1 + ? Ok(undefined) + : yield* fitPlain( + renderer, + view.size, + asked.split("\n").map((text, offset) => ({ + key: `message:${offset}`, + text, + runs: sourceRuns(text), + })), + widths.drawer, + ); if (!preview.ok) { return preview; } @@ -6324,7 +6415,12 @@ export function* prepareFrame( if (!edited.ok) { return edited; } - return Ok({ context, admission, presentation: presentationFor(view, context) }); + return Ok({ + context, + admission, + presentation: presentationFor(view, context), + ...(approval === undefined ? {} : { approvalCell: (line: string) => approvalCells.get(line) }), + }); } /** diff --git a/packages/cli/src/repl/sidekick/admission.ts b/packages/cli/src/repl/sidekick/admission.ts index 6650ae14a..7a1221af5 100644 --- a/packages/cli/src/repl/sidekick/admission.ts +++ b/packages/cli/src/repl/sidekick/admission.ts @@ -408,6 +408,7 @@ export function controlJournal( order, component: occurrence.component, arguments: occurrence.arguments, + ...(occurrence.elicit === undefined ? {} : { elicit: occurrence.elicit }), ...(reserved === undefined ? {} : { submit: reserved }), }); return order; diff --git a/packages/cli/src/repl/sidekick/control-outcomes.ts b/packages/cli/src/repl/sidekick/control-outcomes.ts new file mode 100644 index 000000000..7ced41d4d --- /dev/null +++ b/packages/cli/src/repl/sidekick/control-outcomes.ts @@ -0,0 +1,184 @@ +import { + elicitationEffectName, + readElicitationAnswer, + readGeneratedRootAdmission, +} from "@executablemd/core/host"; +import { parseDurableEvent, SOURCE_POSITION_FIELD } from "@executablemd/durable-streams"; +import type { Yield } from "@executablemd/durable-streams"; +import { Err, Ok } from "effection"; +import type { Result } from "effection"; +import { SidekickHistoryError } from "./history.ts"; +import type { ControlOutcomeRecord, ControlRecord, SidekickRecord } from "./history.ts"; + +export function controlSlot(conversation: string, response: string, order: number): string { + return `${conversation}\0${response}\0${order}`; +} + +interface Coroutine { + index: number; + closed: boolean; + admission?: Yield; + asking?: ControlRecord; +} + +export interface ControlOutcome { + readonly intent: ControlRecord; + readonly outcome: ControlOutcomeRecord; + readonly at: number; +} + +function generatedRoot(id: string): string | undefined { + return /^(generated_root:[1-9][0-9]*)(?:\..+)?$/.exec(id)?.[1]; +} + +/** Resolve operations in the order their facts were acknowledged. */ +export function acknowledgedControlOutcomes( + records: readonly SidekickRecord[], +): Result> { + const outcomes = new Map(); + const intents = new Map(); + const coroutines = new Map(); + function coroutine(conversation: string, id: string): Coroutine { + const key = `${conversation}\0${id}`; + let state = coroutines.get(key); + if (state === undefined) { + state = { index: 0, closed: false }; + coroutines.set(key, state); + } + return state; + } + const refused = (detail = "has inconsistent control outcome associations") => + Err(new SidekickHistoryError(`this chat history ${detail}.`)); + + for (const [at, record] of records.entries()) { + if (record.kind === "control") { + const key = controlSlot(record.conversation, record.response, record.order); + if (intents.has(key)) { + return refused(`records two operations at one position (record ${at + 1})`); + } + intents.set(key, record); + const ref = record.elicit; + if (ref !== undefined) { + const rootId = generatedRoot(ref.coroutineId); + if (rootId === undefined) { + return refused(); + } + const root = coroutine(record.conversation, rootId); + const site = coroutine(record.conversation, ref.coroutineId); + if ( + root.closed || + site.closed || + site.index !== ref.index || + root.admission === undefined || + readGeneratedRootAdmission(root.admission, record.response) === undefined + ) { + return refused(); + } + site.asking = record; + } + } else if (record.kind === "control-outcome") { + const key = controlSlot(record.conversation, record.response, record.order); + const intent = intents.get(key); + if (intent === undefined) { + return refused( + `records what an operation answered without recording the operation (record ${at + 1})`, + ); + } + if (intent.elicit !== undefined || outcomes.has(key)) { + return refused(`answers one operation twice (record ${at + 1})`); + } + outcomes.set(key, { intent, outcome: record, at }); + } else if (record.kind === "event") { + const parsed = parseDurableEvent(record.event); + if (!parsed.ok) { + return refused(); + } + const event = parsed.value; + const rootId = generatedRoot(event.coroutineId); + if (rootId === undefined) { + continue; + } + const root = coroutine(record.conversation, rootId); + const site = coroutine(record.conversation, event.coroutineId); + if (event.type === "close") { + site.closed = true; + site.asking = undefined; + continue; + } + const index = site.index++; + if (event.coroutineId === rootId && index === 0) { + root.admission = event; + } + const intent = site.asking; + site.asking = undefined; + if (intent?.elicit === undefined || event.description.type !== "elicit") { + continue; + } + const source = + root.admission === undefined + ? undefined + : readGeneratedRootAdmission(root.admission, intent.response); + const key = controlSlot(intent.conversation, intent.response, intent.order); + if ( + root.closed || + site.closed || + source === undefined || + outcomes.has(key) || + !matchesQuestion(event, intent, source) + ) { + return refused(); + } + if (event.result.status === "cancelled") { + continue; + } + const result = event.result; + const observed = result.status === "ok" ? readElicitationAnswer(event, index) : undefined; + if (observed !== undefined && !observed.ok) { + return observed; + } + outcomes.set(key, { + intent, + at, + outcome: { + kind: "control-outcome", + conversation: intent.conversation, + response: intent.response, + order: intent.order, + outcome: result.status === "ok" ? { status: "ok" } : result, + ...(observed === undefined ? {} : { observed: observed.value }), + }, + }); + } + } + return Ok(outcomes); +} + +function matchesQuestion(event: Yield, intent: ControlRecord, source: string): boolean { + const ref = intent.elicit; + const position = event.description[SOURCE_POSITION_FIELD]; + if ( + ref === undefined || + event.description.name !== ref.name || + event.description.input !== ref.fingerprint || + position === null || + typeof position !== "object" || + Array.isArray(position) || + Object.keys(position).length !== 4 || + position.generatedSource !== intent.response || + typeof position.offset !== "number" || + !Number.isSafeInteger(position.offset) || + position.offset < 0 || + typeof position.line !== "number" || + typeof position.column !== "number" + ) { + return false; + } + const { offset, line, column } = position; + const prefix = source.slice(0, offset); + return ( + /^])/.test(source.slice(offset)) && + line === prefix.split("\n").length && + column === offset - prefix.lastIndexOf("\n") && + ref.name === elicitationEffectName({ offset, line, column }) + ); +} diff --git a/packages/cli/src/repl/sidekick/controls.ts b/packages/cli/src/repl/sidekick/controls.ts index 3e08cb78a..adaf2d103 100644 --- a/packages/cli/src/repl/sidekick/controls.ts +++ b/packages/cli/src/repl/sidekick/controls.ts @@ -62,7 +62,7 @@ import type { } from "@executablemd/core"; import type { Json } from "@executablemd/durable-streams"; -import type { RetainedPrefix } from "./history.ts"; +import type { ControlRecord, RetainedPrefix } from "./history.ts"; import type { OutputRead, OutputReadRequest } from "./output.ts"; import type { EntryJournalPage, @@ -175,6 +175,7 @@ export interface SubmitReceipt { export interface ControlOccurrence { readonly component: string; readonly arguments: Json; + readonly elicit?: ControlRecord["elicit"]; } /** diff --git a/packages/cli/src/repl/sidekick/feedback.ts b/packages/cli/src/repl/sidekick/feedback.ts index caa5409b9..60c1f73b5 100644 --- a/packages/cli/src/repl/sidekick/feedback.ts +++ b/packages/cli/src/repl/sidekick/feedback.ts @@ -35,9 +35,12 @@ import { Err, Ok } from "effection"; import type { Operation, Result } from "effection"; +import type { FeedbackSlice } from "@executablemd/core/host"; import { DESIGNATED, DRAFT_FILLED, renderRead, renderReceipt } from "./controls.ts"; -import type { ControlOutcomeRecord, ControlRecord } from "./history.ts"; +import { acknowledgedControlOutcomes, controlSlot } from "./control-outcomes.ts"; +import type { ControlOutcome } from "./control-outcomes.ts"; +import type { ControlOutcomeRecord, ControlRecord, ElicitReference } from "./history.ts"; import type { OutputRead } from "./output.ts"; import { describeGuideIndex, readSidekickBootstrap, sidekickGuideIndex } from "./guide.ts"; import { fingerprint } from "./history.ts"; @@ -157,16 +160,20 @@ export function turnRequest(input: { readonly recovered: string | undefined; readonly draft: string; readonly message: string; -}): string { +}): { text: string; recoveredAt: number | undefined } { const parts: string[] = []; if (input.context !== undefined) { parts.push(input.context.bootstrap, input.context.index, input.context.surface); } + const recoveredAt = + input.recovered === undefined + ? undefined + : parts.reduce((length, part) => length + part.length + 2, 0); if (input.recovered !== undefined) { parts.push(input.recovered); } parts.push(describeDraft(input.draft), `The person says:\n\n${input.message}`); - return parts.join("\n\n"); + return { text: parts.join("\n\n"), recoveredAt }; } /** @@ -190,6 +197,7 @@ export function describeDraft(draft: string): string { /** One designated operation's result, as the next turn is told about it. */ export interface DesignatedResult { + readonly elicit?: ElicitReference; /** Its position among this response's controls, in admission order. */ readonly order: number; /** The control that produced it, spelled as the reply wrote it. */ @@ -246,7 +254,27 @@ export function designatedResults( }[] = [], designated: ReadonlySet = DESIGNATED, ): readonly DesignatedResult[] { - const intents = new Map(); + return resultsFrom(outcomesOf(records), at, refused, designated); +} + +function outcomesOf(records: readonly SidekickRecord[]): ReadonlyMap { + const outcomes = acknowledgedControlOutcomes(records); + if (!outcomes.ok) { + throw outcomes.error; + } + return outcomes.value; +} + +function resultsFrom( + outcomes: ReadonlyMap, + at: { readonly conversation: string; readonly response: string }, + refused: readonly { + readonly order: number; + readonly component: string; + readonly reason: string; + }[], + designated: ReadonlySet, +): readonly DesignatedResult[] { const results: DesignatedResult[] = refused // Through the same set, so a control that is not designated does not get // its refusals delivered either. One set decides what comes back. @@ -258,24 +286,14 @@ export function designatedResults( text: held.reason, json: false, })); - for (const record of records) { - if (record.kind === "control") { - if (record.conversation === at.conversation && record.response === at.response) { - intents.set(record.order, record); - } - continue; - } - if (record.kind !== "control-outcome") { - continue; - } - if (record.conversation !== at.conversation || record.response !== at.response) { + for (const { intent, outcome } of outcomes.values()) { + if (intent.conversation !== at.conversation || intent.response !== at.response) { continue; } - const intent = intents.get(record.order); - if (intent === undefined || !designated.has(intent.component)) { + if (!designated.has(intent.component)) { continue; } - results.push({ order: record.order, ...answered(intent, record) }); + results.push({ order: intent.order, ...answered(intent, outcome) }); } // Stable, so a refusal and the pair that followed it keep the order they // happened in: the refusal was pushed first and holds the order that pair @@ -307,21 +325,24 @@ export function recoveredWork( records: readonly SidekickRecord[], at: { readonly conversation: string; readonly response: string }, designated: ReadonlySet = DESIGNATED, + withheld: ReadonlySet = new Set(), ): string | undefined { const items: { order: number; component: string; body: string }[] = []; - for (const result of designatedResults(records, at, [], designated)) { + const outcomes = outcomesOf(records); + for (const result of resultsFrom(outcomes, at, [], designated)) { items.push({ order: result.order, component: result.component, - body: - result.kind === "answered" + body: withheld.has(result.order) + ? "This operation settled. Its retained answer was withheld from recovery feedback." + : result.kind === "answered" ? result.json ? fenced(result.text) : result.text : `${refusal(result.component)}: ${result.text}`, }); } - for (const waiting of unanswered(records, at, designated)) { + for (const waiting of unanswered(records, outcomes, at, designated)) { items.push(waiting); } if (items.length === 0) { @@ -348,31 +369,22 @@ export function recoveredWork( */ function unanswered( records: readonly SidekickRecord[], + outcomes: ReadonlyMap, at: { readonly conversation: string; readonly response: string }, designated: ReadonlySet = DESIGNATED, ): { order: number; component: string; body: string }[] { - const intents = new Map(); - const settled = new Set(); - for (const record of records) { - if (record.kind === "control") { - if (record.conversation === at.conversation && record.response === at.response) { - intents.set(record.order, record); - } - continue; - } - if (record.kind !== "control-outcome") { - continue; - } - if (record.conversation === at.conversation && record.response === at.response) { - settled.add(record.order); - } - } const waiting: { order: number; component: string; body: string }[] = []; - for (const [order, intent] of intents) { - if (settled.has(order) || !designated.has(intent.component)) { + for (const intent of records) { + if ( + intent.kind !== "control" || + intent.conversation !== at.conversation || + intent.response !== at.response || + !designated.has(intent.component) || + outcomes.has(controlSlot(intent.conversation, intent.response, intent.order)) + ) { continue; } - waiting.push({ order, component: intent.component, body: asking(intent) }); + waiting.push({ order: intent.order, component: intent.component, body: asking(intent) }); } return waiting; } @@ -453,6 +465,7 @@ function answered( kind: "answered", text: JSON.stringify(record.observed ?? null, null, 2), json: true, + elicit: intent.elicit, }; } } @@ -516,7 +529,7 @@ function submitted( export function sealedFeedback( results: readonly DesignatedResult[], failure?: string, -): string | undefined { +): { text: string; slices: readonly FeedbackSlice[] } | undefined { if (results.length === 0 && failure === undefined) { // Nothing was asked for and nothing went wrong, so there is nothing to say // and no turn to spend saying it. The person has the reply; it is their @@ -524,35 +537,50 @@ export function sealedFeedback( return undefined; } const parts: string[] = []; + const slices: FeedbackSlice[] = []; + let length = 0; + function append(text: string, result?: DesignatedResult, offset = 0): void { + if (result?.elicit !== undefined) { + const start = length + offset; + const { coroutineId, index } = result.elicit; + slices.push({ coroutineId, index, start, end: start + result.text.length }); + } + length += text.length + 2; + parts.push(text); + } if (results.length === 1 && failure === undefined) { const only = results[0]; if (only !== undefined) { // One answer to one question, with nothing round it — not even a fence // around a JSON value, because the whole message is that value and a // reply can parse it as it stands. - parts.push(only.kind === "answered" ? only.text : `${refusal(only.component)}: ${only.text}`); + if (only.kind === "answered") { + append(only.text, only); + } else { + append(`${refusal(only.component)}: ${only.text}`); + } } } else { for (const result of results) { - const body = - result.kind === "answered" - ? result.json - ? // Fenced, because this one is sitting among other things now and a - // reply has to know where the value ends. Longer than any run inside - // it, since a published string may hold a fence of its own. - fenced(result.text) - : result.text - : `${refusal(result.component)}: ${result.text}`; - parts.push(`## ${result.component}\n\n${body}`); + const heading = `## ${result.component}\n\n`; + if (result.kind !== "answered") { + append(`${heading}${refusal(result.component)}: ${result.text}`); + } else if (result.json) { + const fence = jsonFence(result.text); + const opening = `${heading}${fence}json\n`; + append(`${opening}${result.text}\n${fence}`, result, opening.length); + } else { + append(heading + result.text, result, heading.length); + } } } if (failure !== undefined) { - parts.push( + append( `Your response did not finish: ${failure}\n\nWhat it did before that stands. ` + "Continue from there rather than repeating it.", ); } - return parts.join("\n\n"); + return { text: parts.join("\n\n"), slices }; } /** @@ -569,10 +597,14 @@ function refusal(component: string): string { /** One JSON value in a fence longer than any backtick run inside it. */ function fenced(value: string): string { - const fence = "`".repeat(Math.max(3, longestRun(value, "`") + 1)); + const fence = jsonFence(value); return [`${fence}json`, value, fence].join("\n"); } +function jsonFence(value: string): string { + return "`".repeat(Math.max(3, longestRun(value, "`") + 1)); +} + /** The digest of exactly these bytes, which is what a version is. */ function* digest(text: string): Operation { return (yield* fingerprint(text)).sha256; diff --git a/packages/cli/src/repl/sidekick/history.ts b/packages/cli/src/repl/sidekick/history.ts index 8ae2eebe2..ae523bd59 100644 --- a/packages/cli/src/repl/sidekick/history.ts +++ b/packages/cli/src/repl/sidekick/history.ts @@ -304,6 +304,7 @@ export interface ControlRecord { readonly order: number; readonly component: string; readonly arguments: unknown; + readonly elicit?: ElicitReference; /** Submit alone binds the slot it reserved, so a crash can be reconciled. */ readonly submit?: { /** @@ -325,6 +326,13 @@ export interface ControlRecord { }; } +export interface ElicitReference { + readonly coroutineId: string; + readonly index: number; + readonly name: string; + readonly fingerprint: string; +} + /** What one control answered, written once it has answered. */ export interface ControlOutcomeRecord { readonly kind: "control-outcome"; @@ -719,7 +727,7 @@ function control(value: Record): ControlRecord | undefined { !exactly( value, ["kind", "conversation", "response", "order", "component", "arguments"], - ["submit"], + ["submit", "elicit"], ) ) { return undefined; @@ -731,6 +739,26 @@ function control(value: Record): ControlRecord | undefined { if (held === undefined || named === undefined || order === undefined || component === undefined) { return undefined; } + let elicit: ElicitReference | undefined; + if (Object.hasOwn(value, "elicit")) { + const ref = value.elicit; + if ( + component !== "Elicit" || + Object.hasOwn(value, "submit") || + !plain(ref) || + !exactly(ref, ["coroutineId", "index", "name", "fingerprint"]) || + typeof ref.coroutineId !== "string" || + typeof ref.name !== "string" || + typeof ref.fingerprint !== "string" + ) { + return undefined; + } + const index = whole(ref.index); + if (index === undefined) { + return undefined; + } + elicit = { coroutineId: ref.coroutineId, index, name: ref.name, fingerprint: ref.fingerprint }; + } let submit: ControlRecord["submit"]; if (Object.hasOwn(value, "submit")) { if ( @@ -787,6 +815,7 @@ function control(value: Record): ControlRecord | undefined { order, component, arguments: value.arguments, + ...(elicit === undefined ? {} : { elicit }), ...(submit === undefined ? {} : { submit }), }; } diff --git a/packages/cli/src/repl/sidekick/model.ts b/packages/cli/src/repl/sidekick/model.ts index 90edb5d6e..e73818c7a 100644 --- a/packages/cli/src/repl/sidekick/model.ts +++ b/packages/cli/src/repl/sidekick/model.ts @@ -38,11 +38,17 @@ import type { MessageOrigin } from "./history.ts"; import { Err, Ok } from "effection"; import type { Result } from "effection"; import { previewGeneratedXmdRoot } from "@executablemd/core/host"; -import { AGENT_PROMPT, parsePromptRecord } from "@executablemd/agent/host"; +import { + acceptedReplies, + AGENT_PROMPT, + AGENT_PROMPT_REPLY, + parsePromptRecord, +} from "@executablemd/agent/host"; import type { SessionConfiguration } from "@executablemd/agent/api"; import type { PromptRecord } from "@executablemd/agent/host"; import { projectUsage, type ReplUsage } from "../usage.ts"; import { parseDurableEvent } from "@executablemd/durable-streams"; +import type { DurableEvent } from "@executablemd/durable-streams"; import type { ConversationRecord, @@ -315,6 +321,14 @@ export interface SidekickSavedTurn { readonly incomplete: { readonly output: string; readonly failure: string } | undefined; /** The turn completed and secret detection withheld its reply, so it has none. */ readonly withheld?: true; + /** + * The turn's withheld reply was accepted by the person in front of the REPL, + * and its exact text is saved after the withheld completion. + * + * The same turn, completed successfully: `reply` is its evaluated response + * when one was recorded, and absent when the process ended before it ran. + */ + readonly accepted?: true; } /** One control a response ran, and what became of it. */ @@ -870,6 +884,15 @@ export const SAVED_WITHOUT_REPLY = "Turn saved; no evaluated reply recorded."; */ export const REPLY_WITHHELD = "Turn saved; its reply was withheld by secret detection and not run."; +/** + * What a turn whose withheld reply was accepted says, before anything ran it. + * + * The person accepted the reply and its exact text is saved; the process + * ended before its evaluated response was recorded, and nothing runs it on + * the turn's behalf now. + */ +export const REPLY_ACCEPTED = "Turn saved; its reply was accepted and saved, and was not run."; + /** * The input a canonical description carries, as text. * @@ -936,6 +959,12 @@ export function projectHistory(records: readonly SidekickRecord[]): Result [ + outcome.at, + { key, ...outcome }, + ]), + ); /** Each request by the record that began it. */ const began = new Map( requests.value.map((request) => [request.at, request]), @@ -1043,6 +1072,11 @@ export function projectHistory(records: readonly SidekickRecord[]): Result; + /** + * This conversation's root yields, in history order, for the one rule + * that joins an acceptance to its withheld turn. + */ + roots: DurableEvent[]; } >(); const order: string[] = []; @@ -1077,6 +1111,7 @@ export function projectHistory(records: readonly SidekickRecord[]): Result(); @@ -1152,6 +1187,22 @@ export function projectHistory(records: readonly SidekickRecord[]): Result(), at: new Map(), verified: new Map(), + roots: [], }); order.push(record.conversation); } else { @@ -1237,6 +1289,34 @@ export function projectHistory(records: readonly SidekickRecord[]): Result one.conversation === record.conversation && one.name === name, + ); + if (amended !== undefined && joined.value.has(name)) { + amended.withheld = false; + amended.accepted = true; + } + break; + } if (canonical.type !== "yield" || canonical.description.type !== AGENT_PROMPT) { break; } @@ -1282,6 +1362,7 @@ export function projectHistory(records: readonly SidekickRecord[]): Result } | undefined; + /** + * The request whose withheld reply may be reviewed, and how its delivery + * is settled first. + * + * Registered beside `preparingInput`, for the same request slot, and asked + * only by the review of the turn that slot's `begin()` handle names. + */ + let reviewingInput: + | { + readonly at: ResponseAt; + settle(): Operation; + accepted(hold: AgentReplyHold): void; + } + | undefined; + /** The question waiting on the person, where one is. */ + let approval: SidekickApproval | undefined; + let approvalOrder = 0; /** Which response each begun turn belongs to, by the handle it began with. */ const begun = new WeakMap(); @@ -924,7 +1026,13 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation { + yield* useAgentReplyReview({ + *review(hold, decide): Operation { + const slot = + typeof hold.begun === "object" && hold.begun !== null + ? begun.get(hold.begun) + : undefined; + const owed = reviewingInput; + if ( + input.reviewsReplies !== true || + slot === undefined || + owed === undefined || + owed.at !== slot || + watching === undefined || + watching.at !== slot + ) { + return; + } + reviewingInput = undefined; + const turn = yield* owed.settle(); + if (turn === undefined) { + return; + } + const decided = withResolvers(); + let waiting = true; + let accepting = false; + let revealed: number | undefined; + const asked: SidekickApproval = Object.freeze({ + key: `sidekick-approval:${++approvalOrder}`, + conversation: slot.conversation, + response: slot.response, + turn, + findings: hold.findings, + reveal: (finding: number) => (waiting ? hold.reveal(finding) : undefined), + decide(decision: AgentReplyDecision): boolean { + if (!waiting || approval !== asked) { + return false; + } + waiting = false; + accepting = decision === "accept"; + decided.resolve(decision); + moved(); + return true; + }, + get accepting() { + return accepting; + }, + get revealed() { + return waiting ? revealed : undefined; + }, + toggleReveal(index: number): boolean { + if (!waiting || approval !== asked || hold.findings[index] === undefined) { + return false; + } + revealed = revealed === index ? undefined : index; + moved(); + return true; + }, + }); + yield* ensure(() => { + waiting = false; + if (approval === asked) { + approval = undefined; + moved(); + } + }); + approval = asked; + moved(); + const decision = yield* decided.operation; + if (decision === "accept") { + owed.accepted(hold); + } + yield* decide(decision); + }, + }); + } + /** * Re-read this history. * @@ -1180,7 +1376,13 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation watcher() }], + installations: [ + privacy, + ...installations, + capturing, + { install: () => watcher() }, + { install: () => reviewer() }, + ], }); ready.resolve(opened); if (!opened.ok) { @@ -2395,6 +2597,7 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation(); + function* checkSidekickText(value: unknown): Operation { + const cleared = yield* handle.checkSidekickText(JSON.stringify(value)); + if (!cleared.ok) { + throw cleared.error; + } + } const journal: ControlJournal = { *began(occurrence, reserved): Operation { pairs += 1; let order: number; try { + yield* checkSidekickText({ arguments: occurrence.arguments, source: reserved?.source }); order = yield* written.began(occurrence, reserved); } catch (error) { if (occurrence.component === "REPL.ImportAsset") { @@ -2485,6 +2696,10 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation { try { + yield* checkSidekickText({ + outcome: outcome.ok ? undefined : failedOutcome(outcome.error), + observed, + }); yield* written.settled(order, outcome, observed); } catch (error) { if (imports.has(order)) { @@ -2567,6 +2782,7 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation(); + for (const outcome of designatedResults(records, held, [], designated)) { + if (!(yield* handle.checkSidekickText(outcome.text)).ok) { + withheld.add(outcome.order); + } + } + recovered = recoveredWork(records, held, designated, withheld); + } + const recoveryParts = [sending.options.recovery?.account, recovered].filter( + (part) => part !== undefined, + ); + const composed = turnRequest({ context, recovered: - [sending.options.recovery?.account, recovered, sending.finished?.text] + [...recoveryParts, sending.finished?.text] .filter((part) => part !== undefined) .join("\n\n") || undefined, draft: input.draft(), message: sending.text, }); + request = composed.text; + if (sending.finished !== undefined && composed.recoveredAt !== undefined) { + carryAt = + composed.recoveredAt + recoveryParts.reduce((n, part) => n + part.length + 2, 0); + } const delivering = sending.finished?.response ?? (recovered === undefined ? undefined : held?.response); if (delivering !== undefined) { @@ -2711,6 +2942,7 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation { + const saved = yield* turnForRequest(paired, began, conversation); + if (saved?.withheld !== true) { + return undefined; + } + // What a withheld completion settles, settled now rather than after + // the decision: the TODO batch it carried was delivered, and so was + // whatever input it carried, under its own turn. + todo?.finishPrompt(true); + confirmed = true; + relinquish(asked); + if (deliveringInput !== undefined) { + yield* paired.writer.append({ + kind: "feedback", + conversation, + response: deliveringInput, + turn: saved.turn, + text: request, + }); + } + reviewed = { turn: saved.turn }; + yield* refresh(); + if (watching === live) { + // The saved withheld turn is this row now, and nothing of its reply + // is shown while the person decides. + live.turn = saved.turn; + live.shown = ""; + live.reviewing = true; + moved(); + } + return saved.turn; + }, + accepted(hold: AgentReplyHold): void { + if (reviewed !== undefined) { + reviewed.hold = hold; + } + }, + }; + reviewingInput = reviewing; + yield* ensure(() => { + if (reviewingInput === reviewing) { + reviewingInput = undefined; + } + }); + // Where this text carries an accepted reply's selected answer, said to + // Core before the turn is sent; Core decides whether it carries anything. + const carry = sending.kind === "feedback" ? sending.carry : sending.finished?.carry; + if (carry !== undefined && carryAt !== undefined) { + const base = carryAt; + carry.hold.carry( + request, + carry.slices.map((slice) => ({ + ...slice, + start: base + slice.start, + end: base + slice.end, + })), + ); + } const replied = yield* handle.prompt( request, desired(settings.value, conversation), sending.kind === "feedback" ? sending.images : sending.finished?.images, + deliveringInput !== undefined, ); // A withheld reply is a completion the provider made, so the TODO batch // it carried was delivered — but only where Core said so with its own // refusal and this request's saved turn agrees. A withheld record beside // any other refusal proves nothing about this turn, and that turn's // batch stays pending for recovery like every other failure. + if ( + reviewed !== undefined && + !replied.ok && + replied.error instanceof AgentReplyWithheldError + ) { + // Asked, and not accepted: the withheld completion whose delivery was + // settled before the question is this exchange's whole outcome. That is + // so whatever ended the asking — Withhold, a close, or an acceptance + // that could not be saved. + yield* refresh(); + unwatch(); + return Ok({ + held: { + conversation, + turn: reviewed.turn, + response, + output: "", + hasOutput: false, + failure: undefined, + }, + answered: false, + withheld: true, + next: undefined, + }); + } const withheld = !replied.ok && replied.error instanceof AgentReplyWithheldError ? yield* turnForRequest(paired, began, conversation) : undefined; const completedWithheld = withheld?.withheld === true ? withheld : undefined; - todo?.finishPrompt(replied.ok || completedWithheld !== undefined); - confirmed = true; + if (!confirmed) { + todo?.finishPrompt(replied.ok || completedWithheld !== undefined); + confirmed = true; + } if (!replied.ok) { // No control runs: a partial, failed or cancelled response is source // nobody completed, and evaluating it would perform effects from a @@ -2914,11 +3240,14 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation + const evaluated = yield* asking.answering(response, journal, reviewed?.hold, () => // Named for the length of this one evaluation, and given up only once // the evaluation and everything it registered have joined — so the // controls that ran file their pairs under this response and a control @@ -2948,11 +3277,10 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation - handle.evaluateGeneratedXmdRoot({ - id: response, - source: replied.value, - allow: [...ceiling.allow], - }), + handle.evaluateGeneratedXmdRoot( + { id: response, source: replied.value, allow: [...ceiling.allow] }, + reviewed?.hold, + ), ), ); if (contributedFailure !== undefined) { @@ -2981,12 +3309,16 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation 0 + ? { hold: acceptedHold, slices: sealed.slices } + : undefined; if (submitted !== undefined) { const round = finishedRound(root, submitted); if (!round.ok) { @@ -3023,6 +3361,7 @@ export function useSidekickOwner(input: SidekickOwnerInput): Operation(journal: ControlJournal, work: () => Operation): Operation; + answering( + response: string, + journal: ControlJournal, + hold: AgentReplyHold | undefined, + work: () => Operation, + ): Operation; } /** @@ -98,12 +110,19 @@ export interface SidekickAsking { */ export function* useSidekickQuestions(): Operation { /** The response being evaluated right now, and therefore asking. */ - let current: ControlJournal | undefined; + let current: + | { + readonly response: string; + readonly journal: ControlJournal; + readonly hold?: AgentReplyHold; + } + | undefined; + const selections = new WeakMap(); yield* Elicitation.around({ *elicit([request]: [ElicitationRequest], next) { - const journal = current; - if (journal === undefined) { + const owner = current; + if (owner === undefined) { // Fails closed, and before anybody is asked. A question whose answer // has nowhere to be recorded is a question the reply that asked it // would never be told the answer to, so asking it would waste a @@ -113,6 +132,33 @@ export function* useSidekickQuestions(): Operation { "evaluating none — so there is no response for an answer to belong to.", ); } + const { journal, response } = owner; + const expansion = yield* questionExpansion(); + if (expansion?.name === ELICIT_CONTROL && expansion.position?.generatedSource === response) { + const position = yield* durablePosition(); + const message = (yield* retainsOptionalText(request.message)) + ? request.message + : "The original elicitation message was withheld."; + const elicit = { + ...position, + name: elicitationEffectName(expansion.position), + fingerprint: questionFingerprint(request), + }; + yield* journal.began({ + component: ELICIT_CONTROL, + arguments: { message }, + elicit, + }); + const answer = yield* next(request); + if (owner.hold !== undefined && typeof answer === "object" && answer !== null) { + const chosen = selections.get(answer); + selections.delete(answer); + for (const selection of chosen ?? []) { + owner.hold.selectedElicitation({ ...elicit, ...selection }); + } + } + return answer; + } // Before the asking. A question waiting when a process stops has an // intent and no outcome, which is exactly what an interrupted operation // looks like everywhere else in this history. @@ -149,14 +195,21 @@ export function* useSidekickQuestions(): Operation { // No waits reading. `ReplWaits` counts what an *entry* is held on, and a // Sidekick question holds no entry: it holds the response that asked it. - const questions = yield* useReplElicitation(); + const questions = yield* useReplElicitation(undefined, (answer, field, option) => { + selections.set(answer, [...(selections.get(answer) ?? []), { field, option }]); + }); return { questions, - answering(journal: ControlJournal, work: () => Operation): Operation { + answering( + response: string, + journal: ControlJournal, + hold: AgentReplyHold | undefined, + work: () => Operation, + ): Operation { return scoped(function* () { const previous = current; - current = journal; + current = { response, journal, hold }; yield* ensure(() => { current = previous; }); @@ -165,3 +218,11 @@ export function* useSidekickQuestions(): Operation { }, }; } + +function* questionExpansion(): Operation { + try { + return yield* getExpansion(); + } catch { + return undefined; + } +} diff --git a/packages/cli/src/repl/sidekick/recovery-state.ts b/packages/cli/src/repl/sidekick/recovery-state.ts index 411a264e3..904e9f591 100644 --- a/packages/cli/src/repl/sidekick/recovery-state.ts +++ b/packages/cli/src/repl/sidekick/recovery-state.ts @@ -5,6 +5,8 @@ import { parseDurableEvent } from "@executablemd/durable-streams"; import type { SidekickRecord } from "./history.ts"; import type { SidekickRequest } from "./model.ts"; import type { RecoveryDecisionRecord, RecoveryReferenceRecord } from "./recovery-record.ts"; +import { acknowledgedControlOutcomes, controlSlot } from "./control-outcomes.ts"; +import type { ControlOutcome } from "./control-outcomes.ts"; export interface HumanStopFact { readonly at: number; @@ -84,7 +86,13 @@ export function recoveryFacts( ): Result<{ readonly dispositions: readonly RecoveryDisposition[]; readonly stops: readonly HumanStopFact[]; + readonly outcomes: ReadonlyMap; }> { + const joined = acknowledgedControlOutcomes(records); + if (!joined.ok) { + return joined; + } + const outcomes = joined.value; const decisions = new Map(); const stops: HumanStopFact[] = []; const messages = new Set(); @@ -112,20 +120,19 @@ export function recoveryFacts( one.decision.controls.map((control) => `${control.response}\0${control.order}`), ), ); - return prefix.flatMap((one) => - one.kind === "control" && - one.conversation === conversation && - !previouslyRetired.has(`${one.response}\0${one.order}`) && - !prefix.some( - (outcome) => - outcome.kind === "control-outcome" && - outcome.conversation === one.conversation && - outcome.response === one.response && - outcome.order === one.order, - ) - ? [{ response: one.response, order: one.order }] - : [], - ); + return prefix.flatMap((one) => { + if ( + one.kind !== "control" || + one.conversation !== conversation || + previouslyRetired.has(`${one.response}\0${one.order}`) + ) { + return []; + } + const outcome = outcomes.get(controlSlot(conversation, one.response, one.order)); + return outcome !== undefined && outcome.at < index + ? [] + : [{ response: one.response, order: one.order }]; + }); } for (const [index, record] of records.entries()) { if (record.kind === "human-stop") { @@ -245,7 +252,7 @@ export function recoveryFacts( } } } - return Ok({ dispositions: [...decisions.values()], stops }); + return Ok({ dispositions: [...decisions.values()], stops, outcomes }); } export function recoveryBlocked( diff --git a/packages/cli/tests/protected-input-forms.test.ts b/packages/cli/tests/protected-input-forms.test.ts index c3e74a1e7..d56040bed 100644 --- a/packages/cli/tests/protected-input-forms.test.ts +++ b/packages/cli/tests/protected-input-forms.test.ts @@ -97,6 +97,7 @@ describe("protected provider observations before validation", () => { let failed = false; try { yield* asking.answering( + "protected-provider-test", { *began() { return 0; @@ -105,6 +106,7 @@ describe("protected provider observations before validation", () => { outcomes.push({ outcome, observed }); }, }, + undefined, () => elicit({ schema, message: "Enter input." }), ); } catch { diff --git a/packages/cli/tests/repl-sidekick-acp.test.ts b/packages/cli/tests/repl-sidekick-acp.test.ts index 70f34353b..001cba258 100644 --- a/packages/cli/tests/repl-sidekick-acp.test.ts +++ b/packages/cli/tests/repl-sidekick-acp.test.ts @@ -1,4 +1,4 @@ -import { InMemoryStream } from "@executablemd/durable-streams"; +import { DurablePersistenceError, InMemoryStream } from "@executablemd/durable-streams"; import { retainedSource } from "@executablemd/core"; import { executeInstalled } from "@executablemd/core/host"; import { @@ -9,7 +9,7 @@ import { runSidekickCommand } from "../src/sidekick-acp/command.ts"; import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { cliCommand, cliShellCommand, shellQuote } from "@executablemd/test-support/launch"; -import { readTextFile, writeTextFile } from "@effectionx/fs"; +import { copyFile, ensureDir, exists, readTextFile, writeTextFile } from "@effectionx/fs"; import { exec, ProcessApi, Stdio } from "@effectionx/process"; import { selectJson } from "@executablemd/core/host"; import { openReplSession } from "../src/repl/session.ts"; @@ -22,6 +22,7 @@ import { chmod, lstat, mkdtemp, + readFile, readdir, realpath, rename, @@ -70,22 +71,52 @@ import { createSidekickAdmission, useSidekickDispatch } from "../src/repl/sideki import { useSidekickAttachment } from "../src/sidekick-acp/server.ts"; import { requestAttachment } from "../src/sidekick-acp/client.ts"; import { usePosixSidekickTransport } from "../src/sidekick-acp/posix.ts"; -import { recordingTerminal, useMountedHost, MOUNTED_AGENT } from "./support/mounted-repl.ts"; import { + appendGate, + mountedExecution, + recordingTerminal, + useMountedHost, + MOUNTED_AGENT, +} from "./support/mounted-repl.ts"; +import { + chooseOption, + intoSecondField, drawnScreen, + openQuestion, sendMessage, + submitForm, intoComposer, typed, composerHolds, sidekickIdle, + scrollDrawerTo, } from "./support/screen-journeys.ts"; import { useReplPage } from "./support/repl-page.ts"; +import type { ReplPage } from "./support/repl-page.ts"; import { settledReading } from "./support/mounted-history.ts"; +import { projectSidekick } from "../src/repl/sidekick/model.ts"; import { awaitedFact } from "./support/mounted-repl.ts"; import { runReplProgram } from "../src/repl/program.ts"; +import { + initialState, + NO_AGENT, + presentedDrawer, + reduceRepl, + viewFor, +} from "../src/repl/application.ts"; +import type { ReplLive } from "../src/repl/application.ts"; +import { NO_LIFECYCLE } from "../src/repl/lifecycle.ts"; +import { resolvePointer } from "../src/repl/renderer.ts"; +import { usePresenter } from "./fixtures/repl/presentation.ts"; import { ReplHost } from "../src/repl/host.ts"; -import { PAIRED_DIRECTORY, parseEntryRecords } from "../src/repl/sidekick/history.ts"; -import { AGENT_PROMPT } from "@executablemd/agent/host"; +import { + pairedLocation, + PAIRED_DIRECTORY, + parseEntryRecords, + parseSidekickRecord, +} from "../src/repl/sidekick/history.ts"; +import { AGENT_PROMPT, AGENT_PROMPT_REPLY } from "@executablemd/agent/host"; +import { parseDurableEvent } from "@executablemd/durable-streams"; import { Descriptor, Request, @@ -172,7 +203,7 @@ function* world(options: Parameters[1] = {}, install?: () => } type World = ReturnType extends Operation ? T : never; function sdk( - world: World, + world: Pick, ): Operation<{ connection: acp.ClientConnection; updates: acp.SessionNotification[] }> { return resource(function* (provide) { let child: ChildProcess | undefined; @@ -193,7 +224,7 @@ function sdk( } }); child = spawnChild(world.launch.command, world.launch.arguments, { - cwd: world.held.workspace, + cwd: world.descriptor.cwd, env: { PATH: process.env.PATH, DENO_DIR: process.env.DENO_DIR, HOME: world.home }, stdio: ["pipe", "pipe", "inherit"], }); @@ -2031,54 +2062,92 @@ describe("minimal live Sidekick ACP attachment", () => { }); it("A6: unestablished feedback refuses another prompt without guessing or resend", function* () { - const one = yield* world(); - const a = attachResult.parse( - yield* requestAttachment(one.path, { method: "attach", cwd: one.descriptor.cwd }), - ); - let prompts = 0; - yield* ReplHost.around({ - *appendRecord([file, text], next) { - const row = z - .object({ kind: z.string(), event: z.string().optional() }) + const storageFailure = new Error("Controlled feedback acknowledgement loss."); + let verifiedExchange = false; + try { + yield* scoped(function* () { + const one = yield* world(); + const a = attachResult.parse( + yield* requestAttachment(one.path, { method: "attach", cwd: one.descriptor.cwd }), + ); + let prompts = 0; + let failedPrompt: string | undefined; + yield* ReplHost.around({ + *appendRecord([file, text], next) { + const row = parseSidekickRecord(text.trim()); + if (row?.kind === "event") { + for (const event of parseEntryRecords(row.event)) { + if (event.type === "yield" && event.description.type === AGENT_PROMPT) { + prompts++; + if (prompts === 2) { + failedPrompt = event.description.name; + throw storageFailure; + } + } + } + } + yield* next(file, text); + }, + }); + one.held.fake.script({ reply: "" }); + one.held.fake.script({ reply: "Backend accepted feedback." }); + const ended = exchangeSchema.parse( + yield* requestAttachment(one.path, { + method: "prompt", + attachment: a.attachment, + text: "inspect", + }), + ); + expect(ended.ending).toMatchObject({ + joined: true, + outcome: "failed", + error: "Failed to persist durable yield event", + }); + expect(one.held.fake.prompts).toHaveLength(one.baselineTurns + 2); + const status = z + .object({ feedbackUnestablished: z.number() }) .passthrough() - .parse(JSON.parse(text)); - if (row.kind === "event" && row.event !== undefined) { - const events = parseEntryRecords(row.event); - if ( - events.some( - (event) => event.type === "yield" && event.description.type === AGENT_PROMPT, - ) && - ++prompts === 2 - ) { - throw new Error("Controlled feedback acknowledgement loss."); - } + .parse(yield* requestAttachment(one.path, { method: "status" })); + expect(status.feedbackUnestablished).toBe(1); + const before = yield* records(one.held.execution); + expect(failedPrompt).toBeDefined(); + expect( + before.some( + (row) => + row.kind === "event" && + parseEntryRecords(row.event).some( + (event) => + event.type === "yield" && + event.description.type === AGENT_PROMPT && + event.description.name === failedPrompt, + ), + ), + ).toBe(false); + let refusal: unknown; + try { + yield* requestAttachment(one.path, { + method: "prompt", + attachment: a.attachment, + text: "do not resend", + }); + } catch (error) { + refusal = error; } - yield* next(file, text); - }, - }); - one.held.fake.script({ reply: "" }); - one.held.fake.script({ reply: "Backend accepted feedback." }); - yield* requestAttachment(one.path, { - method: "prompt", - attachment: a.attachment, - text: "inspect", - }); - const status = z - .object({ feedbackUnestablished: z.number() }) - .passthrough() - .parse(yield* requestAttachment(one.path, { method: "status" })); - expect(status.feedbackUnestablished).toBeGreaterThan(0); - const before = yield* records(one.held.execution); - const count = one.held.fake.prompts.length; - yield* refused( - requestAttachment(one.path, { - method: "prompt", - attachment: a.attachment, - text: "do not resend", - }), - ); - expect(one.held.fake.prompts).toHaveLength(count); - expect(yield* records(one.held.execution)).toEqual(before); + expect(errorOf(refusal).message).toBe( + "This conversation holds feedback whose acceptance cannot be established.", + ); + expect(one.held.fake.prompts).toHaveLength(one.baselineTurns + 2); + expect(yield* records(one.held.execution)).toEqual(before); + verifiedExchange = true; + }); + } catch (error) { + if (!verifiedExchange || !(error instanceof DurablePersistenceError)) { + throw error; + } + expect(error.cause).toBe(storageFailure); + return; + } + throw new Error("Expected the conversation release to report the failed append."); }); it("A6: the 64-update observer bound detaches a slow connection while its root producer continues", function* () { @@ -4581,3 +4650,786 @@ describe("Passive workflow supervision", () => { ); }); }); + +/** + * A completed reply secret detection withheld, decided by the foreground person. + * + * The real mounted REPL, a real attachment socket, the real scanner and the + * real journals; only the provider is a double. The synthetic credential is + * assembled at runtime, and every case reads the bytes actually written. + */ +describe("Sidekick reply approval", () => { + const value = ["Zx81", "qPm40", "aLw2tYu"].join(""); + const secret = `apiKey=${value}`; + const choice = { + type: "object", + properties: { choice: { type: "string", enum: [secret, "none"] } }, + required: ["choice"], + additionalProperties: false, + }; + const reply = + `Which value?\n\n` + + `Use ${secret}\n`; + + /** Every byte this execution's two journals hold. */ + function* journalBytes( + page: { repl: Parameters[0] }, + host: { + repository: { open(execution: string): Operation }; + }, + ): Operation { + const execution = yield* mountedExecution(page.repl); + const pair = (yield* host.repository.open(execution)) as { + chat: { form: string; location: { sidekick: string; entries: string } }; + }; + if (pair.chat.form !== "paired") { + throw new Error("Expected paired history."); + } + const sidekick = yield* until(readFile(pair.chat.location.sidekick, "utf8")); + const entries = yield* until(readFile(pair.chat.location.entries, "utf8").catch(() => "")); + return sidekick + entries; + } + + /** The canonical events of this execution's Sidekick, in history order. */ + function* canonical(page: { repl: Parameters[0] }) { + const read = yield* settledReading(page.repl); + return read.chat.flatMap((record) => { + if (record.kind !== "event") { + return []; + } + const parsed = parseDurableEvent(record.event); + return parsed.ok && parsed.value.type === "yield" ? [parsed.value] : []; + }); + } + + /** Those of them a turn, an acceptance or an admission is, by type and name. */ + function* rootRecords(page: { + repl: Parameters[0]; + }): Operation { + return (yield* canonical(page)).flatMap((event) => { + const type = event.description.type; + return type === AGENT_PROMPT || type === AGENT_PROMPT_REPLY || type === "generated_xmd" + ? [`${type} ${event.description.name}`] + : []; + }); + } + + function* verifyColdCut( + page: ReplPage, + verify: (read: ReturnType) => void, + ) { + const execution = yield* mountedExecution(page.repl); + const source = pairedLocation(page.repl.root, execution); + yield* scoped(function* () { + const cold = yield* useMountedHost(); + const copy = pairedLocation(cold.root, execution); + yield* ensureDir(copy.directory); + yield* copyFile(source.sidekick, copy.sidekick); + if (yield* exists(source.entries)) { + yield* copyFile(source.entries, copy.entries); + } + const before = yield* readTextFile(copy.sidekick); + const reopened = yield* useReplPage({ + host: cold, + location: `xmd://repl/${execution}/entries`, + }); + yield* reopened.until("the reopened history is idle", sidekickIdle); + verify(projectSidekick((yield* settledReading(reopened.repl)).chat)); + expect(cold.fake.prompts).toEqual([]); + expect(yield* readTextFile(copy.sidekick)).toBe(before); + expect(reopened.screen().shows("[accept reply]")).toBe(false); + }); + } + + it("Pending and exact Accept: private while asked, then the original runs once and reaches ACP and feedback", function* () { + yield* usePosixSidekickTransport(process.platform); + const home = yield* publishingHome("xmd-reply-approval-"); + const path = join(home, "attachment.json"); + const gate = appendGate(); + const host = yield* useMountedHost({ gate }); + host.fake.script({ reply: "Ready for attachment." }); + const page = yield* useReplPage({ host, sidekickAcp: path }); + const descriptor = Descriptor.parse(JSON.parse(yield* readTextFile(path))); + const { connection, updates } = yield* sdk({ + descriptor, + home, + launch: cliCommand(["sidekick", "acp", "--attach", path]), + }); + const session = yield* until( + connection.agent.request(acp.methods.agent.session.new, { + cwd: descriptor.cwd, + mcpServers: [], + }), + ); + host.fake.script({ reply, held: true }); + host.fake.script({ reply: "Feedback received." }); + const prompting = yield* spawn(() => + until( + connection.agent.request(acp.methods.agent.session.prompt, { + sessionId: session.sessionId, + prompt: [{ type: "text", text: "Pick the example value." }], + }), + ), + ); + yield* host.fake.startedTurns(2); + yield* page.press("TODO", { onRow: "SIDEKICK" }); + yield* page.until("Add task is visible", (one) => one.shows("+")); + yield* page.press("+"); + yield* page.until("the new TODO editor", (one) => one.shows("Task:")); + yield* page.type("Verify the composed input"); + yield* page.key("Enter"); + yield* page.until("the committed TODO edit", (one) => one.shows("☐ Verify the composed input")); + expect(host.fake.prompts[1]).not.toContain("Verify the composed input"); + yield* page.press("SIDEKICK", { onRow: "TODO" }); + host.fake.releaseTurn(); + yield* page.until("the withheld reply's question", (one) => one.shows("[accept reply]")); + + // Pending: the withheld completion is saved and its preview withdrawn, and + // nothing of the reply is anywhere but this screen's masked excerpt. + expect(host.fake.prompts).toHaveLength(2); + expect(page.screen().shows(value)).toBe(false); + expect(page.screen().shows("Which value?")).toBe(false); + const pending = yield* journalBytes(page, host); + expect(pending.includes(value)).toBe(false); + expect(pending.includes("Which value?")).toBe(false); + expect(pending.includes(AGENT_PROMPT_REPLY)).toBe(false); + const asked = yield* rootRecords(page); + // The bootstrap's own turn and evaluation, then this turn's withheld + // completion — and nothing after it: no acceptance, no admission. + expect(asked.at(-1)).toBe(`${AGENT_PROMPT} turn:2`); + expect(asked.filter((one) => one.startsWith(AGENT_PROMPT))).toEqual([ + `${AGENT_PROMPT} turn:1`, + `${AGENT_PROMPT} turn:2`, + ]); + const status = JSON.stringify(yield* requestAttachment(path, { method: "status" })); + expect(status.includes(value)).toBe(false); + expect(status.includes("Which value?")).toBe(false); + expect(updates.length).toBeGreaterThan(0); + expect(JSON.stringify(updates).includes(value)).toBe(false); + expect(JSON.stringify(updates).includes("Which value?")).toBe(false); + + for (const size of [ + { columns: 160, rows: 36 }, + { columns: 120, rows: 30 }, + { columns: 72, rows: 20 }, + ]) { + if (page.screen().size.columns !== size.columns || page.screen().size.rows !== size.rows) { + yield* page.resize(size); + } + yield* scrollDrawerTo(page, "[reveal match 1]"); + expect(page.screen().shows(value)).toBe(false); + yield* page.press("[reveal match 1]"); + const toggled = yield* page.until("Reveal applied", (one) => one.shows("[hide match 1]")); + if (!toggled.shows(value)) { + yield* page.key("PageUp"); + } + yield* page.until("the revealed match", (one) => one.shows(value)); + expect(yield* journalBytes(page, host)).toBe(pending); + yield* scrollDrawerTo(page, "[hide match 1]"); + yield* page.press("[hide match 1]"); + yield* page.until("the masked match", (one) => !one.shows(value)); + console.log( + `Approval drawer ${size.columns}x${size.rows}: pending and Reveal verified on the completed terminal frame.`, + ); + } + yield* page.resize({ columns: 160, rows: 36 }); + + // Accept, held at the acceptance's own append: nothing has run yet. + const held = gate.holdNext((_path, record) => record.includes(AGENT_PROMPT_REPLY)); + const admitting = gate.holdNext((_path, line) => { + const record = parseSidekickRecord(line.trim()); + if (record?.kind !== "event") { + return false; + } + const event = parseDurableEvent(record.event); + return ( + event.ok && event.value.type === "yield" && event.value.description.type === "generated_xmd" + ); + }); + yield* ensure(() => { + held.release(); + admitting.release(); + }); + yield* page.press("[accept reply]"); + yield* held.started(); + expect(page.screen().shows(`Use ${secret}`)).toBe(false); + expect((yield* journalBytes(page, host)).includes(value)).toBe(false); + held.release(); + yield* admitting.started(); + expect((yield* rootRecords(page)).at(-1)).toBe(`${AGENT_PROMPT_REPLY} turn:2`); + yield* verifyColdCut(page, (read) => { + if (!read.ok) { + throw read.error; + } + const turn = read.value.turns.find((one) => one.turn === 2); + expect(turn).toMatchObject({ accepted: true, status: "completed" }); + expect(turn?.reply).toBeUndefined(); + expect(read.value.chat.at(-1)).toMatchObject({ kind: "saved", turn: 2 }); + expect(read.value.operations).toEqual([]); + }); + admitting.release(); + + // The reply runs: its own question offers the choice it wrote. + yield* page.until("the accepted reply's question", (one) => one.shows("[answer sidekick]")); + yield* openQuestion(page); + const answering = gate.holdNext((_path, record) => + record.includes("executablemd.elicitation-schema"), + ); + const responding = gate.holdNext((_path, record) => record.includes('"kind":"response"')); + yield* chooseOption(page, secret); + yield* submitForm(page); + yield* answering.started(); + const pendingAnswer = projectSidekick((yield* settledReading(page.repl)).chat); + expect( + pendingAnswer.ok && pendingAnswer.value.unfinished.some((one) => one.component === "Elicit"), + ).toBe(true); + expect(host.fake.prompts).toHaveLength(2); + answering.release(); + yield* responding.started(); + const acknowledgedAnswer = yield* settledReading(page.repl); + const projectedAnswer = projectSidekick(acknowledgedAnswer.chat); + expect( + projectedAnswer.ok && + projectedAnswer.value.unfinished.some((one) => one.component === "Elicit"), + ).toBe(false); + expect( + projectedAnswer.ok && + projectedAnswer.value.operations + .filter((one) => one.component === "Elicit") + .map((one) => one.status), + ).toEqual(["ok"]); + expect(acknowledgedAnswer.chat.some((one) => one.kind === "control-outcome")).toBe(false); + expect(host.fake.prompts).toHaveLength(2); + responding.release(); + const answer = yield* prompting; + expect(answer.stopReason).toBe("end_turn"); + expect(answer._meta?.["executablemd.sidekick-exchange/v1"]).toMatchObject({ joined: true }); + const output = updates.flatMap(({ update }) => + update.sessionUpdate === "agent_message_chunk" && update.content.type === "text" + ? [update.content.text] + : [], + ); + expect(output.join("")).toBe(`Use ${secret}Feedback received.`); + yield* page.until("all Send work joined", sidekickIdle); + + // One provider turn for the reply — the same request and native turn — + // and one ordinary feedback turn carrying the selected value. + expect(host.fake.prompts).toHaveLength(3); + expect(host.fake.prompts[1]).toContain("Pick the example value."); + expect(host.fake.prompts[2]).toContain(`"choice": "${secret}"`); + const key = host.fake.turns[0]?.handle.sessionKey; + expect(host.fake.turns.map((one) => one.handle.sessionKey)).toEqual([key, key, key]); + const after = yield* rootRecords(page); + const evaluatedAs = (yield* settledReading(page.repl)).chat.flatMap((one) => + one.kind === "response" && one.turn >= 2 ? [one.response] : [], + ); + // Saved before it was admitted, and the same turn: no new ordinal between. + expect(after.slice(after.indexOf(`${AGENT_PROMPT} turn:2`))).toEqual([ + `${AGENT_PROMPT} turn:2`, + `${AGENT_PROMPT_REPLY} turn:2`, + `generated_xmd generated:${evaluatedAs[0]}`, + `${AGENT_PROMPT} turn:3`, + `generated_xmd generated:${evaluatedAs[1]}`, + ]); + const events = yield* canonical(page); + const accepted = events.find((one) => one.description.type === AGENT_PROMPT_REPLY); + expect( + accepted?.result.status === "ok" && + (accepted.result.value as { text?: string } | undefined)?.text, + ).toBe(reply); + const feedback = events.find( + (one) => one.description.type === AGENT_PROMPT && one.description.name === "turn:3", + ); + const composed = host.fake.prompts[2]; + expect(composed).toContain("Committed TODO changes (in commit order)"); + expect(composed).toContain('"origin":"human"'); + expect(composed).toContain("+- [ ] Verify the composed input"); + expect(feedback?.description.input).toBe(composed); + const read = yield* settledReading(page.repl); + const acknowledged = read.chat.filter((one) => one.kind === "feedback" && one.turn === 3); + expect(acknowledged).toEqual([expect.objectContaining({ text: composed })]); + const projection = projectSidekick(read.chat); + if (!projection.ok) { + throw projection.error; + } + expect(projection.value.delivered).toEqual([ + { conversation: projection.value.active, turn: 3 }, + ]); + const responses = read.chat.flatMap((one) => (one.kind === "response" ? [one] : [])); + expect(responses.find((one) => one.turn === 2)?.output).toBe(`Use ${secret}`); + }); + + for (const choiceFirst of [false, true]) { + it(`carries a selected choice through protected feedback with choice first: ${choiceFirst}`, function* () { + const gate = appendGate(); + const host = yield* useMountedHost({ gate }); + const marked = "synthetic-private-note-48"; + const mixed = { + ...choice, + properties: { ...choice.properties, note: { type: "string", writeOnly: true } }, + required: ["choice", "note"], + }; + host.fake.script({ + reply: `Pick and annotate.`, + }); + host.fake.script({ reply: "Feedback received." }); + const page = yield* useReplPage({ host }); + yield* sendMessage(page, "Ask for the choice and note."); + yield* page.until("reply approval", (one) => one.shows("[accept reply]")); + yield* page.press("[accept reply]"); + yield* page.until("the mixed question", (one) => one.shows("[answer sidekick]")); + yield* openQuestion(page); + const answering = gate.holdNext((_path, record) => + record.includes("executablemd.elicitation-schema"), + ); + yield* ensure(() => answering.release()); + if (choiceFirst) { + yield* chooseOption(page, secret); + } + yield* intoSecondField(page); + yield* page.type(marked); + if (choiceFirst) { + yield* submitForm(page); + } else { + yield* chooseOption(page, secret); + } + yield* answering.started(); + + const pending = projectSidekick((yield* settledReading(page.repl)).chat); + expect(pending.ok && pending.value.unfinished.some((one) => one.component === "Elicit")).toBe( + true, + ); + expect(host.fake.prompts).toHaveLength(1); + answering.release(); + yield* page.until("mixed answer feedback joined", sidekickIdle); + + function verify(read: ReturnType) { + if (!read.ok) { + throw read.error; + } + expect(read.value.unfinished).toEqual([]); + expect(read.value.operations.filter((one) => one.component === "Elicit")).toEqual([ + expect.objectContaining({ status: "ok" }), + ]); + } + + verify(projectSidekick((yield* settledReading(page.repl)).chat)); + yield* verifyColdCut(page, verify); + expect(host.fake.prompts).toHaveLength(2); + expect(host.fake.prompts[1]).toContain(`"choice": "${secret}"`); + expect(host.fake.prompts[1]).toContain('"note": "[redacted]"'); + expect(JSON.stringify(host.fake.prompts)).not.toContain(marked); + const bytes = yield* journalBytes(page, host); + expect(bytes).not.toContain(marked); + const answer = (yield* canonical(page)).find((event) => event.description.type === "elicit"); + expect(answer?.result).toMatchObject({ + status: "ok", + value: { + "executablemd.input": { + kind: "object", + members: { + choice: { kind: "json", value: secret }, + note: { kind: "input" }, + }, + }, + }, + }); + }); + } + + it("carries both explicitly selected fields into automatic feedback", function* () { + const other = `apiKey=${[...value].reverse().join("")}`; + const two = { + ...choice, + properties: { ...choice.properties, other: { type: "string", enum: [other, "skip"] } }, + required: ["choice", "other"], + }; + const host = yield* useMountedHost(); + host.fake.script({ + reply: `Pick two.`, + }); + host.fake.script({ reply: "Feedback received." }); + const page = yield* useReplPage({ host }); + yield* sendMessage(page, "Ask for two choices."); + yield* page.until("reply approval", (one) => one.shows("[accept reply]")); + yield* page.press("[accept reply]"); + yield* page.until("the two-choice question", (one) => one.shows("[answer sidekick]")); + yield* openQuestion(page); + yield* chooseOption(page, secret); + yield* chooseOption(page, other); + yield* page.until("both choices joined", sidekickIdle); + expect(host.fake.prompts).toHaveLength(2); + expect(host.fake.prompts[1]).toContain(`"choice": "${secret}"`); + expect(host.fake.prompts[1]).toContain(`"other": "${other}"`); + }); + + it("acknowledges a refused question and sends ordinary feedback", function* () { + const host = yield* useMountedHost(); + const unsupported = { ...choice, additionalProperties: true }; + host.fake.script({ + reply: `Pick one.`, + }); + host.fake.script({ reply: "Feedback received." }); + const page = yield* useReplPage({ host }); + yield* sendMessage(page, "Ask an unsupported question."); + yield* page.until("reply approval", (one) => one.shows("[accept reply]")); + yield* page.press("[accept reply]"); + yield* page.until("refusal feedback joined", sidekickIdle); + expect(host.fake.prompts).toHaveLength(2); + expect(host.fake.prompts[1]).toContain("Elicit failed:"); + expect(host.fake.prompts[1]).not.toContain(value); + const answer = (yield* canonical(page)).find((event) => event.description.type === "elicit"); + expect(answer?.result.status).toBe("err"); + expect(JSON.stringify(answer?.description)).toContain(secret); + expect(JSON.stringify(answer?.result)).not.toContain(value); + }); + + for (const size of [ + { columns: 160, rows: 36 }, + { columns: 120, rows: 30 }, + { columns: 72, rows: 20 }, + ]) { + it(`shows long Reveal and decision disclosure at ${size.columns}x${size.rows}`, function* () { + const long = value.repeat(5) + [...value].reverse().join(""); + const host = yield* useMountedHost(); + host.fake.script({ reply: `Use apiKey=${long} for the example.` }); + const page = yield* useReplPage({ host, size: { columns: 160, rows: 36 } }); + yield* sendMessage(page, "Check approval."); + yield* page.until("reply approval", (one) => one.shows("[reveal match]")); + if (size.columns !== 160) { + yield* page.resize(size); + } + const pending = page.screen(); + const positioned = pending.rows.map((row) => row.trim()).join(" "); + expect(positioned).toContain("CREDENTIAL_FIELD"); + expect(positioned).toContain("@executablemd/secretlint-rule-credentials"); + expect(positioned).toMatch(/line 1, column 20/); + expect(pending.shows(value)).toBe(false); + const retained = yield* journalBytes(page, host); + yield* scrollDrawerTo(page, "[reveal match]"); + yield* page.press("[reveal match]"); + const revealed = yield* page.until("long match revealed", (one) => one.shows(value)); + const pieces = revealed.rows.flatMap((row) => + (row.match(/[A-Za-z0-9]{8,}/g) ?? []).filter((word) => long.includes(word)), + ); + expect(pieces.join("")).toBe(long); + expect(yield* journalBytes(page, host)).toBe(retained); + yield* scrollDrawerTo(page, "[accept reply]"); + const decisions = page.screen(); + expect(decisions.shows("Accept saves the original and permits ACP Output")).toBe(true); + expect(decisions.shows("and automatic feedback.")).toBe(true); + expect(decisions.shows("[withhold reply]")).toBe(true); + expect(decisions.shows("[accept reply]")).toBe(true); + const decision = size.columns === 72 ? "[accept reply]" : "[withhold reply]"; + yield* page.press(decision); + yield* page.until("the decision joined", sidekickIdle); + const accepted = (yield* rootRecords(page)).includes(`${AGENT_PROMPT_REPLY} turn:1`); + expect(accepted).toBe(size.columns === 72); + expect(host.fake.prompts).toHaveLength(1); + console.log( + `Approval drawer ${size.columns}x${size.rows}: first rule and position, complete long match, disclosure and both mounted decisions verified.`, + ); + }); + } + + for (const choice of ["[withhold reply]", "[close]", "[stop]"]) { + it(`keeps the automatic follow-up withheld through ${choice} and a cold reopen`, function* () { + const host = yield* useMountedHost(); + host.fake.script({ reply: "Earlier work finished.\n" }); + host.fake.script({ reply }); + const execution = yield* scoped(function* () { + const page = yield* useReplPage({ host }); + yield* sendMessage(page, "Read the current bindings."); + yield* page.until("the automatic reply approval", (one) => one.shows("Reply withheld")); + const before = yield* settledReading(page.repl); + expect(before.chat.some((one) => one.kind === "feedback" && one.turn === 2)).toBe(true); + expect( + before.chat.some((one) => one.kind === "control" && one.component === "Elicit"), + ).toBe(false); + expect((yield* journalBytes(page, host)).includes(value)).toBe(false); + if (choice === "[close]") { + yield* verifyColdCut(page, (read) => { + if (!read.ok) { + throw read.error; + } + expect(read.value.turns.find((one) => one.turn === 2)?.withheld).toBe(true); + expect(read.value.delivered).toEqual([{ conversation: read.value.active, turn: 2 }]); + }); + } + yield* page.press(choice, choice === "[stop]" ? { onRow: "[close]" } : undefined); + yield* page.until( + "the withheld continuation joined", + (one) => !one.shows("Reply withheld") && sidekickIdle(one), + ); + expect(yield* rootRecords(page)).not.toContain(`${AGENT_PROMPT_REPLY} turn:2`); + expect(host.fake.prompts).toHaveLength(2); + return before.execution; + }); + yield* scoped(function* () { + const reopened = yield* useReplPage({ host, location: `xmd://repl/${execution}/entries` }); + const read = projectSidekick((yield* settledReading(reopened.repl)).chat); + expect(read.ok && read.value.turns.find((one) => one.turn === 2)?.withheld).toBe(true); + expect(reopened.screen().shows("[accept reply]")).toBe(false); + expect(host.fake.prompts).toHaveLength(2); + }); + }); + } + + it("rejects a stale mounted target and action, and an attached attempt to accept the next reply", function* () { + yield* usePosixSidekickTransport(process.platform); + const home = yield* publishingHome("xmd-reply-stale-"); + const path = join(home, "attachment.json"); + const host = yield* useMountedHost(); + host.fake.script({ reply: "Ready for attachment." }); + const page = yield* useReplPage({ host, sidekickAcp: path }); + const descriptor = Descriptor.parse(JSON.parse(yield* readTextFile(path))); + const size = { columns: 160, rows: 36 }; + const presenter = yield* usePresenter(size); + function* drawApproval(turn: number) { + const { execution, model, chat } = yield* settledReading(page.repl); + const record = projectSidekick(chat); + if (!record.ok) { + throw record.error; + } + const conversation = record.value.active; + if (conversation === undefined) { + throw new Error("Expected a live conversation."); + } + const live: ReplLive = { + output: "", + question: undefined, + expansion: "playing", + pausable: false, + running: false, + agent: NO_AGENT, + lifecycle: NO_LIFECYCLE, + sidekick: { + chat: [], + busy: true, + agent: MOUNTED_AGENT, + turns: [], + reading: conversation, + current: true, + availability: { chats: [conversation], sessions: [], refusal: undefined }, + refusal: undefined, + question: undefined, + conversation, + requested: undefined, + verified: undefined, + approval: { + key: `fixture-reply:${turn}`, + turn, + findingCount: 1, + accepting: false, + revealed: undefined, + finding: () => ["rule at line 1", "••••••••"], + }, + }, + }; + const state = initialState(execution); + const view = viewFor(state, model, live, size); + if (!view.ok) { + throw view.error; + } + const frame = yield* presenter.commit(view.value); + const bounds = frame.boundsOf("drawer:reply-approval:accept"); + const pointer = + bounds === undefined + ? undefined + : resolvePointer(frame.committed.rendered, { + column: bounds.x, + row: bounds.y, + }); + if (pointer === undefined) { + throw new Error("Expected the mounted Accept target."); + } + return { state, model, live, view: view.value, pointer }; + } + + host.fake.script({ reply }); + yield* sendMessage(page, "Review the first reply."); + yield* page.until("the first approval", (one) => one.shows("[accept reply]")); + const first = yield* drawApproval(2); + const original = yield* presenter.tree.dispatch(first.pointer); + if (!original.ok || original.value.outcome !== "action") { + throw new Error("Expected Accept to produce an action in its own frame."); + } + yield* page.press("[withhold reply]"); + yield* page.until("the first reply withholding joined", sidekickIdle); + + host.fake.script({ reply }); + yield* sendMessage(page, "Review the second reply."); + yield* page.until("the second approval", (one) => one.shows("[accept reply]")); + const before = yield* journalBytes(page, host); + const second = yield* drawApproval(3); + expect(yield* presenter.tree.dispatch(first.pointer)).toMatchObject({ + ok: true, + value: { outcome: "dropped" }, + }); + const reduce = (action: typeof original.value.action) => + reduceRepl( + second.state, + action, + second.model, + second.live, + undefined, + presentedDrawer(second.view), + ); + expect(reduce(original.value.action).intent).toEqual({ kind: "none" }); + const fresh = yield* presenter.tree.dispatch(second.pointer); + if (!fresh.ok || fresh.value.outcome !== "action") { + throw new Error("Expected Accept to produce an action in the replacement frame."); + } + expect(reduce(fresh.value.action).intent).toMatchObject({ + kind: "decide-sidekick-reply", + approval: "fixture-reply:3", + decision: "accept", + }); + yield* scoped(function* () { + const { wire } = yield* (yield* SidekickTransport.expect()).connect(path); + yield* wire.send({ id: 0, request: { method: "hello", descriptor } }); + yield* wire.receive(); + yield* wire.send({ + id: 1, + request: { method: "approve-reply", key: "sidekick-approval:2", decision: "accept" }, + }); + yield* refused(wire.receive()); + }); + expect(page.screen().shows("[accept reply]")).toBe(true); + expect(yield* journalBytes(page, host)).toBe(before); + expect(host.fake.prompts).toHaveLength(3); + yield* page.press("[accept reply]"); + yield* page.until("the fresh local action accepted the reply", (one) => + one.shows("[answer sidekick]"), + ); + expect(yield* rootRecords(page)).toContain(`${AGENT_PROMPT_REPLY} turn:3`); + expect(yield* rootRecords(page)).not.toContain(`${AGENT_PROMPT_REPLY} turn:2`); + yield* page.press("[stop]"); + yield* page.until("the accepted reply's question joined Stop", sidekickIdle); + }); + + it("acknowledges one interleaved child Elicit without settling its sibling", function* () { + const gate = appendGate(); + const host = yield* useMountedHost({ gate }); + const schema = { + type: "object", + properties: { choice: { type: "string", enum: ["confirmed", "declined"] } }, + required: ["choice"], + additionalProperties: false, + }; + host.fake.script({ + reply: + "\n" + + ["First child", "Second child"] + .map( + (message) => + `\n${message}\n`, + ) + .join("\n") + + "\n", + }); + const page = yield* useReplPage({ host }); + yield* sendMessage(page, "Ask both children."); + yield* page.until("a child question", (one) => one.shows("[answer sidekick]")); + const intents = function* () { + return (yield* settledReading(page.repl)).chat.flatMap((one) => + one.kind === "control" && one.elicit !== undefined + ? [{ order: one.order, ...one.elicit }] + : [], + ); + }; + yield* awaitedFact("both child intents are acknowledged", function* () { + return (yield* intents()).length === 2; + }); + const children = yield* intents(); + expect(new Set(children.map((one) => one.coroutineId)).size).toBe(2); + expect(children.map((one) => one.index)).toEqual([0, 0]); + yield* openQuestion(page); + const answering = gate.holdNext((_path, record) => + record.includes("executablemd.elicitation-schema"), + ); + yield* ensure(() => answering.release()); + yield* chooseOption(page, "confirmed"); + yield* submitForm(page); + yield* answering.started(); + const before = projectSidekick((yield* settledReading(page.repl)).chat); + expect(before.ok && before.value.unfinished.map((one) => one.order)).toEqual( + children.map((one) => one.order), + ); + answering.release(); + yield* awaitedFact("only the answered child's canonical event is public", function* () { + return (yield* canonical(page)).some((one) => one.description.type === "elicit"); + }); + const events = (yield* canonical(page)).filter((one) => one.description.type === "elicit"); + expect(events).toHaveLength(1); + const answered = children.find((one) => one.coroutineId === events[0].coroutineId); + if (answered === undefined) { + throw new Error("The canonical answer did not belong to either child."); + } + const sibling = children.find((one) => one !== answered); + const verify = (read: ReturnType) => { + if (!read.ok) { + throw read.error; + } + expect(read.value.operations.find((one) => one.order === answered.order)?.status).toBe("ok"); + expect(read.value.unfinished.map((one) => one.order)).toEqual([sibling?.order]); + }; + const live = yield* settledReading(page.repl); + verify(projectSidekick(live.chat)); + expect(live.chat.some((one) => one.kind === "control-outcome")).toBe(false); + yield* verifyColdCut(page, verify); + yield* page.press("[stop]"); + yield* page.until("both child lifetimes joined Stop", sidekickIdle); + expect(host.fake.prompts).toHaveLength(1); + }); + + it("refuses independently entered bytes before either journal retains the answer", function* () { + const host = yield* useMountedHost({}); + const freeText = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + }; + host.fake.script({ + reply: + `Enter a value.\n\n` + + `Example ${secret}\n`, + }); + const mounted: { page?: ReplPage } = {}; + let failure: unknown; + try { + yield* scoped(function* () { + const page = yield* useReplPage({ host }); + mounted.page = page; + yield* sendMessage(page, "Ask me for a separate value."); + yield* page.until("the approval", (one) => one.shows("[accept reply]")); + yield* page.press("[accept reply]"); + yield* page.until("the free-text question", (one) => one.shows("[answer sidekick]")); + yield* openQuestion(page); + yield* page.type(secret); + yield* page.until("the field containing the synthetic answer", (one) => one.shows(secret)); + yield* submitForm(page); + yield* page.until("all Send work joined", sidekickIdle); + }); + } catch (error) { + failure = error; + } + expect(failure instanceof Error && failure.name).toBe("SecretDetectedError"); + const page = mounted.page; + if (page === undefined) { + throw new Error("The fixture did not mount."); + } + + const history = yield* settledReading(page.repl); + const canonicalAnswer = (yield* canonical(page)).some( + (event) => event.description.type === "elicit" && event.result.status === "ok", + ); + expect(canonicalAnswer).toBe(false); + const independentlyRetained = history.chat.some( + (one) => one.kind === "control-outcome" && JSON.stringify(one.observed).includes(secret), + ); + expect(independentlyRetained).toBe(false); + expect(host.fake.prompts).toHaveLength(1); + }); +}); diff --git a/packages/cli/tests/repl-sidekick-lifecycle.test.ts b/packages/cli/tests/repl-sidekick-lifecycle.test.ts index c73c1318d..444a236af 100644 --- a/packages/cli/tests/repl-sidekick-lifecycle.test.ts +++ b/packages/cli/tests/repl-sidekick-lifecycle.test.ts @@ -37,6 +37,7 @@ import type { ExecutionInstallation } from "@executablemd/core/host"; import { submitForSidekick } from "../src/repl/sidekick/admission.ts"; import type { AdmittingSession } from "../src/repl/sidekick/admission.ts"; import { stillAdmissible } from "../src/repl/sidekick/gate.ts"; +import { acknowledgedControlOutcomes } from "../src/repl/sidekick/control-outcomes.ts"; import type { ControlJournal } from "../src/repl/sidekick/controls.ts"; import type { ReplSubmission } from "../src/repl/session.ts"; @@ -99,7 +100,7 @@ import { } from "../src/repl/sidekick/history.ts"; import { join } from "node:path"; import { readFile } from "node:fs/promises"; -import type { SidekickRecord } from "../src/repl/sidekick/history.ts"; +import type { ControlRecord, SidekickRecord } from "../src/repl/sidekick/history.ts"; /** * Real time for an assertion that something did not happen. @@ -597,6 +598,20 @@ describe("Tier R6 — Stop ends the asking and nothing else", () => { }); }); +function elicitIntents(records: readonly SidekickRecord[]) { + return records.filter( + (record): record is ControlRecord => record.kind === "control" && record.component === "Elicit", + ); +} + +function elicitOutcomes(written: readonly SidekickRecord[]) { + const outcomes = acknowledgedControlOutcomes(written); + if (!outcomes.ok) { + throw outcomes.error; + } + return [...outcomes.value.values()].filter(({ intent }) => intent.elicit !== undefined); +} + /** * Tier R6 — the question the Sidekick owns (#883, J5). * @@ -681,17 +696,6 @@ describe("Tier R6 — a reply asks, and the Sidekick owns the answer", () => { return { text: item.text, failure: item.failure }; } - /** Every control record this response wrote, by the component that wrote it. */ - function pairs( - held: readonly unknown[], - kind: "control" | "control-outcome", - ): readonly Record[] { - return held.filter((record) => (record as { kind: string }).kind === kind) as readonly Record< - string, - unknown - >[]; - } - it("R6: a reply's question is this owner's, asked once, and its answer comes back", function* () { const held = yield* useHarness(); held.fake.script({ @@ -735,16 +739,15 @@ describe("Tier R6 — a reply asks, and the Sidekick owns the answer", () => { // the next turn's input — alone, so it is the value itself. expect(JSON.parse(held.fake.prompts[1] ?? "")).toEqual({ decision: "approve" }); - // Retained as the pair every control writes: the intent before the asking, - // naming the question, and the outcome holding the validated answer. + // The generated intent and its acknowledged Elicit Yield retain the answer. const written = yield* records(held.execution); - const intents = pairs(written, "control").filter((one) => one["component"] === "Elicit"); + const intents = elicitIntents(written); expect(intents).toHaveLength(1); - expect(intents[0]?.["arguments"]).toMatchObject({ message: "Approve the plan?" }); - const outcomes = pairs(written, "control-outcome").filter( - (one) => one["order"] === intents[0]?.["order"], - ); - expect(outcomes[0]?.["observed"]).toEqual({ decision: "approve" }); + expect(intents[0]?.arguments).toMatchObject({ message: "Approve the plan?" }); + const outcomes = elicitOutcomes(written); + expect(outcomes).toHaveLength(1); + expect(outcomes[0]?.intent).toEqual(intents[0]); + expect(outcomes[0]?.outcome.observed).toEqual({ decision: "approve" }); }); it("R6: an invalid answer answers nobody and leaves the same question pending", function* () { @@ -769,9 +772,7 @@ describe("Tier R6 — a reply asks, and the Sidekick owns the answer", () => { expect(held.owner.questions.pending).toBe(question); expect(held.owner.questions.asked).toBe(1); // And no outcome was retained: an answer nobody gave is not written down. - expect( - pairs(yield* records(held.execution), "control-outcome").filter((one) => one["order"] === 0), - ).toHaveLength(0); + expect(elicitOutcomes(yield* records(held.execution))).toHaveLength(0); expect(question.submit({ decision: "reject" }).kind).toBe("answered"); const answered = yield* sent; @@ -848,11 +849,9 @@ describe("Tier R6 — a reply asks, and the Sidekick owns the answer", () => { // is what an operation that began and whose result nobody knows looks // like — an interruption is an absence, never a receipt saying so. const written = yield* records(held.execution); - const intents = pairs(written, "control").filter((one) => one["component"] === "Elicit"); + const intents = elicitIntents(written); expect(intents).toHaveLength(1); - expect( - pairs(written, "control-outcome").filter((one) => one["order"] === intents[0]?.["order"]), - ).toHaveLength(0); + expect(elicitOutcomes(written)).toHaveLength(0); // No response record either: the root never finished. expect(written.filter((one) => (one as { kind: string }).kind === "response")).toHaveLength(0); // And the projection says so in the terms the rest of this history uses: @@ -1622,33 +1621,17 @@ describe("Tier MQ — a Sidekick question, filled in and answered on the screen" }; } - /** Every Elicit record of one kind this history holds. */ - function elicits( - chat: readonly SidekickRecord[], - kind: "control" | "control-outcome", - ): readonly SidekickRecord[] { - return chat.filter( - (record) => - record.kind === kind && (record.kind === "control" ? record.component === "Elicit" : true), - ); - } - it("MQ2: an invalid answer keeps the question, and closing keeps what was typed", function* () { const done = yield* answeringJourney(true); // Exactly one question was asked and exactly one answer was retained: the // invalid attempts wrote nothing at all. - const intents = elicits(done.chat, "control"); + const intents = elicitIntents(done.chat); expect(intents).toHaveLength(1); - const answers = done.chat.filter( - (record) => - record.kind === "control-outcome" && - intents.some((one) => one.kind === "control" && one.order === record.order), - ); + const answers = elicitOutcomes(done.chat); expect(answers).toHaveLength(1); - expect(answers[0]).toMatchObject({ - observed: { decision: "approve", message: "okay" }, - }); + expect(answers[0]?.intent).toEqual(intents[0]); + expect(answers[0]?.outcome.observed).toEqual({ decision: "approve", message: "okay" }); // And it reached the reply that asked, as its own designated feedback. expect(JSON.parse(done.prompts[1] ?? "")).toEqual({ decision: "approve", @@ -1664,7 +1647,7 @@ describe("Tier MQ — a Sidekick question, filled in and answered on the screen" it("MQ2: control — the same answer without closing or changing panes", function* () { const done = yield* answeringJourney(false); - expect(elicits(done.chat, "control")).toHaveLength(1); + expect(elicitIntents(done.chat)).toHaveLength(1); expect(JSON.parse(done.prompts[1] ?? "")).toEqual({ decision: "approve", message: "okay", diff --git a/packages/cli/tests/repl-sidekick-recovery.test.ts b/packages/cli/tests/repl-sidekick-recovery.test.ts index 6c6eac929..d834d2887 100644 --- a/packages/cli/tests/repl-sidekick-recovery.test.ts +++ b/packages/cli/tests/repl-sidekick-recovery.test.ts @@ -5150,7 +5150,14 @@ describe("1311 — a completed reply that secret detection withholds", () => { host.fake.script({ reply: "" }); host.fake.script({ reply: `${refused}` }); yield* sendMessage(page, "Read the current bindings."); - yield* page.until("the withheld reply", showsWithheldReply); + // The foreground person is asked, with the match masked, and withholds. + yield* page.until("the withheld reply's question", (one) => one.shows("[withhold reply]")); + expect(page.screen().shows("Use a separate approval for this example issue #34")).toBe(false); + yield* page.press("[withhold reply]"); + yield* page.until( + "the withheld reply", + (one) => showsWithheldReply(one) && !one.shows("[withhold reply]"), + ); clean(page.screen()); expect(host.fake.prompts).toHaveLength(2); execution = yield* mountedExecution(page.repl); diff --git a/packages/core/host.ts b/packages/core/host.ts index e36a1f9ac..674ccba94 100644 --- a/packages/core/host.ts +++ b/packages/core/host.ts @@ -180,7 +180,13 @@ export { props as elicitProps, returns as elicitReturns } from "./src/components * the description is journal data, and a schema that merely looked plausible * would reach a form as one. */ -export { ELICITATION_SCHEMA_FIELD, readElicitationSchema } from "./src/elicit-journal.ts"; +export { + ELICITATION_SCHEMA_FIELD, + elicitationEffectName, + questionFingerprint, + readElicitationAnswer, + readElicitationSchema, +} from "./src/elicit-journal.ts"; export { clearsLiveOutput, retainsOptionalText } from "./src/secrets/policy.ts"; /** @@ -199,6 +205,39 @@ export { hasInvocationTeardown, } from "./src/errors.ts"; export { readSourceOwnerDescription } from "./src/source-owner.ts"; + +/** + * The closed exception secret detection makes for one accepted reply — see + * `src/secrets/reply-grant.ts`. + * + * A host that withheld a completed reply and then had a person accept it + * reads where the execution's own scanner found each match, and adds one + * allowance per event that may carry the exact accepted bytes: the acceptance + * record, one evaluation of exactly that source, and the one turn that carries + * its selected answer back. The gate still scans every event; an allowance + * only accounts for findings at the positions it names, with the same rule + * and bytes. The scanner and the gate that enforce it stay core's own. + */ +export { + acceptanceGrant, + approvedMatches, + carriedCopies, + evaluationGrant, + feedbackGrant, +} from "./src/secrets/reply-grant.ts"; +export type { + ApprovedMatch, + ElicitChoiceSelection, + FeedbackSlice, + ReplyCopyGrant, +} from "./src/secrets/reply-grant.ts"; +export { + captureReplyTextCheck, + grantReplyCopies, + locateReplySecrets, +} from "./src/secrets/policy.ts"; +export type { ReplyTextCheck } from "./src/secrets/policy.ts"; +export type { SecretSpan } from "./src/secrets/scanner.ts"; export { WorkflowBundleError } from "./src/components/bundle.ts"; export type { WorkflowBundleComponent, WorkflowComponentBundle } from "./src/components/bundle.ts"; @@ -310,7 +349,7 @@ export { GeneratedXmdError } from "./src/generated-xmd.ts"; * selection its execution already captured, so a caller has nothing to say * about those. `GeneratedEffectClass` is here because that request names it. */ -export { GeneratedXmdRootError } from "./src/generated-xmd.ts"; +export { GeneratedXmdRootError, readGeneratedRootAdmission } from "./src/generated-xmd.ts"; export type { GeneratedEffectClass, GeneratedXmdRequest, @@ -319,6 +358,20 @@ export type { export type { HostExecution } from "./src/execute.ts"; export { HostExecutionError } from "./src/generated-root-owner.ts"; +/** + * The same execution, with its bound root evaluator handed to the host that + * started it. + * + * That evaluator also takes an `AcceptedRootReply`: the approved matches of a + * reply a person accepted and the allowance for one evaluation of exactly that + * source. Only the host holding the execution can pass one, because the + * evaluator reaches it through this call and is never on the handle a + * document or another caller can see. + */ +export { executeInstalledBound } from "./src/execute.ts"; +export type { BoundRootEvaluator } from "./src/generated-root-owner.ts"; +export type { AcceptedRootReply } from "./src/generated-xmd.ts"; + /** * What an arriving generated root already says — see * `src/generated-xmd-preview.ts`. diff --git a/packages/core/src/components/Elicit.ts b/packages/core/src/components/Elicit.ts index 82a3b8cd2..cab1e9c4e 100644 --- a/packages/core/src/components/Elicit.ts +++ b/packages/core/src/components/Elicit.ts @@ -44,6 +44,7 @@ import type { Operation } from "effection"; import { content } from "../component-api.ts"; import { getExpansion } from "../expansion.ts"; import { + elicitationLocation, persistElicitation, persistProtectedElicitation, questionFingerprint, @@ -51,7 +52,6 @@ import { } from "../elicit-journal.ts"; import { prepareElicitation, runPreparedElicitation } from "../elicit.ts"; import type { Json, PropsSchema, ReturnsSchema } from "../types.ts"; -import type { Expansion } from "../expansion.ts"; export const props = { type: "object", @@ -97,7 +97,7 @@ export default function* Elicit(props: Record): Operation const expansion = yield* getExpansion(); const identity = { - location: formatLocation(expansion), + location: elicitationLocation(expansion.position), fingerprint: questionFingerprint({ schema: prepared.schema, message }), schema: prepared.schema, position: expansion.position, @@ -112,13 +112,3 @@ export default function* Elicit(props: Record): Operation } return yield* persistElicitation(identity, () => runPreparedElicitation(prepared, message)); } - -/** `path:line:column`, `line:column`, or `unknown` — the durable identity. */ -function formatLocation(metadata: Expansion): string { - const position = metadata.position; - if (!position) { - return "unknown"; - } - const at = `${position.line}:${position.column}`; - return position.path ? `${position.path}:${at}` : at; -} diff --git a/packages/core/src/elicit-journal.ts b/packages/core/src/elicit-journal.ts index b817defee..54819cf0c 100644 --- a/packages/core/src/elicit-journal.ts +++ b/packages/core/src/elicit-journal.ts @@ -1,6 +1,8 @@ import { durablePosition } from "@executablemd/durable-streams"; import { inputValues } from "./secrets/inputs.ts"; import { parseInputRecord } from "./secrets/input-record.ts"; +import type { RecordedInputValue } from "./secrets/input-record.ts"; +import { INPUT_REPLACEMENT, secretInputFields } from "./secrets/input-schema.ts"; import { prepareElicitation, runPreparedElicitation, ElicitValidationError } from "./elicit.ts"; import type { PreparedElicitation } from "./elicit.ts"; import { validateParsed } from "./components/parse-schema.ts"; @@ -40,8 +42,10 @@ import type { EffectDescription, Json as DurableJson, Workflow, + Yield, } from "@executablemd/durable-streams"; -import type { Operation } from "effection"; +import { Err, Ok } from "effection"; +import type { Operation, Result } from "effection"; import { canonicalFingerprint } from "./canonical.ts"; import { isJsonObject, parseJson } from "./json.ts"; @@ -82,6 +86,18 @@ function recordName(location: string): string { return `elicit:${location}`; } +export function elicitationLocation(position: Readonly | undefined): string { + if (position === undefined) { + return "unknown"; + } + const at = `${position.line}:${position.column}`; + return position.path ? `${position.path}:${at}` : at; +} + +export function elicitationEffectName(position: Readonly | undefined): string { + return recordName(elicitationLocation(position)); +} + /** A stable name for what was asked. */ export function questionFingerprint(question: ElicitationQuestion): string { return canonicalFingerprint({ schema: question.schema, message: question.message }); @@ -100,6 +116,63 @@ export function readElicitationSchema(description: EffectDescription): JsonObjec return isJsonObject(field) ? field : undefined; } +/** Read a canonical Elicit answer for display without restoring live input. */ +export function readElicitationAnswer(event: Yield, index: number): Result { + try { + const schema = readElicitationSchema(event.description); + if (event.description.type !== ELICIT || event.result.status !== "ok" || schema === undefined) { + throw new Error(); + } + const fields = secretInputFields(schema); + const answer = parseJson(event.result.value); + if (fields.length === 0) { + return Ok(answer); + } + if (!isJsonObject(answer) || Object.keys(answer).length !== 1) { + throw new Error(); + } + const receipt = parseInputRecord(answer["executablemd.input"]); + if (receipt.kind !== "object") { + throw new Error(); + } + return Ok( + Object.fromEntries( + Object.entries(receipt.members).map(([name, member]) => { + if (!fields.includes(name)) { + return [name, readPublicInput(member)]; + } + if ( + member.kind !== "input" || + member.field.coroutine !== event.coroutineId || + member.field.index !== index || + member.field.field !== name + ) { + throw new Error(); + } + return [name, INPUT_REPLACEMENT]; + }), + ), + ); + } catch { + return Err(new Error("The retained elicitation answer cannot be safely read.")); + } +} + +function readPublicInput(member: RecordedInputValue): Json { + switch (member.kind) { + case "json": + return member.value; + case "array": + return member.items.map(readPublicInput); + case "object": + return Object.fromEntries( + Object.entries(member.members).map(([name, child]) => [name, readPublicInput(child)]), + ); + case "input": + throw new Error("Invalid protected elicitation field identity."); + } +} + /** * Refuse to replay an answer to a different question. * diff --git a/packages/core/src/execute.ts b/packages/core/src/execute.ts index 4ccdee8ed..679bd206e 100644 --- a/packages/core/src/execute.ts +++ b/packages/core/src/execute.ts @@ -3301,6 +3301,7 @@ function* executeDocument( ); } : bindGeneratedRootEvaluator({ + detection, filesystem, // The selection this execution captured, resolved per evaluation // through the one machinery the authored site also goes through, and @@ -4785,10 +4786,20 @@ function initialRootBindings( * host holds and passes, so a separately loaded workflow package composes by * handing its closure over rather than by agreeing on a name. */ -export function* executeInstalled( +export function executeInstalled( options: ExecuteOptions, installations: readonly ExecutionInstallation[], initialization?: ExecutionInitialization, +): Operation { + return executeInstalledBound(options, installations, initialization); +} + +/** `executeInstalled`, handing its bound root evaluator to the host that started it. */ +export function* executeInstalledBound( + options: ExecuteOptions, + installations: readonly ExecutionInstallation[], + initialization?: ExecutionInitialization, + bindEvaluator?: (evaluate: BoundRootEvaluator) => void, ): Operation { const invoked = yield* runInvocation( options, @@ -4803,6 +4814,7 @@ export function* executeInstalled( }, }); const { execution } = invoked; + bindEvaluator?.(invoked.evaluateGeneratedXmdRoot); const clearDisclosure = invoked.computed.disclose; // Assembled here, at the trusted boundary, and nowhere else. The document // handle this delegates to is the authoritative one: the consumer's own @@ -4813,7 +4825,7 @@ export function* executeInstalled( *[Symbol.iterator]() { return yield* execution; }, - evaluateGeneratedXmdRoot: invoked.evaluateGeneratedXmdRoot, + evaluateGeneratedXmdRoot: (request) => invoked.evaluateGeneratedXmdRoot(request), bindings: invoked.computed.read, disclose(data) { return clearDisclosure(data, disclosure.operations.disclose); diff --git a/packages/core/src/expand.ts b/packages/core/src/expand.ts index f96fc3e48..18804e4ee 100644 --- a/packages/core/src/expand.ts +++ b/packages/core/src/expand.ts @@ -348,6 +348,7 @@ export interface OutputFrame { readonly active: boolean; readonly private: boolean; readonly marks: WeakMap; + readonly replyLiteralOffsets?: WeakMap; readonly stringSelection?: Set; } @@ -389,6 +390,7 @@ function bodyOutput(segments: Segment[], path: string, inherited?: OutputFrame): active: false, private: false, marks: inherited?.marks ?? new WeakMap(), + replyLiteralOffsets: inherited?.replyLiteralOffsets, }; } @@ -1236,7 +1238,8 @@ function* expandListSegments( ? textEvalEnv.values.props : parentProps; const textInputs = yield* inputValues(); - const final = splitTextFences(segment, healed) + const chunks = splitTextFences(segment, healed); + const final = chunks .map(({ content, passive }) => { if (passive) { return content; @@ -1249,10 +1252,21 @@ function* expandListSegments( return bound.replaceAll(ESCAPED_BRACE_PLACEHOLDER, "{"); }) .join(""); - result.push({ + const rendered: Segment = { type: "text", content: final, - }); + }; + const literals = outputFrame?.replyLiteralOffsets; + const literal = literals?.get(segment); + if ( + literal !== undefined && + healed === segment.content && + final === healed && + chunks.every(({ content, passive }) => passive || !/[{\uE000]/.test(content)) + ) { + literals?.set(rendered, literal); + } + result.push(rendered); break; } diff --git a/packages/core/src/generated-expressions.ts b/packages/core/src/generated-expressions.ts index 349e35dbc..e3bc2cc13 100644 --- a/packages/core/src/generated-expressions.ts +++ b/packages/core/src/generated-expressions.ts @@ -159,6 +159,36 @@ export function evaluateDataExpression(text: string, values: Record key(property) === member)?.value; + } else { + return undefined; + } + } + if ( + !isNode(node) || + node.type !== "Literal" || + !("value" in node) || + typeof node.value !== "string" || + text.slice(node.start + 1, node.end - 1) !== node.value + ) { + return undefined; + } + return { value: node.value, start: node.start + 1 }; +} + function binding(name: string): string { return `a generated expression names \`${name}\`, which is not one of the fragment's bindings.`; } diff --git a/packages/core/src/generated-root-owner.ts b/packages/core/src/generated-root-owner.ts index cf3465cb9..ddf9818d6 100644 --- a/packages/core/src/generated-root-owner.ts +++ b/packages/core/src/generated-root-owner.ts @@ -53,7 +53,11 @@ import type { Operation, Queue, Result } from "effection"; import { durableRun, preserveJournalProvenance, retainEvents } from "@executablemd/durable-streams"; import type { DurableEvent, DurableStream } from "@executablemd/durable-streams"; -import type { GeneratedXmdRequest, GeneratedXmdRootResult } from "./generated-xmd.ts"; +import type { + AcceptedRootReply, + GeneratedXmdRequest, + GeneratedXmdRootResult, +} from "./generated-xmd.ts"; /** Why this execution could not take the root evaluation it was asked for. */ export class HostExecutionError extends Error { @@ -63,6 +67,7 @@ export class HostExecutionError extends Error { /** One execution's bound root evaluator, as the owner and the handle hold it. */ export type BoundRootEvaluator = ( request: GeneratedXmdRequest, + accepted?: AcceptedRootReply, ) => Operation>; /** What canonical execution hands its owner once both exist. */ @@ -137,6 +142,7 @@ export interface GeneratedRootOwner { /** What one caller asked this execution to evaluate. */ interface RootJob { readonly request: GeneratedXmdRequest; + readonly accepted?: AcceptedRootReply; readonly answer: Resolvers; readonly cancelled: Resolvers; readonly joined: Resolvers; @@ -306,7 +312,10 @@ export function createGeneratedRootOwner(): GeneratedRootOwner { return Ok(undefined); } - function evaluate(request: GeneratedXmdRequest): Operation> { + function evaluate( + request: GeneratedXmdRequest, + accepted?: AcceptedRootReply, + ): Operation> { return scoped(function* () { // Before readiness, so two calls that arrive while the runner is still // starting are one admitted call and one refusal rather than two waiters. @@ -344,7 +353,7 @@ export function createGeneratedRootOwner(): GeneratedRootOwner { if (!started.ok) { return started; } - jobs.add({ request, answer, cancelled, joined }); + jobs.add({ request, accepted, answer, cancelled, joined }); sent = true; // Raced against the execution closing, because a halted worker answers // nobody: the work it was doing has joined by the time this resolves, and @@ -504,7 +513,7 @@ function* attempt( // journal: a read that fails is this caller's failure to be handed and this // execution's outcome, not an availability answer about a run that is still // open. - job.answer.resolve({ value: yield* sequence(inputs, job.request, yield* address()) }); + job.answer.resolve({ value: yield* sequence(inputs, job, yield* address()) }); return undefined; } catch (error) { // The caller is handed the same object with the same classification, and @@ -527,15 +536,16 @@ function* attempt( */ function sequence( inputs: GeneratedRootInputs, - request: GeneratedXmdRequest, + job: RootJob, order: number, ): Operation> { return scoped(function* () { const settled: { outcome?: Result } = {}; const coroutineId = sequenceName(order); + job.accepted?.grant.bind(coroutineId); yield* durableRun( function* (): Operation { - settled.outcome = yield* inputs.evaluator(request); + settled.outcome = yield* inputs.evaluator(job.request, job.accepted); }, { stream: evaluationJournal(inputs.journal, order), coroutineId }, ); diff --git a/packages/core/src/generated-xmd.ts b/packages/core/src/generated-xmd.ts index a15164016..e832ec7c7 100644 --- a/packages/core/src/generated-xmd.ts +++ b/packages/core/src/generated-xmd.ts @@ -108,7 +108,7 @@ import type { FilesystemExpansion } from "./filesystem-placement.ts"; */ import { createDurableOperation } from "@executablemd/durable-streams"; -import type { Json as DurableJson } from "@executablemd/durable-streams"; +import type { Json as DurableJson, Yield } from "@executablemd/durable-streams"; import { ensure, Err, Ok, scoped } from "effection"; import type { Operation, Result } from "effection"; @@ -138,7 +138,10 @@ import { isJsonObject, parseJson } from "./json.ts"; import { markGeneratedRequestRefusal } from "./generated-request-refusal.ts"; import { GeneratedDataExpressions, validateDataExpression } from "./generated-expressions.ts"; import { capturedBinding } from "./invocation-rules.ts"; -import { renderSegments } from "./render.ts"; +import { renderSegment, renderSegments } from "./render.ts"; +import type { ApprovedMatch, evaluationGrant } from "./secrets/reply-grant.ts"; +import type { SecretDetection } from "./secrets/policy.ts"; +import type { SourceRange } from "./document-targets.ts"; import { scanSegments } from "./scanner.ts"; import { readsBinding } from "./generated-interpolation.ts"; import { SourceOwner, sourceOwnerDescription } from "./source-owner.ts"; @@ -697,6 +700,8 @@ export class GeneratedXmdRootError extends Error { /** Where a root's rendering goes, and what the admitted source said about it. */ interface RootRendering { + readonly literalOffsets?: WeakMap; + readonly schemas?: Map; readonly owner: Segment[]; /** Called once, with what the admitted source declared. */ readonly declared: (hasOutput: boolean) => void; @@ -712,6 +717,7 @@ interface RootRendering { * hands it over across one Core-private call and keeps it. */ export interface GeneratedRootBinding { + readonly detection: SecretDetection; readonly filesystem?: FilesystemExpansion; /** * The execution's captured selection, resolved for the classes one root asks @@ -732,6 +738,11 @@ export interface GeneratedRootBinding { readonly live: () => boolean; } +export interface AcceptedRootReply { + readonly approved: readonly ApprovedMatch[]; + readonly grant: ReturnType; +} + /** * One root evaluation, with its selection already resolved. * @@ -1607,6 +1618,8 @@ function* preflight( source: string, table: ReadonlyMap, language: AdmissionLanguage, + literalOffsets?: WeakMap, + schemas?: Map, ): Operation { const named: Planned[] = []; // Scanned as the fragment it is: generated text has no path, so every @@ -1614,7 +1627,16 @@ function* preflight( // reader of the history say which fragment performed an effect — and it is // decided here, in the scan preflight runs, rather than by anything watching // the expansion afterwards. - const segments = scanSegments(source, { generatedSource: id, baseOffset: 0, baseLine: 1 }); + const segments = scanSegments( + source, + { generatedSource: id, baseOffset: 0, baseLine: 1 }, + undefined, + undefined, + undefined, + 0, + literalOffsets, + schemas, + ); // A generated fragment starts with no bindings: it imports none from the // document that admitted it, and exports none back to it. yield* walk(segments, table, named, new Set(), topLevelLexical(language)); @@ -2187,6 +2209,20 @@ const REVOKED_IMPLEMENTATIONS = const FRAGMENT_RECORD = 4; const ROOT_RECORD = 5; +/** The admitted source of a retained generated root for the named response. */ +export function readGeneratedRootAdmission(event: Yield, response: string): string | undefined { + if ( + event.description.type !== GENERATED_XMD || + event.description.name !== `generated:${response}` || + event.result.status !== "ok" || + event.result.value === undefined + ) { + return undefined; + } + const admission = readAdmission(event.result.value, ROOT_RECORD); + return admission?.decision === "admitted" ? admission.source : undefined; +} + /** Which of the two a record is expected to be, at the result and its policy. */ type RecordMarker = typeof FRAGMENT_RECORD | typeof ROOT_RECORD; @@ -2339,6 +2375,7 @@ function expand( */ owner: Segment[] | undefined, filesystem?: FilesystemExpansion, + literalOffsets?: WeakMap, ): Operation { return scoped(function* () { yield* SourceOwner.set(undefined); @@ -2431,6 +2468,16 @@ function expand( undefined, undefined, filesystem, + literalOffsets === undefined + ? undefined + : { + source: path, + declared: true, + active: false, + private: false, + marks: new WeakMap(), + replyLiteralOffsets: literalOffsets, + }, ); return renderSegments(owner ?? []); } @@ -2592,7 +2639,14 @@ function* evaluateGenerated( // The retained source is what expands, so a continuation runs exactly the // bytes this run admitted rather than a caller's copy of them. - const restored = yield* preflight(request.id, decided.source, table, language); + const restored = yield* preflight( + request.id, + decided.source, + table, + language, + rendering?.literalOffsets, + rendering?.schemas, + ); // Read from the admitted segments, which is the same reading `expand` selects // regions with, and reported before the first effect so a failure partway // still describes the root truthfully. @@ -2609,6 +2663,7 @@ function* evaluateGenerated( reference, rendering?.owner, filesystem, + rendering?.literalOffsets, ); } @@ -2688,12 +2743,22 @@ function safeDiagnostic(error: unknown): string { function* evaluateRoot( request: GeneratedXmdRequest, binding: GeneratedRootBinding, + accepted?: AcceptedRootReply, ): Operation> { // Allocated out here, so a failure partway leaves this frame holding exactly // what the root rendered before it. The flag is read from the source core // admitted rather than from the caller's copy, and it is the same reading // `expand` selects regions with. const owner: Segment[] = []; + const literalOffsets = accepted === undefined ? undefined : new WeakMap(); + if (accepted !== undefined) { + yield* ensure(binding.detection.allowReplyCopies(accepted.grant)); + } + const check = (output: string) => + binding.detection.checkReplyText( + output, + outputCopies(owner, literalOffsets, accepted?.approved ?? []), + ); let hasOutput = false; try { // Resolved inside the boundary, and once. Once, because the tables this @@ -2722,6 +2787,8 @@ function* evaluateRoot( undefined, { owner, + literalOffsets, + schemas: accepted?.grant.schemas, declared: (value) => { hasOutput = value; }, @@ -2735,6 +2802,9 @@ function* evaluateRoot( binding.filesystem, ); }); + if (hasOutput) { + yield* check(output); + } return Ok({ output, hasOutput }); } catch (error) { if (!ordinaryGeneratedFailure(error)) { @@ -2745,10 +2815,44 @@ function* evaluateRoot( // something that never happened. const partial: GeneratedXmdRootResult | undefined = hasOutput || owner.length > 0 ? { output: renderSegments(owner), hasOutput } : undefined; + if (partial?.hasOutput) { + yield* check(partial.output); + } return Err(new GeneratedXmdRootError(safeDiagnostic(error), partial)); } } +function outputCopies( + selected: readonly Segment[], + literals: WeakMap | undefined, + approved: readonly ApprovedMatch[], +): ApprovedMatch[] { + const copies: ApprovedMatch[] = []; + if (literals === undefined) { + return copies; + } + let offset = 0; + for (const segment of selected) { + const rendered = renderSegment(segment); + const start = literals.get(segment); + if (start !== undefined) { + for (const match of approved) { + const end = Math.min(match.end, start + rendered.length); + if (match.start >= start && end > match.start) { + copies.push({ + ...match, + start: offset + match.start - start, + end: offset + end - start, + bytes: match.bytes.slice(0, end - match.start), + }); + } + } + } + offset += rendered.length; + } + return copies; +} + /** * Bind one execution's root evaluator to the authority that execution captured. * @@ -2766,6 +2870,9 @@ function* evaluateRoot( */ export function bindGeneratedRootEvaluator( binding: GeneratedRootBinding, -): (request: GeneratedXmdRequest) => Operation> { - return (request) => evaluateRoot(request, binding); +): ( + request: GeneratedXmdRequest, + accepted?: AcceptedRootReply, +) => Operation> { + return (request, accepted) => evaluateRoot(request, binding, accepted); } diff --git a/packages/core/src/scanner.ts b/packages/core/src/scanner.ts index 53985de23..edc733c03 100644 --- a/packages/core/src/scanner.ts +++ b/packages/core/src/scanner.ts @@ -147,6 +147,8 @@ export interface SourceOrigin { interface PositionIndex { origin?: SourceOrigin; lineStarts: number[]; + literalOffsets?: WeakMap; + elicitSchemas?: Map; } function computeLineStarts(text: string): number[] { @@ -236,8 +238,15 @@ export function scanSegments( unfinished?: UnfinishedConstruct[], /** A suffix rescan retains its authored prefix for line-boundary decisions. */ start = 0, + literalOffsets?: WeakMap, + elicitSchemas?: Map, ): Segment[] { - const index: PositionIndex = { origin, lineStarts: computeLineStarts(text) }; + const index: PositionIndex = { + origin, + lineStarts: computeLineStarts(text), + literalOffsets, + elicitSchemas, + }; const segments: Segment[] = []; let pos = start; let textStart = start; @@ -246,7 +255,7 @@ export function scanSegments( const fenceMatch = matchFenceOpen(text, pos); if (fenceMatch) { if (pos > textStart) { - pushText(segments, text, textStart, pos); + pushText(segments, text, index, textStart, pos); } const fenceEnd = findFenceClose( @@ -274,7 +283,7 @@ export function scanSegments( } satisfies ExecutableCodeBlock); } else { // Non-executable code block: preserve as text - pushText(segments, text, pos, fenceEnd.fenceEnd); + pushText(segments, text, index, pos, fenceEnd.fenceEnd); } pos = fenceEnd.fenceEnd; @@ -299,7 +308,7 @@ export function scanSegments( const component = parseComponentTag(text, pos, index, elements, unfinished); if (component) { if (pos > textStart) { - pushText(segments, text, textStart, pos); + pushText(segments, text, index, textStart, pos); } segments.push(component.segment); spans?.push({ start: pos, end: component.end }); @@ -313,7 +322,7 @@ export function scanSegments( } if (textStart < text.length) { - pushText(segments, text, textStart); + pushText(segments, text, index, textStart); } return segments; @@ -570,13 +579,17 @@ function parseComponentTag( return null; } + const schema = + name === "Elicit" && index.elicitSchemas !== undefined + ? { seen: false, range: undefined as SourceRange | undefined } + : undefined; const { props, expressions, authoredExpressions, end: attrEnd, exhausted, - } = parseAttributes(text, pos, name); + } = parseAttributes(text, pos, name, schema); if (attrEnd === -1) { if (exhausted) { unfinished?.push({ start, name }); @@ -654,6 +667,9 @@ function parseComponentTag( ); elements.push(...nested); } + if (schema?.range !== undefined) { + index.elicitSchemas?.set(start, schema.range); + } return { segment: { @@ -693,6 +709,7 @@ function parseAttributes( pos: number, /** Whose attributes these are: a construct may bind one of them by reference. */ componentName: string, + schema?: { seen: boolean; range: SourceRange | undefined }, ): ParsedAttributes { const props: Record = {}; const expressions: Record = {}; @@ -734,6 +751,11 @@ function parseAttributes( return { props, expressions, authoredExpressions, end: -1, exhausted: false }; } + const capture = schema !== undefined && attrName === "schema" && !schema.seen; + if (schema !== undefined && attrName === "schema") { + schema.seen = true; + schema.range = undefined; + } pos = skipWhitespace(text, pos); // Boolean attribute (no value): `verbose` @@ -785,7 +807,12 @@ function parseAttributes( if (exprEnd === -1) { return { props, expressions, authoredExpressions, end: -1, exhausted: true }; } - const exprText = text.slice(pos + 1, exprEnd).trim(); + const raw = text.slice(pos + 1, exprEnd); + const exprText = raw.trim(); + if (capture && schema !== undefined) { + const start = pos + 1 + raw.length - raw.trimStart().length; + schema.range = { start, end: start + exprText.length }; + } if (bindsByReference(componentName, attrName)) { expressions[attrName] = exprText; } else if (exprText === "undefined" && !isStructural(componentName)) { @@ -990,7 +1017,7 @@ function parseChildren( while (pos < text.length) { if (text.startsWith(closingTag, pos)) { if (pos > textStart) { - pushText(children, text, textStart, pos); + pushText(children, text, index, textStart, pos); } return { children, end: pos + closingTag.length }; } @@ -998,7 +1025,7 @@ function parseChildren( const fenceMatch = matchFenceOpen(text, pos); if (fenceMatch) { if (pos > textStart) { - pushText(children, text, textStart, pos); + pushText(children, text, index, textStart, pos); } const fenceEnd = findFenceClose( @@ -1024,7 +1051,7 @@ function parseChildren( position: positionAt(index, pos), } satisfies ExecutableCodeBlock); } else { - pushText(children, text, pos, fenceEnd.fenceEnd); + pushText(children, text, index, pos, fenceEnd.fenceEnd); } pos = fenceEnd.fenceEnd; @@ -1046,7 +1073,7 @@ function parseChildren( const nested = parseComponentTag(text, pos, index, elements, unfinished); if (nested) { if (pos > textStart) { - pushText(children, text, textStart, pos); + pushText(children, text, index, textStart, pos); } children.push(nested.segment); pos = nested.end; @@ -1110,7 +1137,13 @@ function skipWhitespace(text: string, pos: number): number { return pos; } -function pushText(segments: Segment[], text: string, start: number, end = text.length): void { +function pushText( + segments: Segment[], + text: string, + index: PositionIndex, + start: number, + end = text.length, +): void { const content = text.slice(start, end); if (content.length === 0) { return; @@ -1119,6 +1152,10 @@ function pushText(segments: Segment[], text: string, start: number, end = text.l // Merge with previous text segment if possible const last = segments[segments.length - 1]; if (last && last.type === "text") { + const offset = index.literalOffsets?.get(last); + if (offset !== undefined && offset + last.content.length !== start) { + index.literalOffsets?.delete(last); + } last.content += content; return; } @@ -1130,5 +1167,6 @@ function pushText(segments: Segment[], text: string, start: number, end = text.l value: start === 0 || text[start - 1] === "\n", enumerable: true, }); + index.literalOffsets?.set(segment, start); segments.push(segment); } diff --git a/packages/core/src/secrets/policy.ts b/packages/core/src/secrets/policy.ts index 08097d3a0..22f6a75a7 100644 --- a/packages/core/src/secrets/policy.ts +++ b/packages/core/src/secrets/policy.ts @@ -44,10 +44,12 @@ import { serializeDurableEvent, } from "@executablemd/durable-streams"; import type { DurableEvent, DurableStream } from "@executablemd/durable-streams"; -import { SecretDetectedError } from "./findings.ts"; +import { SecretDetectedError, SecretScannerError } from "./findings.ts"; import type { SecretFinding } from "./findings.ts"; -import { createSecretScanner } from "./scanner.ts"; -import type { SecretScanner } from "./scanner.ts"; +import { createSecretScanner, locateSecrets } from "./scanner.ts"; +import type { SecretScanner, SecretSpan } from "./scanner.ts"; +import { boundariesOf } from "./reply-grant.ts"; +import type { ApprovedMatch, ReplyCopyGrant } from "./reply-grant.ts"; import type { JsonObject } from "../types.ts"; /** @@ -92,11 +94,24 @@ class EnabledDetection extends ExecutionDetection { readonly enabled = true; readonly scanner: SecretScanner; readonly inputs: InputValues | undefined; + /** + * The live allowances for an accepted reply's exact copies. + * + * The same object the journal gate closes over, so a grant added here is + * one this execution's own gate consults — and nothing a document binds + * under the policy's name can add to or read from it. + */ + readonly grants: Set; - constructor(scanner: SecretScanner, inputs: InputValues | undefined) { + constructor( + scanner: SecretScanner, + inputs: InputValues | undefined, + grants: Set, + ) { super(); this.scanner = scanner; this.inputs = inputs; + this.grants = grants; Object.freeze(this); } } @@ -400,15 +415,28 @@ export function* useSecretDetection( }; if (requested === false) { yield* CurrentDetection.set(new DisabledDetection(inputs)); - return { journal: stream, disclose: projectedDisclosure(discloseFreely) }; + return { + journal: stream, + disclose: projectedDisclosure(discloseFreely), + checkReplyText: replyTextCheck(undefined, inputs), + allowReplyCopies: () => () => {}, + }; } const create = (yield* CurrentScannerFactory.get()) ?? createSecretScanner; const scanner = create(); - yield* CurrentDetection.set(new EnabledDetection(scanner, inputs)); + const grants = new Set(); + yield* CurrentDetection.set(new EnabledDetection(scanner, inputs, grants)); return { - journal: guardWithSecretDetection(stream, scanner), + journal: guardWithSecretDetection(stream, scanner, grants), disclose: projectedDisclosure(discloseThrough(scanner)), + checkReplyText: replyTextCheck(scanner, inputs), + allowReplyCopies(grant) { + grants.add(grant); + return () => { + grants.delete(grant); + }; + }, }; } @@ -424,6 +452,61 @@ export function* useSecretDetection( export interface SecretDetection { readonly journal: DurableStream; readonly disclose: SecretDisclosure; + readonly checkReplyText: ReplyTextCheck; + allowReplyCopies(grant: ReplyCopyGrant): () => void; +} + +export type ReplyTextCheck = (text: string, copies: readonly ApprovedMatch[]) => Operation; + +export function* captureReplyTextCheck(): Operation { + const detection = yield* currentDetection(); + return replyTextCheck( + detection instanceof EnabledDetection ? detection.scanner : undefined, + detection.inputs, + ); +} + +function replyTextCheck( + scanner: SecretScanner | undefined, + inputs: InputValues | undefined, +): ReplyTextCheck { + return function* (text, copies) { + if (inputs !== undefined) { + const projected = inputs.display(text); + if (!projected.ok || projected.value !== text || inputs.contains(text)) { + throw new SecretScannerError("protected input in generated output"); + } + } + if (scanner === undefined) { + return; + } + const serialized = JSON.stringify(text); + const findings = yield* scanner.scan(serialized); + if (findings.length === 0) { + return; + } + const spans = copies.length === 0 ? undefined : yield* locateSecrets(scanner, serialized); + const offsets = boundariesOf(text); + if ( + spans === undefined || + spans.length !== findings.length || + offsets === undefined || + !spans.every((span) => { + const start = offsets.get(span.start - 1); + const end = offsets.get(span.end - 1); + return copies.some( + (copy) => + copy.start === start && + copy.end === end && + copy.ruleId === span.ruleId && + copy.messageId === span.messageId && + text.slice(copy.start, copy.end) === copy.bytes, + ); + }) + ) { + throw new SecretDetectedError(findings); + } + }; } /** @@ -475,14 +558,118 @@ function discloseThrough(scanner: SecretScanner): SecretDisclosure { * filtered run still publishes into the journal its provider selected. An * ordinary guard elsewhere produces an unproven wrapper. */ -function guardWithSecretDetection(stream: DurableStream, scanner: SecretScanner): DurableStream { +function guardWithSecretDetection( + stream: DurableStream, + scanner: SecretScanner, + grants: ReadonlySet, +): DurableStream { const guarded = guardDurableStream(stream, function* (event) { const serialized = serializeDurableEvent(event); const findings = yield* scanner.scan(serialized); - if (findings.length > 0) { + if (findings.length > 0 && !(yield* granted(event, serialized, findings.length))) { throw new SecretDetectedError(findings); } }); - return preserveJournalProvenance(stream, guarded); + return preserveJournalProvenance(stream, { + readAll: () => guarded.readAll(), + *append(event) { + yield* guarded.append(event); + for (const grant of grants) { + grant.published?.(event); + } + }, + }); + + /** + * Whether one live grant accounts for every finding in this event. + * + * Asked only when there are findings and a grant is live. The spans are the + * same detector's over the same serialized text, and must be as many as the + * findings: a grant judges every match, never a subset. + */ + function* granted(event: DurableEvent, serialized: string, count: number): Operation { + const claiming = [...grants].find((grant) => grant.claims(event)); + if (claiming === undefined) { + return false; + } + const spans = yield* locateSecrets(scanner, serialized); + if (spans === undefined || spans.length !== count) { + return false; + } + return claiming.admits(event, serialized, spans); + } +} + +/** + * Add one allowance for an accepted reply's exact copies to this execution's + * own gate, and answer how to retire it. + * + * Host-only: only the host that accepted the reply adds one, from inside the + * execution whose gate it changes. Outside an execution, or where detection is + * off, nothing is added — off already admits everything, and a missing policy + * admits nothing. + */ +export function* grantReplyCopies(grant: ReplyCopyGrant): Operation<() => void> { + const detection = yield* currentDetection(); + if (!(detection instanceof EnabledDetection)) { + return () => {}; + } + detection.grants.add(grant); + return () => { + detection.grants.delete(grant); + }; +} + +/** + * Where this execution's scanner finds credentials in one completed reply, in + * the reply's own coordinates — or nothing when it cannot say exactly. + * + * Scanned the way the journal will hold it, as a JSON string, so a span is a + * span of what the gate refused. A reply that holds a protected input, a + * scanner this module did not build, a range that falls inside an escape, and + * a scanner that fails all answer nothing: the reply stays withheld and is + * never offered for approval. + */ +export function* locateReplySecrets(text: string): Operation { + let detection: ExecutionDetection; + try { + detection = yield* currentDetection(); + } catch { + return undefined; + } + if (!(detection instanceof EnabledDetection)) { + return undefined; + } + const inputs = detection.inputs; + if (inputs !== undefined) { + const shown = inputs.display(text); + if (inputs.contains(text) || !shown.ok || shown.value !== text) { + return undefined; + } + } + let spans: SecretSpan[] | undefined; + try { + spans = yield* locateSecrets(detection.scanner, JSON.stringify(text)); + } catch { + return undefined; + } + if (spans === undefined || spans.length === 0) { + return undefined; + } + const offsets = boundariesOf(text); + if (offsets === undefined) { + return undefined; + } + const mapped: SecretSpan[] = []; + for (const span of spans) { + // One past the literal's opening quote is the value's first unit. + const start = offsets.get(span.start - 1); + const end = offsets.get(span.end - 1); + if (start === undefined || end === undefined || end <= start) { + return undefined; + } + mapped.push(Object.freeze({ ...span, start, end })); + } + return Object.freeze(mapped); } diff --git a/packages/core/src/secrets/reply-grant.ts b/packages/core/src/secrets/reply-grant.ts new file mode 100644 index 000000000..14e89f8d4 --- /dev/null +++ b/packages/core/src/secrets/reply-grant.ts @@ -0,0 +1,578 @@ +/** + * The closed exception secret detection makes for one accepted reply. + * + * A host that withheld a completed reply may ask the person in front of it + * whether to keep it. When the person accepts, the reply's + * exact bytes are no longer something the journal must refuse — but only + * where those bytes really are that reply: its acceptance record, the + * evaluation of exactly that source, the question choices that source wrote + * and the answer selected from them, and the one feedback turn that carries + * that answer back. Anywhere else the same value is a new finding. + * + * So a grant is not a switch and not a fingerprint list. Each one names one + * event — by the coroutine, effect type and durable name its owner is about to + * append — and the exact string fields of that event that may hold an + * approved match. The gate scans the whole serialized event as always. Every + * finding must then translate, through the event's actual serialization, into + * one of those fields at a position the grant can account for, with the same + * rule, message and bytes. One finding anywhere else, or one the grant cannot + * place, and the event is refused exactly as it would have been. + * + * Nothing here is durable, rendered or reachable from a document: the grants + * live in the execution's own detection, beside its scanner, and are added + * and retired only by the host that accepted the reply. + */ + +import type { DurableEvent } from "@executablemd/durable-streams"; +import { ELICITATION_SCHEMA_FIELD, readElicitationAnswer } from "../elicit-journal.ts"; +import { dataStringLiteral } from "../generated-expressions.ts"; +import type { SourceRange } from "../document-targets.ts"; +import { isJsonObject } from "../json.ts"; +import { secretInputFields } from "./input-schema.ts"; +import type { SecretSpan } from "./scanner.ts"; + +/** One approved match, in the coordinates of the reply it was found in. */ +export interface ApprovedMatch extends SecretSpan { + /** The exact matched bytes, held only in memory for the life of the grant. */ + readonly bytes: string; +} + +/** A value the evaluation's own question answered with, carried to feedback. */ +export interface FeedbackSlice extends SourceRange { + readonly coroutineId: string; + readonly index: number; +} + +interface CarriedAnswer { + readonly coroutineId: string; + readonly index: number; + readonly text: string; + readonly matches: readonly ApprovedMatch[]; +} + +export interface ElicitChoiceSelection { + readonly coroutineId: string; + readonly index: number; + readonly name: string; + readonly fingerprint: string; + readonly field: string; + readonly option: number; +} + +/** One live allowance, as the gate asks it. */ +export interface ReplyCopyGrant { + published?(event: DurableEvent): void; + /** + * Whether this grant speaks for `event` at all. + * + * Exactly one grant may speak for an event; one that does not leaves it to + * the next, and an event no grant speaks for is refused on any finding. + */ + claims(event: DurableEvent): boolean; + /** Whether every span the scanner found in `serialized` is accounted for. */ + admits(event: DurableEvent, serialized: string, spans: readonly SecretSpan[]): boolean; +} + +/** The acceptance record of one withheld turn, and nothing else. */ +export function acceptanceGrant(input: { + readonly type: string; + readonly name: string; + readonly text: string; + readonly approved: readonly ApprovedMatch[]; +}): ReplyCopyGrant { + let used = false; + return { + claims(event) { + return ( + !used && + event.type === "yield" && + event.coroutineId === "root" && + event.description.type === input.type && + event.description.name === input.name + ); + }, + admits(event, serialized, spans) { + used = true; + if (event.type !== "yield" || event.result.status !== "ok") { + return false; + } + const value = event.result.value; + if (!isJsonObject(value) || value.text !== input.text) { + return false; + } + const placed = place(serialized, [["result", "value", "text"]]); + return ( + placed !== undefined && + spans.every((span) => { + const at = within(placed, span); + return at !== undefined && exactly(input.approved, span, at.start, at.end); + }) + ); + }, + }; +} + +/** + * One evaluation of exactly the accepted source. + * + * Bound on its admission record: the first `generated_xmd` admission named + * for this request whose retained source is byte-equal to the accepted reply + * fixes the evaluation's own coroutine. Afterwards the grant speaks only for + * that coroutine's question records. Their schema may hold an approved match + * only as an enum choice the `` element at that record's own source + * position wrote, and their answer only as one of those choices, selected + * whole. Anything else the evaluation journals is unaccounted for. + */ +export function evaluationGrant(input: { + readonly id: string; + readonly source: string; + readonly approved: readonly ApprovedMatch[]; +}): ReplyCopyGrant & { + readonly bind: (id: string) => void; + readonly schemas: Map; + readonly select: (selection: ElicitChoiceSelection) => void; + readonly carried: () => readonly CarriedAnswer[]; +} { + let coroutine: string | undefined; + let admitted = false; + const carried: CarriedAnswer[] = []; + const positions = new Map(); + let pending: CarriedAnswer | undefined; + const schemas = new Map(); + const selections = new Map>(); + const admission = `generated:${input.id}`; + const owns = (id: string): boolean => + coroutine !== undefined && (id === coroutine || id.startsWith(`${coroutine}.`)); + return { + schemas, + select(selection) { + if ( + owns(selection.coroutineId) && + selection.index === (positions.get(selection.coroutineId) ?? 0) + ) { + const fields = selections.get(selection.coroutineId) ?? new Map(); + fields.set(selection.field, selection); + selections.set(selection.coroutineId, fields); + } + }, + published(event) { + if (!owns(event.coroutineId) || event.type !== "yield") { + return; + } + const index = positions.get(event.coroutineId) ?? 0; + if (pending?.coroutineId === event.coroutineId && pending.index === index) { + carried.push(pending); + pending = undefined; + } + positions.set(event.coroutineId, index + 1); + selections.delete(event.coroutineId); + }, + bind(id) { + coroutine ??= id; + }, + carried: () => Object.freeze([...carried]), + claims(event) { + if (event.type !== "yield") { + return false; + } + if (!admitted) { + return ( + event.description.type === "generated_xmd" && + event.description.name === admission && + event.coroutineId === coroutine + ); + } + return owns(event.coroutineId) && event.description.type === "elicit"; + }, + admits(event, serialized, spans) { + if (event.type !== "yield") { + return false; + } + if (!admitted) { + if (event.result.status !== "ok") { + return false; + } + const value = event.result.value; + if (!isJsonObject(value) || value.source !== input.source) { + return false; + } + const placed = place(serialized, [["result", "value", "source"]]); + const matched = + placed !== undefined && + spans.every((span) => { + const at = within(placed, span); + return at !== undefined && exactly(input.approved, span, at.start, at.end); + }); + if (matched) { + admitted = true; + } + return matched; + } + return admitsQuestion(event, serialized, spans); + }, + }; + + function admitsQuestion( + event: DurableEvent & { type: "yield" }, + serialized: string, + spans: readonly SecretSpan[], + ): boolean { + const schema = event.description[ELICITATION_SCHEMA_FIELD]; + const position = event.description["executablemd.source-position"]; + if (!isJsonObject(schema) || !isJsonObject(position)) { + return false; + } + if (position.generatedSource !== input.id || typeof position.offset !== "number") { + return false; + } + const schemaRange = schemas.get(position.offset); + if (schemaRange === undefined) { + return false; + } + const authored = input.source.slice(schemaRange.start, schemaRange.end); + const properties = isJsonObject(schema.properties) ? schema.properties : undefined; + if (properties === undefined) { + return false; + } + const leaves: Leaf[] = []; + const choices = selections.get(event.coroutineId); + selections.delete(event.coroutineId); + const index = positions.get(event.coroutineId) ?? 0; + const projection = + event.result.status === "ok" ? readElicitationAnswer(event, index) : undefined; + if (projection?.ok === false) { + return false; + } + const answer = projection?.value; + const protectedReceipt = secretInputFields(schema).length > 0; + for (const [name, property] of Object.entries(properties)) { + if ( + !isJsonObject(property) || + (projection !== undefined && (property.type !== "string" || property.writeOnly === true)) + ) { + continue; + } + const offered = Array.isArray(property.enum) ? property.enum : []; + offered.forEach((choice, index) => { + const origin = originOf(name, index, choice); + if (origin !== undefined) { + leaves.push({ + path: ["description", ELICITATION_SCHEMA_FIELD, "properties", name, "enum", index], + origin, + }); + } + }); + const selected = choices?.get(name); + if ( + selected !== undefined && + property.type === "string" && + property.writeOnly !== true && + selected.name === event.description.name && + selected.fingerprint === event.description.input && + isJsonObject(answer) && + typeof answer[name] === "string" && + answer[name] === offered[selected.option] + ) { + const origin = originOf(name, selected.option, answer[name]); + if (origin !== undefined) { + const path = protectedReceipt + ? ["result", "value", "executablemd.input", "members", name, "value"] + : ["result", "value", name]; + leaves.push({ path, origin, answerField: name }); + } + } + } + const placed = place( + serialized, + leaves.map((leaf) => leaf.path), + ); + if (placed === undefined) { + return false; + } + const found: ApprovedMatch[] = []; + const text = answer === undefined ? undefined : JSON.stringify(answer, null, 2); + for (const span of spans) { + const at = within(placed, span); + if (at === undefined) { + return false; + } + const leaf = leaves[at.leaf]; + if ( + leaf === undefined || + !exactly(input.approved, span, leaf.origin + at.start, leaf.origin + at.end) + ) { + return false; + } + if (leaf.answerField !== undefined && text !== undefined) { + const fieldPlacement = place(`${text}\n`, [[leaf.answerField]], 2)?.[0]; + const start = encodedOffset(fieldPlacement, at.start); + const end = encodedOffset(fieldPlacement, at.end); + if (start === undefined || end === undefined) { + return false; + } + const bytes = at.value.slice(at.start, at.end); + if (text.slice(start, end) !== bytes) { + return false; + } + found.push({ ...span, bytes, start, end }); + } + } + if (found.length > 0 && text !== undefined) { + pending = { coroutineId: event.coroutineId, index, text, matches: found }; + } + return true; + + function originOf(name: string, index: number, value: unknown): number | undefined { + if (typeof value !== "string" || schemaRange === undefined) { + return undefined; + } + try { + const literal = dataStringLiteral(authored, ["properties", name, "enum", index]); + return literal?.value === value ? schemaRange.start + literal.start : undefined; + } catch { + return undefined; + } + } + } +} + +/** + * The next turn's input, where it carries the evaluation's own answer. + * + * `slices` are the host's statement of where, in the text it handed the turn, + * it placed that answer. The turn's final input must still begin with exactly + * that text, so a composition that changed it inherits nothing, and each + * finding must lie inside one slice with the bytes the evaluation's answer + * carried. + */ +export function feedbackGrant(input: { + readonly name: string; + readonly type: string; + readonly prepare: (delegated: string) => readonly ApprovedMatch[]; +}): ReplyCopyGrant { + let used = false; + return { + claims(event) { + return ( + !used && + event.type === "yield" && + event.coroutineId === "root" && + event.description.type === input.type && + event.description.name === input.name + ); + }, + admits(event, serialized, spans) { + used = true; + if (event.type !== "yield") { + return false; + } + const delegated = event.description.input; + if (typeof delegated !== "string") { + return false; + } + const copies = input.prepare(delegated); + const placed = place(serialized, [["description", "input"]]); + return ( + placed !== undefined && + spans.every((span) => { + const at = within(placed, span); + if (at === undefined) { + return false; + } + return exactly(copies, span, at.start, at.end); + }) + ); + }, + }; +} + +export function carriedCopies( + text: string, + slices: readonly FeedbackSlice[], + carried: readonly CarriedAnswer[], +): ApprovedMatch[] { + return slices.flatMap((slice) => { + const answer = carried.find( + (one) => one.coroutineId === slice.coroutineId && one.index === slice.index, + ); + if (answer === undefined || text.slice(slice.start, slice.end) !== answer.text) { + return []; + } + return answer.matches.map((match) => ({ + ...match, + start: slice.start + match.start, + end: slice.start + match.end, + })); + }); +} + +/** Whether one located span is exactly one approved match. */ +function exactly( + approved: readonly ApprovedMatch[], + span: SecretSpan, + start: number, + end: number, +): boolean { + return approved.some( + (one) => + one.ruleId === span.ruleId && + one.messageId === span.messageId && + one.start === start && + one.end === end, + ); +} + +type Path = readonly (string | number)[]; + +interface Leaf { + readonly path: Path; + readonly origin: number; + readonly answerField?: string; +} + +/** One string field, where the event's serialization holds its literal. */ +interface Placed { + readonly value: string; + /** Offset of the literal's opening quote in the serialized event. */ + readonly at: number; + /** For each content offset of the literal, the value index it begins, or -1. */ + readonly boundaries: Map; + readonly length: number; +} + +/** + * Where each named string field sits in this exact serialization. + * + * Found by serializing the same event with each field replaced by a unique + * marker and then putting the real literals back: what that reconstructs must + * be byte-identical to what the gate scanned, or nothing is placed. A field + * that is absent or not a string places nothing either. + */ +function place(serialized: string, paths: readonly Path[], indent?: number): Placed[] | undefined { + let copy: unknown; + try { + copy = JSON.parse(serialized); + } catch { + return undefined; + } + const nonce = crypto.randomUUID(); + const values: string[] = []; + for (const [index, path] of paths.entries()) { + const parent = walk(copy, path.slice(0, -1)); + const key = path[path.length - 1]; + if (parent === undefined || key === undefined) { + return undefined; + } + const value = (parent as Record)[key]; + if (typeof value !== "string") { + return undefined; + } + values.push(value); + (parent as Record)[key] = `\u0001${nonce}:${index}\u0001`; + } + const marked = `${JSON.stringify(copy, null, indent)}\n`; + const located = values + .map((value, index) => { + const marker = JSON.stringify(`\u0001${nonce}:${index}\u0001`); + const at = marked.indexOf(marker); + return { index, value, at, marker }; + }) + .sort((a, b) => a.at - b.at); + if (located.some((one) => one.at < 0)) { + return undefined; + } + let rebuilt = ""; + let cursor = 0; + const placed: Placed[] = new Array(values.length); + for (const one of located) { + rebuilt += marked.slice(cursor, one.at); + const literal = JSON.stringify(one.value); + const boundaries = boundariesOf(one.value); + if (boundaries === undefined) { + return undefined; + } + placed[one.index] = { + value: one.value, + at: rebuilt.length, + boundaries, + length: literal.length, + }; + rebuilt += literal; + cursor = one.at + one.marker.length; + } + rebuilt += marked.slice(cursor); + return rebuilt === serialized ? placed : undefined; +} + +function encodedOffset(field: Placed | undefined, index: number): number | undefined { + if (field === undefined) { + return undefined; + } + for (const [offset, unit] of field.boundaries) { + if (unit === index) { + return field.at + 1 + offset; + } + } + return undefined; +} + +/** Translate one serialized span into a field's own value coordinates. */ +function within( + placed: readonly Placed[], + span: SecretSpan, +): { leaf: number; value: string; start: number; end: number } | undefined { + for (const [leaf, one] of placed.entries()) { + if (span.start <= one.at || span.end >= one.at + one.length) { + continue; + } + const start = one.boundaries.get(span.start - one.at - 1); + const end = one.boundaries.get(span.end - one.at - 1); + if (start === undefined || end === undefined || start < 0 || end < 0) { + return undefined; + } + return { leaf, value: one.value, start, end }; + } + return undefined; +} + +/** + * Each content offset of `JSON.stringify(value)` that begins a code unit of + * `value`, mapped to that unit's index — with the end of the value mapped to + * its length. Offsets inside an escape sequence or between the halves of a + * surrogate pair are not boundaries. + */ +export function boundariesOf(value: string): Map | undefined { + const boundaries = new Map(); + let offset = 0; + for (let index = 0; index < value.length; ) { + const code = value.charCodeAt(index); + let width = 1; + if (code >= 0xd800 && code <= 0xdbff && index + 1 < value.length) { + const next = value.charCodeAt(index + 1); + if (next >= 0xdc00 && next <= 0xdfff) { + width = 2; + } + } + boundaries.set(offset, index); + offset += JSON.stringify(value.slice(index, index + width)).length - 2; + index += width; + } + boundaries.set(offset, value.length); + return offset === JSON.stringify(value).length - 2 ? boundaries : undefined; +} + +function walk(value: unknown, path: Path): object | undefined { + let current: unknown = value; + for (const key of path) { + if (current === null || typeof current !== "object") { + return undefined; + } + current = (current as Record)[key]; + } + return current !== null && typeof current === "object" ? current : undefined; +} + +/** The approved matches of one reply, from the spans its own scan located. */ +export function approvedMatches(text: string, spans: readonly SecretSpan[]): ApprovedMatch[] { + return spans.map((span) => Object.freeze({ ...span, bytes: text.slice(span.start, span.end) })); +} diff --git a/packages/core/src/secrets/scanner.ts b/packages/core/src/secrets/scanner.ts index 6aae0ecce..ac2748e63 100644 --- a/packages/core/src/secrets/scanner.ts +++ b/packages/core/src/secrets/scanner.ts @@ -24,6 +24,50 @@ export interface SecretScanner { scan(content: string): Operation; } +/** + * Where one detection matched, in the coordinates of the scanned text. + * + * Core-private and never journaled, rendered or attached to an error: a range + * over text that holds a credential is a pointer at that credential. It + * exists so the one owner that may present or approve a reply's findings can + * say exactly which bytes it presented and approved. + */ +export interface SecretSpan { + readonly ruleId: string; + readonly messageId: string; + readonly start: number; + readonly end: number; +} + +class RangedScanner implements SecretScanner { + readonly scan: SecretScanner["scan"]; + readonly #ranges: (content: string) => Operation; + + constructor(scan: SecretScanner["scan"], ranges: (content: string) => Operation) { + this.scan = scan; + this.#ranges = ranges; + } + + static locate(scanner: SecretScanner, content: string): Operation | undefined { + return #ranges in scanner ? scanner.#ranges(content) : undefined; + } +} + +/** + * Where `scanner` finds credentials in `content`, or nothing when this module + * did not build it. + * + * Fails closed exactly as `scan()` does: a detector that throws is a + * {@link SecretScannerError} carrying no source. + */ +export function* locateSecrets( + scanner: SecretScanner, + content: string, +): Operation { + const locate = RangedScanner.locate(scanner, content); + return locate === undefined ? undefined : yield* locate; +} + /** The subset of a Secretlint message a finding is built from. */ interface LintMessage { ruleId: string; @@ -89,7 +133,15 @@ export function createSecretScanner(filePath = "/journal.jsonl"): SecretScanner export function createScannerWith(detect: LintFn, filePath = "/journal.jsonl"): SecretScanner { const fingerprint = createFingerprinter(); - return { + function* detected(content: string): Operation { + try { + return (yield* until(detect(content, filePath))).messages; + } catch { + throw new SecretScannerError("scanning"); + } + } + + const scanner: SecretScanner = { *scan(content: string): Operation { if (content.length === 0) { return []; @@ -119,4 +171,27 @@ export function createScannerWith(detect: LintFn, filePath = "/journal.jsonl"): } }, }; + return new RangedScanner(scanner.scan, function* (content: string): Operation { + if (content.length === 0) { + return []; + } + const messages = yield* detected(content); + try { + return messages.map((message) => { + const [start, end] = message.range; + if ( + !Number.isInteger(start) || + !Number.isInteger(end) || + start < 0 || + end <= start || + end > content.length + ) { + throw new Error("range"); + } + return Object.freeze({ ruleId: message.ruleId, messageId: message.messageId, start, end }); + }); + } catch { + throw new SecretScannerError("normalizing findings"); + } + }); } diff --git a/packages/core/tests/generated-xmd-root.test.ts b/packages/core/tests/generated-xmd-root.test.ts index d39c15f21..b8b52196e 100644 --- a/packages/core/tests/generated-xmd-root.test.ts +++ b/packages/core/tests/generated-xmd-root.test.ts @@ -50,6 +50,8 @@ import { installIdentities } from "../src/invocation-identity.ts"; import type { InvocationForm } from "../src/invocation-identity.ts"; import { Documentation } from "../src/documentation-api.ts"; import { Execution } from "../src/execute.ts"; +import { useSecretDetection } from "../src/secrets/policy.ts"; +import type { SecretDetection } from "../src/secrets/policy.ts"; import { retainedSource } from "../src/root-source.ts"; import { useTempFileCompiler } from "../src/temp-file-compiler.ts"; import { HELD_ROOT_PATH, withHeldHost } from "./support/held-host.ts"; @@ -925,10 +927,12 @@ describe("Tier GRM — a record's marker is what authorizes the evaluation", () */ function boundRoot( observations: readonly GeneratedObservation[], + detection: SecretDetection, ): (request: GeneratedXmdRequest) => Operation> { const identity = installIdentities([], [], [SYNTAX_PROTECTED]); identity.activate(); return bindGeneratedRootEvaluator({ + detection, select: () => ({ observations, mutations: [], describing: [] }), componentRouting: identity.componentRouting, reference: undefined, @@ -942,12 +946,19 @@ describe("Tier GRM — a record's marker is what authorizes the evaluation", () performed: string[], ): Operation> { return scoped(function* () { + const detection = yield* useSecretDetection(undefined, stream); const held: { answer?: Result } = {}; yield* durableRun( function* (): Operation { - held.answer = yield* boundRoot([countedProbe(performed)])({ id: ID, source: COMMON }); + held.answer = yield* boundRoot( + [countedProbe(performed)], + detection, + )({ + id: ID, + source: COMMON, + }); }, - { stream, coroutineId: COROUTINE }, + { stream: detection.journal, coroutineId: COROUTINE }, ); if (held.answer === undefined) { throw new Error("the durable run recorded no root outcome"); @@ -1109,16 +1120,16 @@ describe("Tier GRM — a record's marker is what authorizes the evaluation", () // definitions under one name is a malformed table, refused before // anything is admitted. const performed: string[] = []; + const detection = yield* useSecretDetection(undefined, new InMemoryStream()); const held: { answer?: Result } = {}; yield* durableRun( function* (): Operation { - held.answer = yield* boundRoot([ - countedProbe(performed), - // A second, different implementation under the same name. - countedProbe([]), - ])({ id: ID, source: COMMON }); + held.answer = yield* boundRoot( + [countedProbe(performed), countedProbe([])], + detection, + )({ id: ID, source: COMMON }); }, - { stream: new InMemoryStream(), coroutineId: COROUTINE }, + { stream: detection.journal, coroutineId: COROUTINE }, ); expect(held.answer?.ok).toBe(false); diff --git a/specs/acp-client-spec.md b/specs/acp-client-spec.md index 1927fd018..b06cfecd9 100644 --- a/specs/acp-client-spec.md +++ b/specs/acp-client-spec.md @@ -1139,11 +1139,15 @@ retained without that reply. Before the record is appended, the execution's own scanner reads the reply as the record serializes it; a finding keeps the completed record — the delivered input, session and native identity, configuration, permissions and usage — with empty `text` and `withheld: true`. -The call answers a fixed `AgentReplyWithheldError`, exported from the host -surface and naming no rule, position or fingerprint; it is the only answer that -identifies a withheld completion, so a host acts on it rather than on finding a -withheld record. The conversation stays usable: the next turn continues the same -native conversation, and a reopening replays this one from its record. +The default answer is `AgentReplyWithheldError`, exported from the host +surface with a fixed message. In a foreground REPL the same Prompt call can +await a local decision after the withheld completion is durable. Withhold, +dismissal and interruption leave that completion in force. Accept appends +`agent_prompt_reply` under the same Prompt name in the same root sequence, +then the call returns the original reply as its successful text. The record +adds no provider Prompt, request receipt or native conversation. Reopening +restores the resulting turn from the recorded sequence. Before an acceptance +append succeeds, the withheld completion remains the saved result. Everything else still reaches the journal's own guard, which refuses the append as before: a scanner that cannot answer, a policy that is not the execution's, a failed or cancelled turn's partial text, a credential anywhere else in the @@ -2132,6 +2136,12 @@ its caller supplied, from Prompt `timeout` or the enclosing AgentProvider ## The CLI Sidekick attachment as an ACP backend +An attached exchange can wait on the local REPL's reply approval. Public +updates may report the pending status. Only the local foreground can see and +answer the review. Accept allows standard assistant text from the acknowledged +evaluation of that saved reply. An acceptance with evaluation still absent +has no assistant Output to deliver. + The headless XMD child backend uses the same live dispatch and original input owners through its separately admitted child connection. `xmd repl --attach ` attaches a UI to the same root and selected conversation. Its diff --git a/specs/decisions.md b/specs/decisions.md index a6e8e8095..d3ac176a1 100644 --- a/specs/decisions.md +++ b/specs/decisions.md @@ -13,6 +13,7 @@ conformance belong to [the language specification](executable-mdx-spec.md). | --- | --- | | Mixed durable expansion and intentionally non-durable Operations | [DEC-015](#dec-015-generated-continuation--root-basis-membership-and-an-operation-evaluator) partially supersedes [DEC-001](#dec-001-workflowt-vs-operationt--the-type-boundary) and [DEC-002](#dec-002-expansion-generators-are-workflows); [DEC-034](../packages/durable-streams/specs/DECISIONS.md#dec-034-ephemeral--explicit-escape-hatch-for-non-durable-operations-in-workflows) still governs genuinely non-durable work. Current [generated admission](../architecture.md#agent-permissions-and-generated-xmd) preserves both rules. | | Plan information-request delivery | [DEC-017](#dec-017-plan-information-requests--delivery-history) records sequencing; [Plan architecture](../architecture.md#plan-information-requests) and [the Plan interface](plan-command-spec.md) own current behavior. | +| Foreground approval of a withheld Sidekick reply | [DEC-018](#dec-018-foreground-sidekick-reply-approval); [language](executable-mdx-spec.md#accepted-sidekick-replies), [ACP](acp-client-spec.md#journaling-and-replay) and [REPL](repl-spec.md#review-a-withheld-sidekick-reply) own current behavior. | | Main health and verification ownership | [DEC-013](#dec-013-a-red-main-is-reported-as-an-issue-and-gates-pull-requests-no-merge-queue), [DEC-014](#dec-014-composability-proves-ownership-not-correctness-a-second-time), and [contribution verification](../AGENTS.md#verification). | | Withdrawn terminal-grid surface | [DEC-016](#dec-016-withdraw-the-unimplemented-terminal-grid-surface); [installed structural syntax](../architecture.md#the-installed-structural-syntax-boundary) remains the extension boundary. | | Host-installed Codex CLI for the carried adapter | [Decision record 004](decisions/004-host-installed-codex-cli.md); [ACP Availability](acp-client-spec.md#availability), [Adapter snapshots](acp-client-spec.md#adapter-snapshots) and [the maintained Codex adapter](release-process-spec.md#maintained-codex-adapter) own current behavior. | @@ -885,3 +886,21 @@ selection and user-approval escalation policy. Existing host containment remains in force with its stable-namespace limitation; #227 owns concurrent path-replacement hardening. Deferring those decisions does not defer File or Glob reads. #679 remains separate evaluation work after product delivery. + +## DEC-018: Foreground Sidekick reply approval + +The Product Owner approved a one-reply exception in Issue 1311. Accept keeps +the exact completed reply and allows its ordinary ACP Output and automatic +feedback. The foreground drawer explains those effects before acceptance. + +The first acknowledged Prompt is a completed withholding. It settles earlier +delivery while the local decision is pending and remains the outcome on +dismissal, Stop or process loss before acceptance. A separate acknowledgement +saves the accepted reply in the same turn. The live exception follows exact +producing occurrences into selected Output and feedback. Retained history +restores the saved turn without recreating authority for a new send. + +Canonical Elicit publication establishes the control answer. The accepted +ordering amendment uses a reference in the earlier intent and derives the +answer from the same journal's acknowledged prefix. This gives live, Stop +and cold recovery readers the same boundary. diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index d42a61034..e48d1e090 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -11223,7 +11223,7 @@ identity, its locks and its evidence do not reach its parent or a sibling. Explicit Elicit protection is mandatory and precedes heuristic scanning at publication boundaries. The scanner retains its existing decisions for other content, including when an unmarked value resembles a credential. Scanner opt-out never reveals marked input. -Every execution refuses to persist a durable event that carries a credential. +Every execution checks durable events for credentials. `execute` selects its journal before the durable run starts, so the root component import is already covered, and the same policy holds for every later yield and close. @@ -11240,8 +11240,8 @@ ordinary error handling: the region the effect was written in decides, so a continues, while an effect at a plain root, and a rejection during the root import, fail the run. Either way the offending event is absent, and a later close is a separate append that crosses the policy on its own — so a rejection never -implies an empty journal. There is no allowlist, sanitization, repair, or -approval: a finding is a code or data-flow defect to fix. +implies an empty journal. The foreground REPL can authorize the scoped +Sidekick reply described below. Rendered output is held to the same policy before it leaves the execution. A live chunk the execution's scanner cannot clear — a finding, or a scan that @@ -11261,6 +11261,32 @@ about credentials; it runs whatever gate it is given. XMD's policy is one such gate, installed by default. See the durable-streams README for the decorator's contract. +##### Accepted Sidekick replies + +The REPL can ask about a located finding in one completed Sidekick reply +after the canonical withheld Prompt is acknowledged. Its trusted foreground +decision reaches the Agent package through the still-live continuation of +that turn. The Agent package publishes a distinct acceptance immediately after +that Prompt in the same root sequence, carrying the exact original reply and an +audit of approved rule, message and ranges. The original Prompt and its receipt stand. The +acceptance uses no new turn ordinal. Replay restores the pair without +recovering a live approval or contacting a provider. + +The same execution scans the complete serialization at each append. Its +one-use exception applies to the acceptance's text field, the same Send's +generated admission for the accepted source and approved byte ranges whose +provenance remains known through evaluation and ordinary automatic feedback. +Core clears the actual selected Output, including selected partial Output, +before the Sidekick response writer or ACP receives it. An unchanged literal +in selected rendering can carry its approved range. A normalized Elicit enum +choice can carry its authored range into the acknowledged answer when the +foreground form selected that choice in the same field. The next feedback +turn verifies the exact composed input before retaining an intent or reaching +the provider. Appended context preserves approval for exact ranges in the +unchanged original feedback prefix. The scanner applies ordinary detection to +findings in appended context, across the join, and in the next reply. +Explicitly protected input keeps its mandatory projection throughout. + ##### At the command line `xmd` is a host, so the same policy reaches every document it runs. Detection is diff --git a/specs/repl-spec.md b/specs/repl-spec.md index fdd6f3845..ceb508297 100644 --- a/specs/repl-spec.md +++ b/specs/repl-spec.md @@ -1156,6 +1156,21 @@ Permission composition runs outermost first: core's audit observer, this session's authoritative policy, document/provider composition, then base denial. One private grant ledger belongs to each inherited Prompt scope. +### Review a withheld Sidekick reply + +When secret detection withholds a completed Sidekick reply, the local REPL +offers a foreground drawer for that Send. It shows the rule and reply +position with masked excerpts. Reveal draws the matched text locally. +Withhold and close settle the question as withheld. Accept saves the original +reply and permits ordinary ACP Output and automatic feedback. The drawer +identifies this disclosure before a person accepts. + +The mounted frame and the waiting question identify each action. Pending +reply text and excerpts exist only in the foreground presentation. Attached +clients can see pending status. The live question belongs to the local +foreground user, independent of provider permissions and authored questions. +The Send stays busy until its continuation and owned cleanup join. + ## Move between regions and edit within them From Draft, one Tab enters the tabs at Bindings without selecting it. Continue @@ -3736,11 +3751,17 @@ and a reply that asked a question has to be told the answer. Canonical syntax self-description is the one exception — its return is the reply's to render or capture — and an ordinary component return is never intercepted. -A question's answer is retained the way every other operation's is: an intent -before anybody is asked and an outcome once they have answered. So a question -still waiting when a process stops is readable as an operation that began and -whose result nobody knows, and it is reported as that rather than re-asked or -given an answer nobody gave. +A generated Elicit intent is acknowledged before the form opens. It names the +canonical Elicit effect, question fingerprint and yield coordinate in the +response's admitted evaluation. Live feedback, display and recovery derive +the outcome from the matching acknowledged Elicit Yield in that prefix. +Each child counts its own yields. A later intent at the same coordinate takes +over that live interval. A committed refusal is a settled operation. +Interruption before the Yield leaves the intent unfinished. A protected +receipt supplies public siblings and a redaction for marked fields. Recovery +describes the committed operation and withholds any answer that the ordinary +scanner cannot clear for a new Prompt. Existing intent and outcome pairs +remain readable. Each answer is in its own terms, read back from what was retained. The binding map is the map itself, with no provenance or status put inside it. A filled @@ -4167,14 +4188,16 @@ happened. It claims no entry and reserves nothing. A response that asked for nothing and failed at nothing ends the exchange: the next move is the person's. A turn the provider did not finish ends it too, because a turn that produced nothing is not a reply to correct. A completed -turn whose reply secret detection withheld ends it without pausing: its record -establishes that the turn received what it was sent, so the feedback it carried -and the TODO changes the turn selected are delivered. Nothing from the reply is -saved or evaluated. A live preview may have shown a prefix the disclosure policy -cleared before the credential arrived; the finding withdraws that preview, as -the live preview rule under [Who owns what](#who-owns-what) states, and the chat -then shows the saved turn with this account. The next message from the composer continues the same native -conversation. A turn that names a native conversation other than the one this +turn whose reply secret detection withholds first saves a completed turn with +empty reply text. The save establishes delivery of the feedback and TODO +changes the turn selected. The live preview is withdrawn. The local REPL can +then wait for [reply approval](#review-a-withheld-sidekick-reply). Withhold, +close, Stop, process exit and a reopen with only the withheld turn preserve +that saved outcome. The next explicit message continues the same native +conversation. Accept acknowledges the original source before evaluating it +within the same Send. A reopen after acceptance restores the saved turn and +any separately acknowledged Output. Evaluation still absent at reopen stays +absent. A turn that names a native conversation other than the one this chat established is not withheld: it keeps the refusal any other credential in a record receives, records no completion and delivers nothing. There is no limit on how many turns one message may take: each one costs a provider turn, a