From 34889d2449fc2683bc224d73bfff6d39422eafad Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 10:29:48 -0500 Subject: [PATCH 01/17] feat(core): add image_generate tool --- packages/core/src/plugin/internal.ts | 5 + .../core/src/tool/plugin/image-generate.ts | 336 ++++++++++++++++++ packages/core/src/tool/plugin/opencode.ts | 5 +- packages/core/test/plugin/fixture.ts | 8 +- .../core/test/tool-image-generate.test.ts | 280 +++++++++++++++ 5 files changed, 631 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/tool/plugin/image-generate.ts create mode 100644 packages/core/test/tool-image-generate.test.ts diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index 0b00caa0ccc4..00df9f767a1f 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -39,6 +39,7 @@ import { Environment } from "../environment/index.js" import { FileAccess } from "../file-access.js" import { FileMutation } from "../file-mutation.js" import { Formatter } from "../formatter.js" +import { Generate } from "../generate.js" import { Form } from "../form.js" import { FileSystem } from "../filesystem.js" import { LocationWatcherPolicy } from "../filesystem/location-watcher-policy.js" @@ -71,6 +72,7 @@ import { PatchTool } from "../tool/plugin/patch.js" import { EditTool } from "../tool/plugin/edit.js" import { GlobTool } from "../tool/plugin/glob.js" import { GrepTool } from "../tool/plugin/grep.js" +import { ImageGenerateTool } from "../tool/plugin/image-generate.js" import { McpResourceTools } from "../tool/plugin/mcp-resource.js" import { OpenCodeTools } from "../tool/plugin/opencode.js" import { QuestionTool } from "../tool/plugin/question.js" @@ -119,6 +121,7 @@ const services = [ FileAccess.Service, FileMutation.Service, Formatter.Service, + Generate.Service, LocationWatcherPolicy.Service, FileSystem.Service, FSUtil.Service, @@ -172,6 +175,7 @@ export const requirements = LayerNode.group([ FileAccess.node, FileMutation.node, Formatter.node, + Generate.node, LocationWatcherPolicy.node, FileSystem.node, FSUtil.node, @@ -236,6 +240,7 @@ const pre = [ EditTool.Plugin, GlobTool.Plugin, GrepTool.Plugin, + ImageGenerateTool.Plugin, OpenCodeTools.Plugin, McpResourceTools.Plugin, QuestionTool.Plugin, diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts new file mode 100644 index 000000000000..bb17d668b734 --- /dev/null +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -0,0 +1,336 @@ +export * as ImageGenerateTool from "./image-generate.js" + +import path from "path" +import { ToolFailure } from "@opencode/ai/effect" +import type { Context } from "@opencode/plugin/effect/plugin" +import { Identifier } from "@opencode/schema/identifier" +import type { Session } from "@opencode/schema/session" +import { Global } from "@opencode/util/global" +import { Duration, Effect, Schema, Stream } from "effect" +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http" +import { Environment } from "../../environment/index.js" +import { FileAccess } from "../../file-access.js" +import { Generate } from "../../generate.js" +import { Mime } from "../../mime.js" +import { Model } from "../../model.js" +import { Permission } from "../../permission.js" +import { collectBoundedResponseBody } from "../http-body.js" + +export const name = "image_generate" +export const MAX_INPUT_BYTES = 20 * 1024 * 1024 +const DOWNLOAD_TIMEOUT = Duration.seconds(30) +const GENERATE_TIMEOUT = Duration.minutes(5) +const INPUT_TYPES = ["image/png", "image/jpeg", "image/webp", "image/gif"] +const EXTENSIONS: Record = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/gif": ".gif", +} +// Families tried in order when the session's provider has no image model, before falling back to the newest. +const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] + +export const description = + 'Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: "image". The result is saved as a PNG and shown to you; mention its path to the user. Each call costs money, so don\'t regenerate unless asked or the result is clearly wrong.' + +export const Input = Schema.Struct({ + prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ + description: + "What to create, or for edits, what to change. Be specific about subject, style, composition and any text that should appear in the image.", + }), + images: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1))).check(Schema.isMinLength(1), Schema.isMaxLength(8)), + ).annotate({ + description: + "Images to edit or use as references, in order. Local file paths (relative to the session directory or absolute) or https URLs. Omit to create a new image.", + }), + aspect: Schema.optionalKey(Schema.Literals(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"])).annotate({ + description: + "Shape of the image. 1:1 for icons and avatars, 16:9 for banners and screenshots, 9:16 for phone screens, 3:2 or 2:3 for photos. Defaults to the model's choice.", + }), + resolution: Schema.optionalKey(Schema.Literals(["1K", "2K"])).annotate({ + description: + "Size tier. 1K is about 1024x1024 pixels. Use 2K only when small text or fine detail matters; it is slower and costs more. Defaults to the model's choice.", + }), + model: Schema.optionalKey(Schema.String).annotate({ + description: + 'NEVER set this unless the user asks for a particular image model. Written as "providerID/modelID". Do not guess the ID: look it up with the models tool using type: "image".', + }), + path: Schema.optionalKey(Schema.String).annotate({ + description: + "Where to save the PNG, relative to the session directory or absolute. Never overwrites. Defaults to a new file in the session's temp folder.", + }), +}) + +export const Output = Schema.Struct({ + path: Schema.String, + model: Schema.String.annotate({ description: "providerID/modelID" }), + width: Schema.optionalKey(Schema.Int), + height: Schema.optionalKey(Schema.Int), +}) + +export const Plugin = { + id: "opencode.tool.image-generate", + effect: Effect.fn("ImageGenerateTool.Plugin")(function* (ctx: Context) { + const access = yield* FileAccess.Service + const environment = yield* Environment.Service + const generate = yield* Generate.Service + const global = yield* Global.Service + const http = yield* HttpClient.HttpClient + const models = yield* Model.Service + const permission = yield* Permission.Service + + const resolveModel = Effect.fn("ImageGenerateTool.resolveModel")(function* ( + input: string | undefined, + sessionID: Session.ID, + images: boolean, + ) { + const available = yield* models.available(["image"]) + if (!input) { + const providerID = (yield* ctx.session.get({ sessionID })).model?.providerID + const model = defaultModel(available, { providerID, images }) + if (model) return model + return yield* new ToolFailure({ + message: images ? "None of the available image models accept input images." : "No image model is available.", + }) + } + const ref = yield* Effect.try({ + try: () => Model.Ref.parse(input), + catch: () => new ToolFailure({ message: `Invalid model "${input}". Use "providerID/modelID".` }), + }) + const model = available.find((model) => model.providerID === ref.providerID && model.id === ref.id) + if (model === undefined) + return yield* new ToolFailure({ + message: `Model "${ref.providerID}/${ref.id}" is not an available image model. Use the models tool with type: "image" to see what is available.`, + }) + if (images && !acceptsImages(model)) { + const capable = available.filter(acceptsImages).slice(0, 3) + return yield* new ToolFailure({ + message: + capable.length === 0 + ? `Model "${ref.providerID}/${ref.id}" does not accept input images, and no available image model does.` + : `Model "${ref.providerID}/${ref.id}" does not accept input images. Image models that do: ${capable.map((model) => `${model.providerID}/${model.id}`).join(", ")}.`, + }) + } + return model + }) + + const readImage = Effect.fn("ImageGenerateTool.readImage")(function* ( + source: string, + context: FileAccess.Invocation, + ) { + const bytes = yield* readBytes(source, context) + const mime = Mime.detect(bytes) + if (!INPUT_TYPES.includes(mime)) + return yield* new ToolFailure({ message: `${source} is not a PNG, JPEG, WebP or GIF image.` }) + return { data: Buffer.from(bytes).toString("base64"), mime, name: path.basename(source) } + }) + + const readBytes = Effect.fn("ImageGenerateTool.readBytes")(function* ( + source: string, + context: FileAccess.Invocation, + ) { + if (/^https?:\/\//i.test(source)) { + if (!source.toLowerCase().startsWith("https://")) + return yield* new ToolFailure({ message: `Image URLs must use https: ${source}` }) + return yield* Effect.gen(function* () { + const response = yield* http + .execute(HttpClientRequest.get(source)) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)) + const type = (response.headers["content-type"] ?? "").split(";", 1)[0]?.trim().toLowerCase() ?? "" + if (!INPUT_TYPES.includes(type)) + return yield* new ToolFailure({ + message: `${source} is not a PNG, JPEG, WebP or GIF image (${type || "no content type"}).`, + }) + return yield* collectBoundedResponseBody( + response, + MAX_INPUT_BYTES, + () => new ToolFailure({ message: `${source} is larger than 20 MiB.` }), + ) + }).pipe( + Effect.timeoutOrElse({ + duration: DOWNLOAD_TIMEOUT, + orElse: () => Effect.fail(new ToolFailure({ message: `Timed out downloading ${source}.` })), + }), + Effect.mapError((error) => + error instanceof ToolFailure ? error : new ToolFailure({ message: `Unable to download ${source}`, error }), + ), + ) + } + const target = yield* access.authorizeRead(source, context) + const info = yield* environment.files + .stat(target.absolute) + .pipe(Effect.catchTag("Environment.NotFound", () => new ToolFailure({ message: `File not found: ${source}` }))) + if (info.size > MAX_INPUT_BYTES) return yield* new ToolFailure({ message: `${source} is larger than 20 MiB.` }) + return (yield* environment.files.read(target.absolute)).bytes + }) + + const exists = (file: string) => + environment.files.stat(file).pipe( + Effect.as(true), + Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), + ) + + // Registered only while an image model is available, and re-evaluated whenever the model catalog changes. + // Subscribing before the first read keeps a change in between from being missed. + let enabled = false + const refresh = models.available(["image"]).pipe( + Effect.flatMap((available) => { + if (available.length > 0 === enabled) return Effect.void + enabled = available.length > 0 + return ctx.tool.reload() + }), + ) + yield* ctx.event.subscribe({ types: ["model.updated"] }).pipe( + Stream.runForEach(() => refresh), + Effect.forkScoped({ startImmediately: true }), + ) + enabled = (yield* models.available(["image"])).length > 0 + + yield* ctx.tool + .transform((editor) => { + if (!enabled) return + editor.add({ + name, + options: { codemode: false }, + description, + input: Input, + output: Output, + execute: (input, context) => + Effect.gen(function* () { + const source = { type: "tool" as const, messageID: context.messageID, id: context.id } + const model = yield* resolveModel(input.model, context.sessionID, input.images !== undefined) + const ref = `${model.providerID}/${model.id}` + const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined + const requested = + target?.absolute ?? + path.join(global.tmp, "images", context.sessionID, `image_${Identifier.create(false)}.png`) + if (yield* exists(requested)) + return yield* new ToolFailure({ message: `${requested} already exists. Choose a new path.` }) + + yield* permission.assert({ + action: name, + resources: [ref], + save: ["*"], + metadata: { ...input, model: ref }, + sessionID: context.sessionID, + agent: context.agent, + source, + }) + if (target) { + yield* access.authorizeExternal([target], context) + yield* permission.assert({ + action: "edit", + resources: [target.resource], + save: ["*"], + sessionID: context.sessionID, + agent: context.agent, + source, + }) + } + const images = input.images + ? yield* Effect.forEach(input.images, (image) => readImage(image, context)) + : undefined + + const result = yield* generate + .image({ + model: Model.Ref.make({ providerID: model.providerID, id: model.id }), + prompt: input.prompt, + n: 1, + ...(images ? { images } : {}), + ...(input.aspect ? { aspect: input.aspect } : {}), + ...(input.resolution ? { resolution: input.resolution } : {}), + }) + .pipe( + Effect.mapError( + (error) => new ToolFailure({ message: `Image generation failed: ${error.message}`, error }), + ), + Effect.timeoutOrElse({ + duration: GENERATE_TIMEOUT, + orElse: () => + Effect.fail(new ToolFailure({ message: "Image generation timed out after 5 minutes." })), + }), + ) + const image = result.images[0] + if (!image) return yield* new ToolFailure({ message: `${ref} returned no image.` }) + const bytes = Buffer.from(image.data, "base64") + const mime = Mime.detect(bytes) + // Providers may return JPEG or WebP; core has no encoder, so the file keeps its real type. + const extension = EXTENSIONS[mime] ?? path.extname(requested) + const file = + path.extname(requested).toLowerCase() === extension + ? requested + : path.join(path.dirname(requested), path.basename(requested, path.extname(requested)) + extension) + if (yield* exists(file)) + return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` }) + yield* environment.files.write(file, bytes) + const size = dimensions(bytes, mime) + const note = + file === requested ? "" : ` The model returned ${mime}, so it was saved with a ${extension} extension.` + return { + output: { path: file, model: ref, ...size }, + content: [ + { + type: "text" as const, + text: `Saved ${size ? `${size.width}x${size.height} ` : ""}image to ${file} (${ref}).${note}`, + }, + { type: "file" as const, uri: `data:${mime};base64,${image.data}`, mime, name: file }, + ], + metadata: { path: file, model: ref }, + } + }).pipe( + Effect.mapError((error) => + error instanceof ToolFailure ? error : new ToolFailure({ message: "Unable to generate image", error }), + ), + ), + }) + }) + .pipe(Effect.orDie) + }), +} + +/** Same provider as the session first, then preferred families, then the newest. */ +export function defaultModel( + available: readonly Model.Info[], + input: { readonly providerID?: string; readonly images: boolean }, +) { + const candidates = input.images ? available.filter(acceptsImages) : available + const own = candidates.filter((model) => model.providerID === input.providerID) + const pool = own.length > 0 ? own : candidates + return PREFERRED_FAMILIES.flatMap((family) => pool.find((model) => model.family === family) ?? [])[0] ?? pool[0] +} + +function acceptsImages(model: Model.Info) { + return model.capabilities.input.some((item) => item.startsWith("image")) +} + +/** Reads pixel dimensions from the image header. */ +export function dimensions(bytes: Uint8Array, mime: string) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + if (mime === "image/png" && bytes.length >= 24) return { width: view.getUint32(16), height: view.getUint32(20) } + if (mime === "image/gif" && bytes.length >= 10) + return { width: view.getUint16(6, true), height: view.getUint16(8, true) } + if (mime === "image/webp" && bytes.length >= 31) { + const chunk = String.fromCharCode(...bytes.subarray(12, 16)) + if (chunk === "VP8 ") return { width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff } + if (chunk === "VP8L") { + const bits = view.getUint32(21, true) + return { width: (bits & 0x3fff) + 1, height: ((bits >> 14) & 0x3fff) + 1 } + } + if (chunk === "VP8X") + return { + width: (view.getUint32(24, true) & 0xffffff) + 1, + height: (view.getUint32(27, true) & 0xffffff) + 1, + } + } + if (mime === "image/jpeg") { + let offset = 2 + while (offset + 9 < bytes.length && bytes[offset] === 0xff) { + const marker = bytes[offset + 1]! + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) + return { width: view.getUint16(offset + 7), height: view.getUint16(offset + 5) } + offset += 2 + view.getUint16(offset + 2) + } + } + return undefined +} diff --git a/packages/core/src/tool/plugin/opencode.ts b/packages/core/src/tool/plugin/opencode.ts index c5783f6dc0a0..44649c37347b 100644 --- a/packages/core/src/tool/plugin/opencode.ts +++ b/packages/core/src/tool/plugin/opencode.ts @@ -50,6 +50,9 @@ export const ModelsInput = Schema.Struct({ provider: Schema.optionalKey(Schema.String).annotate({ description: "Provider ID or name to filter by. Try your own provider first.", }), + type: Schema.optionalKey(Schema.Literals(["chat", "image", "decision"])).annotate({ + description: 'Kind of model to list: "chat" for language models (the default), "image" for image generation, or "decision".', + }), all: Schema.optionalKey(Schema.Boolean).annotate({ description: "Include older versions of each model family. By default only the newest version is listed.", }), @@ -231,7 +234,7 @@ export const Plugin = { const terms = input.query?.toLowerCase().split(/\s+/).filter(Boolean) ?? [] const names = new Map((yield* ctx.provider.list()).data.map((provider) => [provider.id, provider.name])) const provider = input.provider?.toLowerCase() - const matching = (yield* ctx.model.list()).data + const matching = (yield* ctx.model.list({ types: [input.type ?? "chat"] })).data .filter( (model) => provider === undefined || diff --git a/packages/core/test/plugin/fixture.ts b/packages/core/test/plugin/fixture.ts index fca86018b82f..545f0126c99e 100644 --- a/packages/core/test/plugin/fixture.ts +++ b/packages/core/test/plugin/fixture.ts @@ -4,6 +4,7 @@ import { Command } from "@opencode/core/command" import { Config } from "@opencode/core/config" import { Credential } from "@opencode/core/credential" import { Database } from "@opencode/core/database/database" +import { Environment } from "@opencode/core/environment/index" import { FileAccess } from "@opencode/core/file-access" import { LayerNodePlatform } from "@opencode/util/effect/app-node-platform" import { AppProcess } from "@opencode/util/process" @@ -11,6 +12,7 @@ import { LayerNode } from "@opencode/util/effect/layer-node" import { Bus } from "@opencode/core/bus" import { FileSystem } from "@opencode/core/filesystem" import { FSUtil } from "@opencode/util/fs-util" +import { Global } from "@opencode/util/global" import { Form } from "@opencode/core/form" import { Generate } from "@opencode/core/generate" import { Integration } from "@opencode/core/integration" @@ -78,7 +80,7 @@ const permissionLayer = Layer.succeed( }), ) -export const makePluginTestLayer = (permission = permissionLayer) => +export const makePluginTestLayer = (permission = permissionLayer, generate = generateLayer) => AppNodeBuilder.build( LayerNode.group([ AppProcess.node, @@ -89,7 +91,9 @@ export const makePluginTestLayer = (permission = permissionLayer) => Credential.node, Config.node, Database.node, + Environment.node, FileAccess.node, + Global.node, Bus.node, Form.node, Generate.node, @@ -125,7 +129,7 @@ export const makePluginTestLayer = (permission = permissionLayer) => Npm.node.replace(npmLayer), Config.node.replace(configLayer), Mcp.node.replace(emptyMcpLayer), - Generate.node.replace(generateLayer), + Generate.node.replace(generate), Permission.node.replace(permission), ], ).pipe(Layer.provideMerge(configLayer)) diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts new file mode 100644 index 000000000000..d2cd9e771169 --- /dev/null +++ b/packages/core/test/tool-image-generate.test.ts @@ -0,0 +1,280 @@ +import { describe, expect } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Generate } from "@opencode/core/generate" +import { Location } from "@opencode/core/location" +import { Model } from "@opencode/core/model" +import type { Permission } from "@opencode/core/permission" +import { Plugin } from "@opencode/core/plugin" +import { PluginHost } from "@opencode/core/plugin/host" +import { Provider } from "@opencode/core/provider" +import { Session } from "@opencode/core/session" +import { Tool } from "@opencode/core/tool" +import { ImageGenerateTool } from "@opencode/core/tool/plugin/image-generate" +import { OpenCodeTools } from "@opencode/core/tool/plugin/opencode" +import type { ImageRequest, ImageResult } from "@opencode/schema/generate" +import { Global } from "@opencode/util/global" +import { Effect, Layer, Schedule } from "effect" +import { testEffect } from "./lib/effect" +import { permissionLayer } from "./lib/permission" +import { executeTool, toolDefinitions, toolIdentity } from "./lib/tool" +import { makePluginTestLayer } from "./plugin/fixture" + +const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +// 5x3, with EXIF and Photoshop segments ahead of the frame header. +const JPEG = + "/9j/4AAQSkZJRgABAQAASABIAAD/4QBMRXhpZgAATU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAABaADAAQAAAABAAAAAwAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgAAwAFAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAgICAgICAwICAwUDAwMFBgUFBQUGCAYGBgYGCAoICAgICAgKCgoKCgoKCgwMDAwMDA4ODg4ODw8PDw8PDw8PD//bAEMBAgICBAQEBwQEBxALCQsQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEP/dAAQAAf/aAAwDAQACEQMRAD8A7iiiiv8AQA+XP//Z" + +const fixture = { + requests: [] as ImageRequest[], + assertions: [] as Permission.AssertInput[], + result: { images: [{ type: "image", data: PNG, mime: "image/png" }] } as ImageResult, +} + +const it = testEffect( + makePluginTestLayer( + permissionLayer({ + assert: (input) => Effect.sync(() => void fixture.assertions.push(input)), + }), + Layer.succeed( + Generate.Service, + Generate.Service.of({ + text: () => Effect.succeed(""), + image: (input) => Effect.sync(() => fixture.requests.push(input)).pipe(Effect.as(fixture.result)), + decisions: () => Effect.die("unused"), + }), + ), + ), +) + +const imageModel = (providerID: string, id: string, update: (model: Model.MutableInfo) => void = () => {}) => + Effect.gen(function* () { + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => + editor.models.update(Provider.ID.make(providerID), Model.ID.make(id), (model) => { + model.type = "image" + update(model) + }), + ) + }) + +const setup = (providerID = "test") => + Effect.gen(function* () { + fixture.requests.length = 0 + fixture.assertions.length = 0 + fixture.result = { images: [{ type: "image", data: PNG, mime: "image/png" }] } + const plugins = yield* Plugin.Service + const sessions = yield* Session.Service + const location = yield* Location.Service + const host = yield* PluginHost.make(plugins) + yield* ImageGenerateTool.Plugin.effect(host) + yield* OpenCodeTools.Plugin.effect(host) + const session = yield* sessions.create({ + location: Location.Ref.make({ directory: location.directory }), + model: Model.Ref.make({ providerID: Provider.ID.make(providerID), id: Model.ID.make("chat") }), + }) + const registry = yield* Tool.Service + const run = (input: Record) => + executeTool(registry, { + sessionID: session.id, + ...toolIdentity, + call: { type: "tool-call", id: `call-${JSON.stringify(input)}`, name: ImageGenerateTool.name, input }, + }) + return { session, run, directory: location.directory } + }) + +const text = (result: { readonly content?: ReadonlyArray }) => + result.content?.[0]?.type === "text" ? result.content[0].text : undefined +const failure = (result: { readonly error?: { readonly message: string } }) => result.error?.message + +describe("image_generate", () => { + it.live("is registered only while an image model is available", () => + Effect.gen(function* () { + yield* setup() + const registry = yield* Tool.Service + const registered = toolDefinitions(registry).pipe( + Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)), + ) + // The catalog change reaches the tool through model.updated, so wait for it instead of a fixed delay. + const until = (expected: boolean) => + registered.pipe( + Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }), + Effect.timeout("10 seconds"), + ) + + expect(yield* registered).toBe(false) + yield* imageModel("test", "painter") + expect(yield* until(true)).toBe(true) + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => editor.models.remove(Provider.ID.make("test"), Model.ID.make("painter"))) + expect(yield* until(false)).toBe(false) + }), + ) + + it.live("reads input images, asks permission, and saves the result without overwriting", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run, directory, session } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + const result = yield* run({ prompt: "make it blue", images: ["in.png"], aspect: "16:9", path: "out.png" }) + const file = path.join(directory, "out.png") + expect(result.output).toEqual({ path: file, model: "test/painter", width: 1, height: 1 }) + expect(result.content?.[0]).toEqual({ + type: "text", + text: `Saved 1x1 image to ${file} (test/painter).`, + }) + expect(result.content?.[1]).toMatchObject({ type: "file", mime: "image/png", name: file }) + expect((yield* Effect.promise(() => fs.readFile(file))).toString("base64")).toBe(PNG) + expect(fixture.requests).toEqual([ + { + model: Model.Ref.make({ providerID: Provider.ID.make("test"), id: Model.ID.make("painter") }), + prompt: "make it blue", + n: 1, + images: [{ data: PNG, mime: "image/png", name: "in.png" }], + aspect: "16:9", + }, + ]) + expect(fixture.assertions.map((item) => [item.action, item.resources])).toEqual([ + ["image_generate", ["test/painter"]], + ["edit", ["out.png"]], + ["read", ["in.png"]], + ]) + expect(fixture.assertions[0]?.sessionID).toBe(session.id) + + // An existing file is never replaced, and nothing is generated or charged for. + fixture.requests.length = 0 + expect(failure(yield* run({ prompt: "again", path: "out.png" }))).toBe( + `${file} already exists. Choose a new path.`, + ) + expect(fixture.requests).toEqual([]) + }), + ) + + it.live("saves to the session temp folder by default and keeps the real type of a non-PNG result", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, session } = yield* setup() + const global = yield* Global.Service + fixture.result = { images: [{ type: "image", data: JPEG, mime: "image/jpeg" }] } + + const result = yield* run({ prompt: "a cat" }) + expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) + expect(path.extname(result.output.path)).toBe(".jpg") + expect(result.output).toMatchObject({ width: 5, height: 3 }) + expect(text(result)).toContain("The model returned image/jpeg, so it was saved with a .jpg extension.") + expect(yield* Effect.promise(() => Bun.file(result.output.path).exists())).toBe(true) + }), + ) + + it.live("validates an explicit model and its support for input images", () => + Effect.gen(function* () { + yield* imageModel("test", "text-only", (model) => { + model.capabilities.input = ["text"] + }) + yield* imageModel("other", "editor", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + expect(failure(yield* run({ prompt: "x", model: "test/missing" }))).toBe( + 'Model "test/missing" is not an available image model. Use the models tool with type: "image" to see what is available.', + ) + expect(failure(yield* run({ prompt: "x", model: "test/text-only", images: ["in.png"] }))).toBe( + 'Model "test/text-only" does not accept input images. Image models that do: other/editor.', + ) + // Without an explicit model, input images steer the default to a model that accepts them. + expect((yield* run({ prompt: "x", images: ["in.png"] })).output).toMatchObject({ model: "other/editor" }) + expect((yield* run({ prompt: "x", model: "test/text-only" })).output).toMatchObject({ model: "test/text-only" }) + expect(fixture.requests).toHaveLength(2) + }), + ) + + it.live("rejects input files that are not images", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "notes.txt"), "hello")) + expect(failure(yield* run({ prompt: "x", images: ["notes.txt"] }))).toBe( + "notes.txt is not a PNG, JPEG, WebP or GIF image.", + ) + expect(failure(yield* run({ prompt: "x", images: ["missing.png"] }))).toBe("File not found: missing.png") + expect(fixture.requests).toEqual([]) + }), + ) + + it.effect("lists image models through the models tool type filter", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => + editor.models.update(Provider.ID.make("test"), Model.ID.make("chat"), () => {}), + ) + const { session } = yield* setup() + const registry = yield* Tool.Service + const models = (input: Record) => + executeTool(registry, { + sessionID: session.id, + ...toolIdentity, + call: { + type: "tool-call", + id: `models-${JSON.stringify(input)}`, + name: "execute", + input: { code: `return await tools.opencode.models(${JSON.stringify(input)})` }, + }, + }).pipe( + Effect.map((result) => + (JSON.parse(text(result) ?? "") as { providers: { models: { id: string }[] }[] }).providers.flatMap( + (provider) => provider.models.map((model) => model.id), + ), + ), + ) + expect(yield* models({ type: "image" })).toEqual(["test/painter"]) + expect(yield* models({})).not.toContain("test/painter") + expect(yield* models({})).toContain("test/chat") + }), + ) +}) + +describe("defaultModel", () => { + const info = (providerID: string, id: string, family?: string, input = ["text", "image"]) => ({ + ...Model.Info.default(Provider.ID.make(providerID), Model.ID.make(id)), + type: "image", + ...(family ? { family: Model.Family.make(family) } : {}), + capabilities: { tools: false, input, output: ["image"] }, + }) + const pick = (models: ReturnType[], providerID?: string, images = false) => { + const model = ImageGenerateTool.defaultModel(models, { providerID, images }) + return model && `${model.providerID}/${model.id}` + } + + it.effect("prefers the session's provider, then known families, then the newest", () => + Effect.sync(() => { + const newest = info("fresh", "newest") + const flux = info("bfl", "flux-2", "flux") + const gemini = info("google", "nano-banana", "gemini-flash") + const gpt = info("openai", "gpt-image-2", "gpt-image") + const own = info("mine", "own-image") + const all = [newest, flux, gemini, gpt, own] + + expect(pick(all, "mine")).toBe("mine/own-image") + expect(pick(all, "elsewhere")).toBe("openai/gpt-image-2") + expect(pick([newest, flux, gemini])).toBe("google/nano-banana") + expect(pick([newest, info("x", "unknown", "mystery")])).toBe("fresh/newest") + expect(pick([])).toBeUndefined() + }), + ) + + it.effect("only considers models that accept input images when images are passed", () => + Effect.sync(() => { + const textOnly = info("mine", "text-only", "gpt-image", ["text"]) + const editor = info("other", "editor") + expect(pick([textOnly, editor], "mine", true)).toBe("other/editor") + expect(pick([textOnly], "mine", true)).toBeUndefined() + expect(pick([textOnly, editor], "mine", false)).toBe("mine/text-only") + }), + ) +}) From 92a9328dcda9b594ac22ec7f04776301c074a37e Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 10:41:20 -0500 Subject: [PATCH 02/17] fix(core): keep explicit image_generate paths exact --- packages/core/src/tool/plugin/image-generate.ts | 13 +++++-------- packages/core/test/tool-image-generate.test.ts | 11 ++++++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index bb17d668b734..62278bed533c 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -255,18 +255,15 @@ export const Plugin = { if (!image) return yield* new ToolFailure({ message: `${ref} returned no image.` }) const bytes = Buffer.from(image.data, "base64") const mime = Mime.detect(bytes) - // Providers may return JPEG or WebP; core has no encoder, so the file keeps its real type. - const extension = EXTENSIONS[mime] ?? path.extname(requested) - const file = - path.extname(requested).toLowerCase() === extension - ? requested - : path.join(path.dirname(requested), path.basename(requested, path.extname(requested)) + extension) + // Providers may return JPEG or WebP and core has no encoder. A default path takes the real extension; + // an explicit path is kept as given, because only that exact path was approved and checked. + const extension = EXTENSIONS[mime] ?? ".png" + const file = target || extension === ".png" ? requested : requested.slice(0, -".png".length) + extension if (yield* exists(file)) return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` }) yield* environment.files.write(file, bytes) const size = dimensions(bytes, mime) - const note = - file === requested ? "" : ` The model returned ${mime}, so it was saved with a ${extension} extension.` + const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` return { output: { path: file, model: ref, ...size }, content: [ diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index d2cd9e771169..61b4977c8dd8 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -153,10 +153,10 @@ describe("image_generate", () => { }), ) - it.live("saves to the session temp folder by default and keeps the real type of a non-PNG result", () => + it.live("saves to the session temp folder by default and handles a non-PNG result", () => Effect.gen(function* () { yield* imageModel("test", "painter") - const { run, session } = yield* setup() + const { run, session, directory } = yield* setup() const global = yield* Global.Service fixture.result = { images: [{ type: "image", data: JPEG, mime: "image/jpeg" }] } @@ -164,8 +164,13 @@ describe("image_generate", () => { expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) expect(path.extname(result.output.path)).toBe(".jpg") expect(result.output).toMatchObject({ width: 5, height: 3 }) - expect(text(result)).toContain("The model returned image/jpeg, so it was saved with a .jpg extension.") + expect(text(result)).toContain("The model returned image/jpeg, so the file is in that format.") expect(yield* Effect.promise(() => Bun.file(result.output.path).exists())).toBe(true) + + // An explicit path is kept exactly, since only that path was approved and checked before paying. + const explicit = yield* run({ prompt: "a cat", path: "cat.png" }) + expect(explicit.output.path).toBe(path.join(directory, "cat.png")) + expect(text(explicit)).toContain("The model returned image/jpeg, so the file is in that format.") }), ) From a3fdab75767a7684ad8927d44611cb6f90411bc6 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 11:29:35 -0500 Subject: [PATCH 03/17] refactor(core): drop aspect and resolution from image_generate --- packages/core/src/tool/plugin/image-generate.ts | 14 ++------------ packages/core/test/tool-image-generate.test.ts | 3 +-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 62278bed533c..bfc41e40db47 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -31,7 +31,7 @@ const EXTENSIONS: Record = { const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] export const description = - 'Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: "image". The result is saved as a PNG and shown to you; mention its path to the user. Each call costs money, so don\'t regenerate unless asked or the result is clearly wrong.' + 'Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: "image". The result is saved as an image file and shown to you; mention its path to the user. Each call costs money, so don\'t regenerate unless asked or the result is clearly wrong.' export const Input = Schema.Struct({ prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ @@ -44,21 +44,13 @@ export const Input = Schema.Struct({ description: "Images to edit or use as references, in order. Local file paths (relative to the session directory or absolute) or https URLs. Omit to create a new image.", }), - aspect: Schema.optionalKey(Schema.Literals(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"])).annotate({ - description: - "Shape of the image. 1:1 for icons and avatars, 16:9 for banners and screenshots, 9:16 for phone screens, 3:2 or 2:3 for photos. Defaults to the model's choice.", - }), - resolution: Schema.optionalKey(Schema.Literals(["1K", "2K"])).annotate({ - description: - "Size tier. 1K is about 1024x1024 pixels. Use 2K only when small text or fine detail matters; it is slower and costs more. Defaults to the model's choice.", - }), model: Schema.optionalKey(Schema.String).annotate({ description: 'NEVER set this unless the user asks for a particular image model. Written as "providerID/modelID". Do not guess the ID: look it up with the models tool using type: "image".', }), path: Schema.optionalKey(Schema.String).annotate({ description: - "Where to save the PNG, relative to the session directory or absolute. Never overwrites. Defaults to a new file in the session's temp folder.", + "Where to save the image, relative to the session directory or absolute. Never overwrites. Defaults to a new file in the session's temp folder.", }), }) @@ -238,8 +230,6 @@ export const Plugin = { prompt: input.prompt, n: 1, ...(images ? { images } : {}), - ...(input.aspect ? { aspect: input.aspect } : {}), - ...(input.resolution ? { resolution: input.resolution } : {}), }) .pipe( Effect.mapError( diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 61b4977c8dd8..7e00b34d21ad 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -119,7 +119,7 @@ describe("image_generate", () => { const { run, directory, session } = yield* setup() yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) - const result = yield* run({ prompt: "make it blue", images: ["in.png"], aspect: "16:9", path: "out.png" }) + const result = yield* run({ prompt: "make it blue", images: ["in.png"], path: "out.png" }) const file = path.join(directory, "out.png") expect(result.output).toEqual({ path: file, model: "test/painter", width: 1, height: 1 }) expect(result.content?.[0]).toEqual({ @@ -134,7 +134,6 @@ describe("image_generate", () => { prompt: "make it blue", n: 1, images: [{ data: PNG, mime: "image/png", name: "in.png" }], - aspect: "16:9", }, ]) expect(fixture.assertions.map((item) => [item.action, item.resources])).toEqual([ From d77d8ea6c4cde5faa3fac0ed1a2cda736ca1ec7c Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 13:23:10 -0500 Subject: [PATCH 04/17] refactor(core): drop image dimensions from image_generate --- .../core/src/tool/plugin/image-generate.ts | 38 +------------------ .../core/test/tool-image-generate.test.ts | 6 +-- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index bfc41e40db47..19a7e95d750f 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -57,8 +57,6 @@ export const Input = Schema.Struct({ export const Output = Schema.Struct({ path: Schema.String, model: Schema.String.annotate({ description: "providerID/modelID" }), - width: Schema.optionalKey(Schema.Int), - height: Schema.optionalKey(Schema.Int), }) export const Plugin = { @@ -252,14 +250,13 @@ export const Plugin = { if (yield* exists(file)) return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` }) yield* environment.files.write(file, bytes) - const size = dimensions(bytes, mime) const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` return { - output: { path: file, model: ref, ...size }, + output: { path: file, model: ref }, content: [ { type: "text" as const, - text: `Saved ${size ? `${size.width}x${size.height} ` : ""}image to ${file} (${ref}).${note}`, + text: `Saved image to ${file} (${ref}).${note}`, }, { type: "file" as const, uri: `data:${mime};base64,${image.data}`, mime, name: file }, ], @@ -290,34 +287,3 @@ export function defaultModel( function acceptsImages(model: Model.Info) { return model.capabilities.input.some((item) => item.startsWith("image")) } - -/** Reads pixel dimensions from the image header. */ -export function dimensions(bytes: Uint8Array, mime: string) { - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - if (mime === "image/png" && bytes.length >= 24) return { width: view.getUint32(16), height: view.getUint32(20) } - if (mime === "image/gif" && bytes.length >= 10) - return { width: view.getUint16(6, true), height: view.getUint16(8, true) } - if (mime === "image/webp" && bytes.length >= 31) { - const chunk = String.fromCharCode(...bytes.subarray(12, 16)) - if (chunk === "VP8 ") return { width: view.getUint16(26, true) & 0x3fff, height: view.getUint16(28, true) & 0x3fff } - if (chunk === "VP8L") { - const bits = view.getUint32(21, true) - return { width: (bits & 0x3fff) + 1, height: ((bits >> 14) & 0x3fff) + 1 } - } - if (chunk === "VP8X") - return { - width: (view.getUint32(24, true) & 0xffffff) + 1, - height: (view.getUint32(27, true) & 0xffffff) + 1, - } - } - if (mime === "image/jpeg") { - let offset = 2 - while (offset + 9 < bytes.length && bytes[offset] === 0xff) { - const marker = bytes[offset + 1]! - if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) - return { width: view.getUint16(offset + 7), height: view.getUint16(offset + 5) } - offset += 2 + view.getUint16(offset + 2) - } - } - return undefined -} diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 7e00b34d21ad..5a7efcd60055 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -21,7 +21,6 @@ import { executeTool, toolDefinitions, toolIdentity } from "./lib/tool" import { makePluginTestLayer } from "./plugin/fixture" const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" -// 5x3, with EXIF and Photoshop segments ahead of the frame header. const JPEG = "/9j/4AAQSkZJRgABAQAASABIAAD/4QBMRXhpZgAATU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAABaADAAQAAAABAAAAAwAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgAAwAFAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAgICAgICAwICAwUDAwMFBgUFBQUGCAYGBgYGCAoICAgICAgKCgoKCgoKCgwMDAwMDA4ODg4ODw8PDw8PDw8PD//bAEMBAgICBAQEBwQEBxALCQsQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEP/dAAQAAf/aAAwDAQACEQMRAD8A7iiiiv8AQA+XP//Z" @@ -121,10 +120,10 @@ describe("image_generate", () => { const result = yield* run({ prompt: "make it blue", images: ["in.png"], path: "out.png" }) const file = path.join(directory, "out.png") - expect(result.output).toEqual({ path: file, model: "test/painter", width: 1, height: 1 }) + expect(result.output).toEqual({ path: file, model: "test/painter" }) expect(result.content?.[0]).toEqual({ type: "text", - text: `Saved 1x1 image to ${file} (test/painter).`, + text: `Saved image to ${file} (test/painter).`, }) expect(result.content?.[1]).toMatchObject({ type: "file", mime: "image/png", name: file }) expect((yield* Effect.promise(() => fs.readFile(file))).toString("base64")).toBe(PNG) @@ -162,7 +161,6 @@ describe("image_generate", () => { const result = yield* run({ prompt: "a cat" }) expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) expect(path.extname(result.output.path)).toBe(".jpg") - expect(result.output).toMatchObject({ width: 5, height: 3 }) expect(text(result)).toContain("The model returned image/jpeg, so the file is in that format.") expect(yield* Effect.promise(() => Bun.file(result.output.path).exists())).toBe(true) From 5f8cc256a1ffef0ff5e1d80355492724eefad129 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 16:01:10 -0500 Subject: [PATCH 05/17] feat(core): add experimental.image_generation config --- .../client/src/promise/generated/types.ts | 1 + packages/core/src/config/normalize.ts | 9 ++ .../core/src/tool/plugin/image-generate.ts | 59 +++++++++---- .../core/test/config/normalization.test.ts | 18 ++++ .../core/test/tool-image-generate.test.ts | 85 ++++++++++++++++--- packages/protocol/openapi.json | 41 +++++++++ packages/schema/src/config/experimental.ts | 10 +++ 7 files changed, 197 insertions(+), 26 deletions(-) diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 738fb50180fa..32c11e903114 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -2426,6 +2426,7 @@ export type ConfigEntry = todo?: boolean background_compaction?: boolean media_blob?: boolean + image_generation?: false | { model?: string | { providerID: string; model: string; variant?: string } } schedules?: boolean subagent_depth?: number location_idle_timeout?: string | false diff --git a/packages/core/src/config/normalize.ts b/packages/core/src/config/normalize.ts index 6e0e1dd588bb..7163888ce85b 100644 --- a/packages/core/src/config/normalize.ts +++ b/packages/core/src/config/normalize.ts @@ -495,6 +495,15 @@ function normalizeExperimental( ) if (value !== undefined) result.media_blob = value } + if (own(experimental, "image_generation")) { + const value = decodeEncoded( + ConfigExperimental.Info.fields.image_generation, + experimental.image_generation, + ["experimental", "image_generation"], + diagnostics, + ) + if (value !== undefined) result.image_generation = value + } if (own(experimental, "schedules")) { const value = decodeEncoded( ConfigExperimental.Info.fields.schedules, diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 19a7e95d750f..4b9a3a6f28c0 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -3,11 +3,13 @@ export * as ImageGenerateTool from "./image-generate.js" import path from "path" import { ToolFailure } from "@opencode/ai/effect" import type { Context } from "@opencode/plugin/effect/plugin" +import type { Document, Entry } from "@opencode/schema/config" import { Identifier } from "@opencode/schema/identifier" import type { Session } from "@opencode/schema/session" import { Global } from "@opencode/util/global" import { Duration, Effect, Schema, Stream } from "effect" import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http" +import { Config } from "../../config.js" import { Environment } from "../../environment/index.js" import { FileAccess } from "../../file-access.js" import { Generate } from "../../generate.js" @@ -67,6 +69,7 @@ export const Plugin = { const generate = yield* Generate.Service const global = yield* Global.Service const http = yield* HttpClient.HttpClient + const config = yield* Config.Service const models = yield* Model.Service const permission = yield* Permission.Service @@ -77,12 +80,25 @@ export const Plugin = { ) { const available = yield* models.available(["image"]) if (!input) { + // A configured default is a preference: when it can't serve this call, pick automatically and say so. + const settings = settingsFrom(yield* config.entries()) + const configured = settings ? settings.model : undefined + const preferred = configured + ? available.find((model) => model.providerID === configured.providerID && model.id === configured.model) + : undefined + if (preferred && (!images || acceptsImages(preferred))) return { model: preferred } const providerID = (yield* ctx.session.get({ sessionID })).model?.providerID const model = defaultModel(available, { providerID, images }) - if (model) return model - return yield* new ToolFailure({ - message: images ? "None of the available image models accept input images." : "No image model is available.", - }) + if (!model) + return yield* new ToolFailure({ + message: images + ? "None of the available image models accept input images." + : "No image model is available.", + }) + if (!configured) return { model } + const ref = `${configured.providerID}/${configured.model}` + const reason = preferred ? "doesn't accept input images" : "isn't available" + return { model, note: ` ${ref} ${reason}; used ${model.providerID}/${model.id}.` } } const ref = yield* Effect.try({ try: () => Model.Ref.parse(input), @@ -102,7 +118,7 @@ export const Plugin = { : `Model "${ref.providerID}/${ref.id}" does not accept input images. Image models that do: ${capable.map((model) => `${model.providerID}/${model.id}`).join(", ")}.`, }) } - return model + return { model } }) const readImage = Effect.fn("ImageGenerateTool.readImage")(function* ( @@ -161,21 +177,24 @@ export const Plugin = { Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), ) - // Registered only while an image model is available, and re-evaluated whenever the model catalog changes. - // Subscribing before the first read keeps a change in between from being missed. + // Registered only while an image model is available and config doesn't turn the tool off, re-evaluated + // whenever either changes. Subscribing before the first read keeps a change in between from being missed. + const wanted = Effect.all([config.entries(), models.available(["image"])]).pipe( + Effect.map(([entries, available]) => settingsFrom(entries) !== false && available.length > 0), + ) let enabled = false - const refresh = models.available(["image"]).pipe( - Effect.flatMap((available) => { - if (available.length > 0 === enabled) return Effect.void - enabled = available.length > 0 + const refresh = wanted.pipe( + Effect.flatMap((next) => { + if (next === enabled) return Effect.void + enabled = next return ctx.tool.reload() }), ) - yield* ctx.event.subscribe({ types: ["model.updated"] }).pipe( + yield* ctx.event.subscribe({ types: ["model.updated", "config.updated"] }).pipe( Stream.runForEach(() => refresh), Effect.forkScoped({ startImmediately: true }), ) - enabled = (yield* models.available(["image"])).length > 0 + enabled = yield* wanted yield* ctx.tool .transform((editor) => { @@ -189,7 +208,8 @@ export const Plugin = { execute: (input, context) => Effect.gen(function* () { const source = { type: "tool" as const, messageID: context.messageID, id: context.id } - const model = yield* resolveModel(input.model, context.sessionID, input.images !== undefined) + const resolved = yield* resolveModel(input.model, context.sessionID, input.images !== undefined) + const model = resolved.model const ref = `${model.providerID}/${model.id}` const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined const requested = @@ -250,7 +270,9 @@ export const Plugin = { if (yield* exists(file)) return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` }) yield* environment.files.write(file, bytes) - const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` + const note = + (resolved.note ?? "") + + (mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.`) return { output: { path: file, model: ref }, content: [ @@ -284,6 +306,13 @@ export function defaultModel( return PREFERRED_FAMILIES.flatMap((family) => pool.find((model) => model.family === family) ?? [])[0] ?? pool[0] } +// Read per field so a later config that sets other experimental options doesn't hide this one. +function settingsFrom(entries: readonly Entry[]) { + return entries.findLast( + (entry): entry is Document => entry.type === "document" && entry.info.experimental?.image_generation !== undefined, + )?.info.experimental?.image_generation +} + function acceptsImages(model: Model.Info) { return model.capabilities.input.some((item) => item.startsWith("image")) } diff --git a/packages/core/test/config/normalization.test.ts b/packages/core/test/config/normalization.test.ts index 887aa2557993..423f80ab5ff0 100644 --- a/packages/core/test/config/normalization.test.ts +++ b/packages/core/test/config/normalization.test.ts @@ -545,6 +545,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + image_generation: { model: "openai/gpt-image-2" }, subagent_depth: 0, location_idle_timeout: "2 hours", policies: [{ action: "provider.use", resource: "custom", effect: "allow" }], @@ -556,6 +557,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + image_generation: { model: { providerID: "openai", model: "gpt-image-2" } }, subagent_depth: 0, location_idle_timeout: "7200000 millis", }, @@ -569,6 +571,22 @@ describe("ConfigNormalize", () => { }) }) + test("normalizes the image generation setting", () => { + expect(normalized({ experimental: { image_generation: false } }).encoded.experimental).toEqual({ + image_generation: false, + }) + expect(normalized({ experimental: { image_generation: {} } }).encoded.experimental).toEqual({ + image_generation: {}, + }) + for (const value of [true, { model: "no-slash" }, { model: 1 }, "openai/gpt-image-2"]) { + const result = normalized({ experimental: { image_generation: value } }) + expect(result.encoded.experimental).toBeUndefined() + expect(result.diagnostics.map((item) => [item.kind, item.path])).toEqual([ + ["invalid", ["experimental", "image_generation"]], + ]) + } + }) + test("never emits experimental policies", () => { expect( normalized({ experimental: { policies: [{ action: "provider.use", resource: "*", effect: "deny" }] } }).encoded, diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 5a7efcd60055..bb48971a94b6 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -1,6 +1,8 @@ import { describe, expect } from "bun:test" import fs from "fs/promises" import path from "path" +import { Bus } from "@opencode/core/bus" +import { Config } from "@opencode/core/config" import { Generate } from "@opencode/core/generate" import { Location } from "@opencode/core/location" import { Model } from "@opencode/core/model" @@ -12,6 +14,8 @@ import { Session } from "@opencode/core/session" import { Tool } from "@opencode/core/tool" import { ImageGenerateTool } from "@opencode/core/tool/plugin/image-generate" import { OpenCodeTools } from "@opencode/core/tool/plugin/opencode" +import { Document, Event, Info } from "@opencode/schema/config" +import { ConfigExperimental } from "@opencode/schema/config/experimental" import type { ImageRequest, ImageResult } from "@opencode/schema/generate" import { Global } from "@opencode/util/global" import { Effect, Layer, Schedule } from "effect" @@ -82,6 +86,33 @@ const setup = (providerID = "test") => return { session, run, directory: location.directory } }) +const configure = (image_generation: ConfigExperimental.Info["image_generation"]) => + Effect.gen(function* () { + const config = yield* Config.Test + yield* config.setEntries([ + new Document({ + type: "document", + info: new Info({ experimental: new ConfigExperimental.Info({ image_generation }) }), + }), + ]) + const bus = yield* Bus.Service + yield* bus.publish(Event.Updated, {}) + }) + +const configuredModel = (providerID: string, model: string) => ({ + model: { providerID: Provider.ID.make(providerID), model: Model.ID.make(model) }, +}) + +const registered = Tool.Service.use((registry) => toolDefinitions(registry)).pipe( + Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)), +) +// Catalog and config changes reach the tool through events, so wait for them instead of a fixed delay. +const until = (expected: boolean) => + registered.pipe( + Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }), + Effect.timeout("10 seconds"), + ) + const text = (result: { readonly content?: ReadonlyArray }) => result.content?.[0]?.type === "text" ? result.content[0].text : undefined const failure = (result: { readonly error?: { readonly message: string } }) => result.error?.message @@ -90,17 +121,6 @@ describe("image_generate", () => { it.live("is registered only while an image model is available", () => Effect.gen(function* () { yield* setup() - const registry = yield* Tool.Service - const registered = toolDefinitions(registry).pipe( - Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)), - ) - // The catalog change reaches the tool through model.updated, so wait for it instead of a fixed delay. - const until = (expected: boolean) => - registered.pipe( - Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }), - Effect.timeout("10 seconds"), - ) - expect(yield* registered).toBe(false) yield* imageModel("test", "painter") expect(yield* until(true)).toBe(true) @@ -110,6 +130,49 @@ describe("image_generate", () => { }), ) + it.live("is not registered while config turns it off", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + yield* configure(false) + yield* setup() + expect(yield* registered).toBe(false) + yield* configure({}) + expect(yield* until(true)).toBe(true) + yield* configure(false) + expect(yield* until(false)).toBe(false) + }), + ) + + it.live("uses the configured model unless the agent picks one, and falls back with a note", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + yield* imageModel("other", "text-only", (model) => { + model.capabilities.input = ["text"] + }) + yield* configure(configuredModel("other", "text-only")) + const { run, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + // The configured model beats the session's provider, and an explicit model beats both. + const configured = yield* run({ prompt: "x" }) + expect(configured.output).toMatchObject({ model: "other/text-only" }) + expect(text(configured)).not.toContain("used") + expect((yield* run({ prompt: "x", model: "test/painter" })).output).toMatchObject({ model: "test/painter" }) + + // A configured model that can't serve the call is skipped, and the result says why. + const edit = yield* run({ prompt: "x", images: ["in.png"] }) + expect(edit.output).toMatchObject({ model: "test/painter" }) + expect(text(edit)).toContain("other/text-only doesn't accept input images; used test/painter.") + + yield* configure(configuredModel("gone", "model")) + const missing = yield* run({ prompt: "x" }) + expect(missing.output).toMatchObject({ model: "test/painter" }) + expect(text(missing)).toContain("gone/model isn't available; used test/painter.") + }), + ) + it.live("reads input images, asks permission, and saves the result without overwriting", () => Effect.gen(function* () { yield* imageModel("test", "painter", (model) => { diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index cdd955603890..af6d5b5c4025 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -15790,6 +15790,47 @@ "media_blob": { "type": "boolean" }, + "image_generation": { + "anyOf": [ + { + "type": "boolean", + "enum": [false] + }, + { + "type": "object", + "properties": { + "model": { + "anyOf": [ + { + "type": "string", + "pattern": "^[^/#]+\\/[^#]+(?:#[^#]+)?$" + }, + { + "type": "object", + "properties": { + "providerID": { + "type": "string", + "pattern": "^[^/#]+$" + }, + "model": { + "type": "string", + "pattern": "^[^#]+$" + }, + "variant": { + "type": "string", + "pattern": "^[^#]+$" + } + }, + "required": ["providerID", "model"], + "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + } + ] + }, "schedules": { "type": "boolean" }, diff --git a/packages/schema/src/config/experimental.ts b/packages/schema/src/config/experimental.ts index 2541219fa5e9..f9f2d181dd15 100644 --- a/packages/schema/src/config/experimental.ts +++ b/packages/schema/src/config/experimental.ts @@ -2,6 +2,7 @@ export * as ConfigExperimental from "./experimental.js" import { Schema } from "effect" import { NonNegativeInt, optional } from "../schema.js" +import { ConfigModel } from "./model.js" import { ConfigPolicy } from "./policy.js" export class Info extends Schema.Class("ConfigExperimental.Info")({ @@ -19,6 +20,15 @@ export class Info extends Schema.Class("ConfigExperimental.Info")({ description: "Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.", }), + image_generation: Schema.Union([ + Schema.Literal(false), + Schema.Struct({ model: ConfigModel.Selection.pipe(optional) }), + ]) + .pipe(optional) + .annotate({ + description: + "Image generation tool settings. `model` sets the default image model (providerID/modelID) used when the agent doesn't choose one; set to false to disable the image_generate tool.", + }), schedules: Schema.Boolean.pipe(optional).annotate({ description: "Enable experimental scheduled tasks that wake a session with a prompt. Defaults to false.", }), From 0c640617b221d556d89e6bc2fb46c75849a8a4c2 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 16:31:04 -0500 Subject: [PATCH 06/17] feat(core): run image_generate in the background --- packages/core/src/job.ts | 6 + packages/core/src/location-activity.ts | 2 +- .../core/src/session/execution/restart.ts | 39 ++++- .../core/src/tool/plugin/image-generate.ts | 121 ++++++++++------ packages/core/test/plugin/fixture.ts | 4 + packages/core/test/session-execution.test.ts | 38 +++++ .../core/test/tool-image-generate.test.ts | 135 +++++++++++++++--- 7 files changed, 277 insertions(+), 68 deletions(-) diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index 7afc0f17f229..4eb0ab39d498 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -23,6 +23,12 @@ const Background = Schema.Struct({ agent: Schema.String, description: Schema.String, }), + Schema.Struct({ + kind: Schema.Literal("image"), + sessionID: SessionSchema.ID, + path: Schema.String, + model: Schema.String, + }), ]), status: Schema.Literals(["running", "completed", "error", "cancelled"]), output: Schema.optionalKey(Schema.String), diff --git a/packages/core/src/location-activity.ts b/packages/core/src/location-activity.ts index 7aa3ed68dc9a..c2416e966dd3 100644 --- a/packages/core/src/location-activity.ts +++ b/packages/core/src/location-activity.ts @@ -77,7 +77,7 @@ export function layer(options: { readonly timeToLive?: Duration.Input; readonly const owners = [ ...(yield* execution.active), ...(yield* jobs.runningBackground).flatMap((recovery) => - recovery.kind === "shell" ? [recovery.sessionID] : [recovery.childSessionID, recovery.parentSessionID], + recovery.kind === "subagent" ? [recovery.childSessionID, recovery.parentSessionID] : [recovery.sessionID], ), ] const occupied = new Set( diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index dde339301837..bf4aa65b35a8 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -135,6 +135,39 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + const recoverImage = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + ) { + const text = + background.status === "running" + ? `Image request for ${recovery.path} was cancelled because the server restarted.` + : background.status === "completed" + ? (background.output ?? `Image request finished. Saved image: ${recovery.path} (${recovery.model}).`) + : background.status === "error" + ? (background.error ?? `Image request failed for ${recovery.path}.`) + : `Image request for ${recovery.path} was cancelled.` + yield* sessions + .synthetic({ + id: background.notificationID, + sessionID: recovery.sessionID, + description: "Image generation", + text, + metadata: { + source: "image_generate", + jobID: background.id, + state: background.status === "running" ? "cancelled" : background.status, + path: recovery.path, + }, + resume: false, + }) + .pipe( + Effect.catchTag("Session.NotFoundError", () => Effect.void), + Effect.orDie, + ) + yield* jobs.completeBackground(background.notificationID) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -209,9 +242,9 @@ export const layer = (options?: Options) => Effect.fnUntraced(function* (background) { if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - yield* recovery.kind === "shell" - ? recoverShell(background, recovery) - : recoverSubagent(background, recovery, suspended) + if (recovery.kind === "shell") return yield* recoverShell(background, recovery) + if (recovery.kind === "image") return yield* recoverImage(background, recovery) + yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, ) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 4b9a3a6f28c0..7c53bc92bc2e 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -5,23 +5,25 @@ import { ToolFailure } from "@opencode/ai/effect" import type { Context } from "@opencode/plugin/effect/plugin" import type { Document, Entry } from "@opencode/schema/config" import { Identifier } from "@opencode/schema/identifier" -import type { Session } from "@opencode/schema/session" import { Global } from "@opencode/util/global" -import { Duration, Effect, Schema, Stream } from "effect" +import { Duration, Effect, Schema, Scope, Stream } from "effect" import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http" import { Config } from "../../config.js" import { Environment } from "../../environment/index.js" import { FileAccess } from "../../file-access.js" import { Generate } from "../../generate.js" +import { Job } from "../../job.js" import { Mime } from "../../mime.js" import { Model } from "../../model.js" import { Permission } from "../../permission.js" +import { Session } from "../../session.js" +import { SessionSchema } from "../../session/schema.js" import { collectBoundedResponseBody } from "../http-body.js" export const name = "image_generate" export const MAX_INPUT_BYTES = 20 * 1024 * 1024 const DOWNLOAD_TIMEOUT = Duration.seconds(30) -const GENERATE_TIMEOUT = Duration.minutes(5) +const GENERATE_TIMEOUT = Duration.minutes(30) const INPUT_TYPES = ["image/png", "image/jpeg", "image/webp", "image/gif"] const EXTENSIONS: Record = { "image/png": ".png", @@ -33,7 +35,7 @@ const EXTENSIONS: Record = { const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] export const description = - 'Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: "image". The result is saved as an image file and shown to you; mention its path to the user. Each call costs money, so don\'t regenerate unless asked or the result is clearly wrong.' + "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: \"image\". Runs in the background: the call returns right away and you are notified when the image is saved; then use the read tool on its path to view it, and mention the path to the user. Don't poll or wait for it. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." export const Input = Schema.Struct({ prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ @@ -70,12 +72,15 @@ export const Plugin = { const global = yield* Global.Service const http = yield* HttpClient.HttpClient const config = yield* Config.Service + const jobs = yield* Job.Service + const scope = yield* Scope.Scope + const sessions = yield* Session.Service const models = yield* Model.Service const permission = yield* Permission.Service const resolveModel = Effect.fn("ImageGenerateTool.resolveModel")(function* ( input: string | undefined, - sessionID: Session.ID, + sessionID: SessionSchema.ID, images: boolean, ) { const available = yield* models.available(["image"]) @@ -171,6 +176,27 @@ export const Plugin = { return (yield* environment.files.read(target.absolute)).bytes }) + const notifyWhenDone = Effect.fn("ImageGenerateTool.notifyWhenDone")( + function* (sessionID: SessionSchema.ID, id: string, file: string) { + const info = (yield* jobs.wait({ id })).info + if (!info || info.status === "running") return + yield* sessions.synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID, + description: "Image generation", + text: + info.status === "completed" + ? (info.output ?? "") + : info.status === "error" + ? (info.error ?? `Image request failed for ${file}.`) + : `Image request for ${file} was cancelled.`, + metadata: { source: name, jobID: id, state: info.status, path: file }, + }) + if (info.notificationID) yield* jobs.completeBackground(info.notificationID) + }, + Effect.forkIn(scope, { startImmediately: true }), + ) + const exists = (file: string) => environment.files.stat(file).pipe( Effect.as(true), @@ -242,47 +268,52 @@ export const Plugin = { ? yield* Effect.forEach(input.images, (image) => readImage(image, context)) : undefined - const result = yield* generate - .image({ - model: Model.Ref.make({ providerID: model.providerID, id: model.id }), - prompt: input.prompt, - n: 1, - ...(images ? { images } : {}), - }) - .pipe( - Effect.mapError( - (error) => new ToolFailure({ message: `Image generation failed: ${error.message}`, error }), - ), - Effect.timeoutOrElse({ - duration: GENERATE_TIMEOUT, - orElse: () => - Effect.fail(new ToolFailure({ message: "Image generation timed out after 5 minutes." })), - }), - ) - const image = result.images[0] - if (!image) return yield* new ToolFailure({ message: `${ref} returned no image.` }) - const bytes = Buffer.from(image.data, "base64") - const mime = Mime.detect(bytes) - // Providers may return JPEG or WebP and core has no encoder. A default path takes the real extension; - // an explicit path is kept as given, because only that exact path was approved and checked. - const extension = EXTENSIONS[mime] ?? ".png" - const file = target || extension === ".png" ? requested : requested.slice(0, -".png".length) + extension - if (yield* exists(file)) - return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` }) - yield* environment.files.write(file, bytes) - const note = - (resolved.note ?? "") + - (mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.`) + // Everything that needs the user or can fail fast happened above; the paid call runs as a background job. + const run = Effect.gen(function* () { + const result = yield* generate + .image({ + model: Model.Ref.make({ providerID: model.providerID, id: model.id }), + prompt: input.prompt, + n: 1, + ...(images ? { images } : {}), + }) + .pipe( + Effect.timeoutOrElse({ + duration: GENERATE_TIMEOUT, + orElse: () => Effect.fail(new Error("Image generation timed out after 30 minutes.")), + }), + ) + const image = result.images[0] + if (!image) return yield* Effect.fail(new Error(`${ref} returned no image.`)) + const bytes = Buffer.from(image.data, "base64") + const mime = Mime.detect(bytes) + // Providers may return JPEG or WebP and core has no encoder. A default path takes the real extension; + // an explicit path is kept as given, because only that exact path was approved and checked. + const extension = EXTENSIONS[mime] ?? ".png" + const file = target || extension === ".png" ? requested : requested.slice(0, -".png".length) + extension + if (yield* exists(file)) return yield* Effect.fail(new Error(`${file} already exists.`)) + yield* environment.files.write(file, bytes) + const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` + return `Image request finished. Saved image: ${file} (${ref}).${note}\nUse the read tool on this path to view the image.` + }).pipe( + // The job keeps this text, so a restart can still deliver the outcome. + Effect.mapError((error) => new Error(`Image request failed for ${requested}: ${error.message}`)), + ) + const id = `image_${Identifier.create(false)}` + const job = yield* jobs.start({ + id, + type: name, + title: input.prompt, + metadata: { sessionID: context.sessionID, path: requested, model: ref }, + recovery: { kind: "image", sessionID: context.sessionID, path: requested, model: ref }, + run, + }) + yield* jobs.background(job.id) + yield* notifyWhenDone(context.sessionID, job.id, requested) return { - output: { path: file, model: ref }, - content: [ - { - type: "text" as const, - text: `Saved image to ${file} (${ref}).${note}`, - }, - { type: "file" as const, uri: `data:${mime};base64,${image.data}`, mime, name: file }, - ], - metadata: { path: file, model: ref }, + output: { path: requested, model: ref }, + content: `Image request started in the background with ${ref}. Output path: ${requested}. You will be notified when it finishes or fails. Do not poll; use the read tool on the path after completion.${resolved.note ?? ""}`, + metadata: { path: requested, model: ref, jobID: job.id }, } }).pipe( Effect.mapError((error) => diff --git a/packages/core/test/plugin/fixture.ts b/packages/core/test/plugin/fixture.ts index 545f0126c99e..c8f3a06a727c 100644 --- a/packages/core/test/plugin/fixture.ts +++ b/packages/core/test/plugin/fixture.ts @@ -16,6 +16,7 @@ import { Global } from "@opencode/util/global" import { Form } from "@opencode/core/form" import { Generate } from "@opencode/core/generate" import { Integration } from "@opencode/core/integration" +import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Location } from "@opencode/core/location" import { Mcp } from "@opencode/core/mcp/index" @@ -43,6 +44,7 @@ import { WorktreeStrategies } from "@opencode/core/worktree/strategies" import { Effect, Layer } from "effect" import { tempLocationLayer } from "../fixture/location" import { emptyMcpLayer } from "../fixture/mcp" +import { offlineModels } from "../fixture/models" const npmLayer = Layer.succeed( Npm.Service, @@ -105,6 +107,7 @@ export const makePluginTestLayer = (permission = permissionLayer, generate = gen Model.node, Command.node, Integration.node, + Job.node, KV.node, Mcp.node, Session.node, @@ -131,6 +134,7 @@ export const makePluginTestLayer = (permission = permissionLayer, generate = gen Mcp.node.replace(emptyMcpLayer), Generate.node.replace(generate), Permission.node.replace(permission), + offlineModels, ], ).pipe(Layer.provideMerge(configLayer)) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index eaa135652993..4cd23d9af22a 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -540,6 +540,44 @@ describe("SessionRestart background recovery", () => { }), ) + it.effect("delivers an image outcome persisted before its notification, and cancels one still running", () => + Effect.gen(function* () { + const database = yield* Database.Service + const jobs = yield* Job.Service + const sessionID = Session.ID.make("ses_background_image") + yield* seedSessions(database, [sessionID]) + const recovery = (path: string) => ({ kind: "image" as const, sessionID, path, model: "test/painter" }) + yield* jobs.start({ + id: "image_done", + type: "image_generate", + recovery: recovery("/tmp/done.png"), + run: Effect.succeed("Image request finished. Saved image: /tmp/done.png (test/painter)."), + }) + yield* jobs.background("image_done") + yield* jobs.wait({ id: "image_done" }) + yield* jobs.start({ + id: "image_running", + type: "image_generate", + recovery: recovery("/tmp/running.png"), + run: Effect.never, + }) + yield* jobs.background("image_running") + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution(scope, () => Effect.void, undefined, restarted) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const inbox = yield* SessionInbox.list(database.db, sessionID) + expect(inbox.map((item) => (item.type === "synthetic" ? item.payload.text : undefined)).toSorted()).toEqual([ + "Image request finished. Saved image: /tmp/done.png (test/painter).", + "Image request for /tmp/running.png was cancelled because the server restarted.", + ]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + for (const delivered of [false, true]) { it.effect(`does not duplicate a shell notification already ${delivered ? "delivered" : "admitted"}`, () => Effect.gen(function* () { diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index bb48971a94b6..6d5ecabc1fff 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -4,13 +4,15 @@ import path from "path" import { Bus } from "@opencode/core/bus" import { Config } from "@opencode/core/config" import { Generate } from "@opencode/core/generate" +import { Job } from "@opencode/core/job" import { Location } from "@opencode/core/location" import { Model } from "@opencode/core/model" -import type { Permission } from "@opencode/core/permission" +import { Permission } from "@opencode/core/permission" import { Plugin } from "@opencode/core/plugin" import { PluginHost } from "@opencode/core/plugin/host" import { Provider } from "@opencode/core/provider" import { Session } from "@opencode/core/session" +import { SessionEvent } from "@opencode/core/session/event" import { Tool } from "@opencode/core/tool" import { ImageGenerateTool } from "@opencode/core/tool/plugin/image-generate" import { OpenCodeTools } from "@opencode/core/tool/plugin/opencode" @@ -18,7 +20,7 @@ import { Document, Event, Info } from "@opencode/schema/config" import { ConfigExperimental } from "@opencode/schema/config/experimental" import type { ImageRequest, ImageResult } from "@opencode/schema/generate" import { Global } from "@opencode/util/global" -import { Effect, Layer, Schedule } from "effect" +import { Effect, Layer, Schedule, Stream } from "effect" import { testEffect } from "./lib/effect" import { permissionLayer } from "./lib/permission" import { executeTool, toolDefinitions, toolIdentity } from "./lib/tool" @@ -32,18 +34,37 @@ const fixture = { requests: [] as ImageRequest[], assertions: [] as Permission.AssertInput[], result: { images: [{ type: "image", data: PNG, mime: "image/png" }] } as ImageResult, + // Holds generation until the test releases it, to observe a running job. + gate: Effect.void as Effect.Effect, + failure: undefined as Generate.Error | undefined, + deny: undefined as string | undefined, } const it = testEffect( makePluginTestLayer( permissionLayer({ - assert: (input) => Effect.sync(() => void fixture.assertions.push(input)), + assert: (input) => + Effect.sync(() => void fixture.assertions.push(input)).pipe( + Effect.andThen( + fixture.deny === input.action + ? Effect.fail( + new Permission.BlockedError({ rules: [], permission: input.action, resources: input.resources }), + ) + : Effect.void, + ), + ), }), Layer.succeed( Generate.Service, Generate.Service.of({ text: () => Effect.succeed(""), - image: (input) => Effect.sync(() => fixture.requests.push(input)).pipe(Effect.as(fixture.result)), + image: (input) => + Effect.sync(() => fixture.requests.push(input)).pipe( + Effect.andThen(Effect.suspend(() => fixture.gate)), + Effect.andThen( + Effect.suspend(() => (fixture.failure ? Effect.fail(fixture.failure) : Effect.succeed(fixture.result))), + ), + ), decisions: () => Effect.die("unused"), }), ), @@ -66,6 +87,9 @@ const setup = (providerID = "test") => fixture.requests.length = 0 fixture.assertions.length = 0 fixture.result = { images: [{ type: "image", data: PNG, mime: "image/png" }] } + fixture.gate = Effect.void + fixture.failure = undefined + fixture.deny = undefined const plugins = yield* Plugin.Service const sessions = yield* Session.Service const location = yield* Location.Service @@ -83,7 +107,25 @@ const setup = (providerID = "test") => ...toolIdentity, call: { type: "tool-call", id: `call-${JSON.stringify(input)}`, name: ImageGenerateTool.name, input }, }) - return { session, run, directory: location.directory } + // Completion notices arrive as synthetic messages posted by the background job. + const notices: { text: string; metadata?: Record }[] = [] + const bus = yield* Bus.Service + yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.runForEach((event) => + Effect.sync(() => { + if (event.data.sessionID === session.id && event.data.item.type === "synthetic") + notices.push(event.data.item.payload) + }), + ), + Effect.forkScoped({ startImmediately: true }), + ) + const notice = (result: { readonly metadata?: Record }) => + Effect.sync(() => notices.find((item) => item.metadata?.jobID === result.metadata?.jobID)).pipe( + Effect.repeat({ until: (item) => item !== undefined, schedule: Schedule.spaced("5 millis") }), + Effect.timeout("10 seconds"), + Effect.map((item) => item?.text), + ) + return { session, run, notice, directory: location.directory } }) const configure = (image_generation: ConfigExperimental.Info["image_generation"]) => @@ -178,17 +220,21 @@ describe("image_generate", () => { yield* imageModel("test", "painter", (model) => { model.capabilities.input = ["text", "image"] }) - const { run, directory, session } = yield* setup() + const { run, notice, directory, session } = yield* setup() yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) const result = yield* run({ prompt: "make it blue", images: ["in.png"], path: "out.png" }) const file = path.join(directory, "out.png") expect(result.output).toEqual({ path: file, model: "test/painter" }) - expect(result.content?.[0]).toEqual({ - type: "text", - text: `Saved image to ${file} (test/painter).`, - }) - expect(result.content?.[1]).toMatchObject({ type: "file", mime: "image/png", name: file }) + expect(result.content).toEqual([ + { + type: "text", + text: `Image request started in the background with test/painter. Output path: ${file}. You will be notified when it finishes or fails. Do not poll; use the read tool on the path after completion.`, + }, + ]) + expect(yield* notice(result)).toBe( + `Image request finished. Saved image: ${file} (test/painter).\nUse the read tool on this path to view the image.`, + ) expect((yield* Effect.promise(() => fs.readFile(file))).toString("base64")).toBe(PNG) expect(fixture.requests).toEqual([ { @@ -217,20 +263,67 @@ describe("image_generate", () => { it.live("saves to the session temp folder by default and handles a non-PNG result", () => Effect.gen(function* () { yield* imageModel("test", "painter") - const { run, session, directory } = yield* setup() + const { run, notice, session, directory } = yield* setup() const global = yield* Global.Service fixture.result = { images: [{ type: "image", data: JPEG, mime: "image/jpeg" }] } + // A default path takes the real extension, and the notice gives the final path. const result = yield* run({ prompt: "a cat" }) expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) - expect(path.extname(result.output.path)).toBe(".jpg") - expect(text(result)).toContain("The model returned image/jpeg, so the file is in that format.") - expect(yield* Effect.promise(() => Bun.file(result.output.path).exists())).toBe(true) + const saved = result.output.path.replace(/\.png$/, ".jpg") + expect(yield* notice(result)).toBe( + `Image request finished. Saved image: ${saved} (test/painter). The model returned image/jpeg, so the file is in that format.\nUse the read tool on this path to view the image.`, + ) + expect(yield* Effect.promise(() => Bun.file(saved).exists())).toBe(true) // An explicit path is kept exactly, since only that path was approved and checked before paying. const explicit = yield* run({ prompt: "a cat", path: "cat.png" }) expect(explicit.output.path).toBe(path.join(directory, "cat.png")) - expect(text(explicit)).toContain("The model returned image/jpeg, so the file is in that format.") + expect(yield* notice(explicit)).toContain(`Saved image: ${path.join(directory, "cat.png")} (test/painter).`) + }), + ) + + it.live("reports a failed generation in the completion notice", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, notice, directory } = yield* setup() + fixture.failure = new Generate.ProviderRequestError({ + providerID: "test", + reason: "content_filter", + message: "The prompt was rejected", + }) + const result = yield* run({ prompt: "x", path: "out.png" }) + expect(yield* notice(result)).toBe( + `Image request failed for ${path.join(directory, "out.png")}: The prompt was rejected`, + ) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) + }), + ) + + it.live("cancels a running generation through its job", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, notice, directory } = yield* setup() + fixture.gate = Effect.never + const result = yield* run({ prompt: "x", path: "out.png" }) + const jobs = yield* Job.Service + const id = String(result.metadata?.jobID) + expect((yield* jobs.get(id))?.status).toBe("running") + yield* jobs.cancel(id) + expect(yield* notice(result)).toBe(`Image request for ${path.join(directory, "out.png")} was cancelled.`) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) + }), + ) + + it.live("starts no job when a permission is denied", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run } = yield* setup() + fixture.deny = "edit" + const result = yield* run({ prompt: "x", path: "out.png" }) + expect(result.status).toBe("error") + expect(fixture.assertions.map((item) => item.action)).toEqual(["image_generate", "edit"]) + expect(fixture.requests).toEqual([]) }), ) @@ -242,7 +335,7 @@ describe("image_generate", () => { yield* imageModel("other", "editor", (model) => { model.capabilities.input = ["text", "image"] }) - const { run, directory } = yield* setup() + const { run, notice, directory } = yield* setup() yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) expect(failure(yield* run({ prompt: "x", model: "test/missing" }))).toBe( @@ -252,8 +345,12 @@ describe("image_generate", () => { 'Model "test/text-only" does not accept input images. Image models that do: other/editor.', ) // Without an explicit model, input images steer the default to a model that accepts them. - expect((yield* run({ prompt: "x", images: ["in.png"] })).output).toMatchObject({ model: "other/editor" }) - expect((yield* run({ prompt: "x", model: "test/text-only" })).output).toMatchObject({ model: "test/text-only" }) + const steered = yield* run({ prompt: "x", images: ["in.png"] }) + expect(steered.output).toMatchObject({ model: "other/editor" }) + const explicit = yield* run({ prompt: "x", model: "test/text-only" }) + expect(explicit.output).toMatchObject({ model: "test/text-only" }) + yield* notice(steered) + yield* notice(explicit) expect(fixture.requests).toHaveLength(2) }), ) From 71e8de1a3ef5dce46202ebdf17844b9ed7d9ca6c Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 16:52:19 -0500 Subject: [PATCH 07/17] feat(core): support model variants in image_generate --- .../core/src/tool/plugin/image-generate.ts | 40 ++++++++++++--- .../core/test/tool-image-generate.test.ts | 51 +++++++++++++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 7c53bc92bc2e..76885da2ce74 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -50,7 +50,7 @@ export const Input = Schema.Struct({ }), model: Schema.optionalKey(Schema.String).annotate({ description: - 'NEVER set this unless the user asks for a particular image model. Written as "providerID/modelID". Do not guess the ID: look it up with the models tool using type: "image".', + 'NEVER set this unless the user explicitly asks for a particular image model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look it up with the models tool using type: "image".', }), path: Schema.optionalKey(Schema.String).annotate({ description: @@ -91,7 +91,12 @@ export const Plugin = { const preferred = configured ? available.find((model) => model.providerID === configured.providerID && model.id === configured.model) : undefined - if (preferred && (!images || acceptsImages(preferred))) return { model: preferred } + const variantMissing = + configured?.variant !== undefined && + preferred !== undefined && + !preferred.variants.some((variant) => variant.id === configured.variant) + if (preferred && !variantMissing && (!images || acceptsImages(preferred))) + return { model: preferred, variant: configured?.variant } const providerID = (yield* ctx.session.get({ sessionID })).model?.providerID const model = defaultModel(available, { providerID, images }) if (!model) @@ -101,19 +106,33 @@ export const Plugin = { : "No image model is available.", }) if (!configured) return { model } - const ref = `${configured.providerID}/${configured.model}` - const reason = preferred ? "doesn't accept input images" : "isn't available" + const ref = `${configured.providerID}/${configured.model}${configured.variant ? `#${configured.variant}` : ""}` + const reason = !preferred + ? "isn't available" + : variantMissing + ? `has no variant "${configured.variant}"` + : "doesn't accept input images" return { model, note: ` ${ref} ${reason}; used ${model.providerID}/${model.id}.` } } const ref = yield* Effect.try({ try: () => Model.Ref.parse(input), - catch: () => new ToolFailure({ message: `Invalid model "${input}". Use "providerID/modelID".` }), + catch: () => + new ToolFailure({ + message: `Invalid model "${input}". Use "providerID/modelID" or "providerID/modelID#variant".`, + }), }) const model = available.find((model) => model.providerID === ref.providerID && model.id === ref.id) if (model === undefined) return yield* new ToolFailure({ message: `Model "${ref.providerID}/${ref.id}" is not an available image model. Use the models tool with type: "image" to see what is available.`, }) + if (ref.variant !== undefined && !model.variants.some((variant) => variant.id === ref.variant)) + return yield* new ToolFailure({ + message: + model.variants.length === 0 + ? `Model "${ref.providerID}/${ref.id}" has no variants. Omit the variant.` + : `Variant "${ref.variant}" is not available for "${ref.providerID}/${ref.id}". Available: ${model.variants.map((variant) => variant.id).join(", ")}.`, + }) if (images && !acceptsImages(model)) { const capable = available.filter(acceptsImages).slice(0, 3) return yield* new ToolFailure({ @@ -123,7 +142,7 @@ export const Plugin = { : `Model "${ref.providerID}/${ref.id}" does not accept input images. Image models that do: ${capable.map((model) => `${model.providerID}/${model.id}`).join(", ")}.`, }) } - return { model } + return { model, variant: ref.variant } }) const readImage = Effect.fn("ImageGenerateTool.readImage")(function* ( @@ -236,7 +255,8 @@ export const Plugin = { const source = { type: "tool" as const, messageID: context.messageID, id: context.id } const resolved = yield* resolveModel(input.model, context.sessionID, input.images !== undefined) const model = resolved.model - const ref = `${model.providerID}/${model.id}` + const variant = resolved.variant + const ref = `${model.providerID}/${model.id}${variant ? `#${variant}` : ""}` const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined const requested = target?.absolute ?? @@ -272,7 +292,11 @@ export const Plugin = { const run = Effect.gen(function* () { const result = yield* generate .image({ - model: Model.Ref.make({ providerID: model.providerID, id: model.id }), + model: Model.Ref.make({ + providerID: model.providerID, + id: model.id, + ...(variant ? { variant } : {}), + }), prompt: input.prompt, n: 1, ...(images ? { images } : {}), diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 6d5ecabc1fff..af13939a37c1 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -355,6 +355,57 @@ describe("image_generate", () => { }), ) + it.live("passes a model variant through, from the input or from config", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.variants = [{ id: Model.VariantID.make("hd"), settings: { quality: "high" } }] + }) + yield* imageModel("test", "plain") + const { run, notice } = yield* setup() + + const explicit = yield* run({ prompt: "x", model: "test/painter#hd" }) + expect(explicit.output).toMatchObject({ model: "test/painter#hd" }) + yield* notice(explicit) + expect(fixture.requests.at(-1)?.model).toEqual( + Model.Ref.make({ + providerID: Provider.ID.make("test"), + id: Model.ID.make("painter"), + variant: Model.VariantID.make("hd"), + }), + ) + expect(failure(yield* run({ prompt: "x", model: "test/painter#bogus" }))).toBe( + 'Variant "bogus" is not available for "test/painter". Available: hd.', + ) + expect(failure(yield* run({ prompt: "x", model: "test/plain#hd" }))).toBe( + 'Model "test/plain" has no variants. Omit the variant.', + ) + + yield* configure({ + model: { + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("hd"), + }, + }) + const configured = yield* run({ prompt: "x" }) + expect(configured.output).toMatchObject({ model: "test/painter#hd" }) + yield* notice(configured) + expect(fixture.requests.at(-1)?.model.variant).toBe(Model.VariantID.make("hd")) + + yield* configure({ + model: { + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("bogus"), + }, + }) + const fallback = yield* run({ prompt: "x" }) + expect(text(fallback)).toContain('test/painter#bogus has no variant "bogus"; used ') + yield* notice(fallback) + expect(fixture.requests.at(-1)?.model.variant).toBeUndefined() + }), + ) + it.live("rejects input files that are not images", () => Effect.gen(function* () { yield* imageModel("test", "painter") From 1247cb18a2897ad9db1f5dd5ec610f803f9742f1 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 17:11:26 -0500 Subject: [PATCH 08/17] feat(core): describe model types in the models tool --- packages/core/src/tool/plugin/opencode.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/core/src/tool/plugin/opencode.ts b/packages/core/src/tool/plugin/opencode.ts index 44649c37347b..5171ff778590 100644 --- a/packages/core/src/tool/plugin/opencode.ts +++ b/packages/core/src/tool/plugin/opencode.ts @@ -51,7 +51,12 @@ export const ModelsInput = Schema.Struct({ description: "Provider ID or name to filter by. Try your own provider first.", }), type: Schema.optionalKey(Schema.Literals(["chat", "image", "decision"])).annotate({ - description: 'Kind of model to list: "chat" for language models (the default), "image" for image generation, or "decision".', + description: [ + 'Which kind of model to list. Defaults to "chat".', + '"chat": a language model (LLM). Powers sessions, agents and subagents, and can call tools.', + '"image": generates or edits images from a prompt. It can\'t chat or call tools.', + '"decision": a fast classifier, not an LLM. Given some input and a set of questions, it returns structured answers (a label, a score, or yes/no). It can\'t chat, call tools, or run a subagent.', + ].join("\n"), }), all: Schema.optionalKey(Schema.Boolean).annotate({ description: "Include older versions of each model family. By default only the newest version is listed.", @@ -222,7 +227,7 @@ export const Plugin = { draft.add({ name: "models", description: - "Search the models available to use. Use this to turn a model name the user mentions into an exact reference before running a subagent on it. Check your own provider first.", + "Search the models available to use. Use this to turn a model name the user mentions into an exact reference for a tool that takes a model. Check your own provider first.", input: ModelsInput, output: ModelsOutput, options: { namespace: "opencode", codemode: true }, From 9590317bf8604b9955fdf7eb3c1566a2ab637097 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 17:21:46 -0500 Subject: [PATCH 09/17] feat(core): align image_generate model guidance with subagent --- packages/core/src/tool/plugin/image-generate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 76885da2ce74..229b86e3b46b 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -35,7 +35,7 @@ const EXTENSIONS: Record = { const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] export const description = - "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default; to choose one, find image models with the models tool and type: \"image\". Runs in the background: the call returns right away and you are notified when the image is saved; then use the read tool on its path to view it, and mention the path to the user. Don't poll or wait for it. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." + "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default. Runs in the background: the call returns right away and you are notified when the image is saved; then use the read tool on its path to view it, and mention the path to the user. Don't poll or wait for it. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." export const Input = Schema.Struct({ prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ @@ -50,7 +50,7 @@ export const Input = Schema.Struct({ }), model: Schema.optionalKey(Schema.String).annotate({ description: - 'NEVER set this unless the user explicitly asks for a particular image model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look it up with the models tool using type: "image".', + 'NEVER set this unless the user explicitly asks for a particular image model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look it up with the models tool using type: "image", filtering to your own provider first.', }), path: Schema.optionalKey(Schema.String).annotate({ description: From 634880e2915f9dc69f24aa354697b89b7e24d975 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 18:13:15 -0500 Subject: [PATCH 10/17] feat(core): stronger background wording for image_generate, chat-only subagent models --- packages/core/src/tool/plugin/image-generate.ts | 9 +++++++-- packages/core/src/tool/plugin/subagent.ts | 12 +++++++++--- packages/core/test/tool-image-generate.test.ts | 7 ++++++- packages/core/test/tool-subagent.test.ts | 14 ++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 229b86e3b46b..cdd2767eb928 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -35,7 +35,7 @@ const EXTENSIONS: Record = { const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] export const description = - "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default. Runs in the background: the call returns right away and you are notified when the image is saved; then use the read tool on its path to view it, and mention the path to the user. Don't poll or wait for it. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." + "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default. Runs in the background: the call returns right away and you are notified automatically when the image is saved or fails. DO NOT sleep, poll, or check whether the file exists while it runs; continue with other work or end your response. After the notice, use the read tool on the path to view the image and mention the path to the user. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." export const Input = Schema.Struct({ prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ @@ -336,7 +336,12 @@ export const Plugin = { yield* notifyWhenDone(context.sessionID, job.id, requested) return { output: { path: requested, model: ref }, - content: `Image request started in the background with ${ref}. Output path: ${requested}. You will be notified when it finishes or fails. Do not poll; use the read tool on the path after completion.${resolved.note ?? ""}`, + content: [ + `Image request started in the background with ${ref}. It will be saved to: ${requested}${resolved.note ?? ""}`, + "You will be notified automatically when it finishes or fails.", + "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", + "Work on other tasks, or briefly tell the user what you started and end your response.", + ].join("\n"), metadata: { path: requested, model: ref, jobID: job.id }, } }).pipe( diff --git a/packages/core/src/tool/plugin/subagent.ts b/packages/core/src/tool/plugin/subagent.ts index bd7cfcc788e3..b3cc81f65150 100644 --- a/packages/core/src/tool/plugin/subagent.ts +++ b/packages/core/src/tool/plugin/subagent.ts @@ -36,7 +36,7 @@ export const Input = Schema.Struct({ prompt: Schema.String.annotate({ description: "The task for the subagent to perform" }), model: Schema.optionalKey(Schema.String).annotate({ description: - 'NEVER set this unless the user explicitly asks for a particular model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look the model up with the models tool, filtering to your own provider first.', + 'NEVER set this unless the user explicitly asks for a particular model or variant. It must be a chat model; image and decision models can\'t run subagents. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look the model up with the models tool, filtering to your own provider first.', }), sessionID: Schema.optionalKey(SessionSchema.ID).annotate({ description: "Send a follow-up to an existing subagent, running or finished. Omit to start a new one.", @@ -83,10 +83,16 @@ export const Plugin = { const model = (yield* models.available()).find( (model) => model.providerID === ref.providerID && model.id === ref.id, ) - if (model === undefined) + if (model === undefined) { + const other = (yield* models.available(["image", "decision"])).find( + (model) => model.providerID === ref.providerID && model.id === ref.id, + ) return yield* new ToolFailure({ - message: `Model "${ref.providerID}/${ref.id}" is not available. Use the models tool to see what is available.`, + message: other + ? `Model "${ref.providerID}/${ref.id}" is ${other.type === "image" ? "an image" : "a decision"} model; subagents need a chat model. Use the models tool to find one.` + : `Model "${ref.providerID}/${ref.id}" is not available. Use the models tool to see what is available.`, }) + } if (ref.variant !== undefined && !model.variants.some((variant) => variant.id === ref.variant)) return yield* new ToolFailure({ message: diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index af13939a37c1..0c660124a191 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -229,7 +229,12 @@ describe("image_generate", () => { expect(result.content).toEqual([ { type: "text", - text: `Image request started in the background with test/painter. Output path: ${file}. You will be notified when it finishes or fails. Do not poll; use the read tool on the path after completion.`, + text: [ + `Image request started in the background with test/painter. It will be saved to: ${file}`, + "You will be notified automatically when it finishes or fails.", + "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", + "Work on other tasks, or briefly tell the user what you started and end your response.", + ].join("\n"), }, ]) expect(yield* notice(result)).toBe( diff --git a/packages/core/test/tool-subagent.test.ts b/packages/core/test/tool-subagent.test.ts index bc579b0d121c..7cd56541bd16 100644 --- a/packages/core/test/tool-subagent.test.ts +++ b/packages/core/test/tool-subagent.test.ts @@ -184,6 +184,12 @@ const withSubagent = (location: Location.Ref) => model.variants = [{ id: Model.VariantID.make("fast") }] }) editor.models.update(Provider.ID.make("test"), Model.ID.make("plain"), () => {}) + editor.models.update(Provider.ID.make("test"), Model.ID.make("painter"), (model) => { + model.type = "image" + }) + editor.models.update(Provider.ID.make("test"), Model.ID.make("classifier"), (model) => { + model.type = "decision" + }) }), ).pipe(Effect.provide(locations.get(location))) yield* Agent.Service.use((agents) => @@ -774,6 +780,14 @@ describe("SubagentTool", () => { ["test/missing", 'Model "test/missing" is not available. Use the models tool to see what is available.'], ["test/override#slow", 'Variant "slow" is not available for "test/override". Available: fast.'], ["test/plain#high", 'Model "test/plain" has no variants. Omit the variant.'], + [ + "test/painter", + 'Model "test/painter" is an image model; subagents need a chat model. Use the models tool to find one.', + ], + [ + "test/classifier", + 'Model "test/classifier" is a decision model; subagents need a chat model. Use the models tool to find one.', + ], ] as const for (const [model, message] of failures) { expect(yield* call(`call-${model}`, { model })).toEqual({ From d6b5311ac9e18187d7c60feccd11a6a2e79c87cb Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 19:05:18 -0500 Subject: [PATCH 11/17] feat(core): simpler default image model, webfetch permission for image URLs, drop decision from models tool --- .../core/src/tool/plugin/image-generate.ts | 28 ++++++++++++----- packages/core/src/tool/plugin/opencode.ts | 3 +- .../core/test/tool-image-generate.test.ts | 30 ++++++++++++++----- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index cdd2767eb928..d5ba4aeccb40 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -31,8 +31,6 @@ const EXTENSIONS: Record = { "image/webp": ".webp", "image/gif": ".gif", } -// Families tried in order when the session's provider has no image model, before falling back to the newest. -const PREFERRED_FAMILIES = ["gpt-image", "gemini-flash", "gemini-pro", "grok", "flux"] export const description = "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default. Runs in the background: the call returns right away and you are notified automatically when the image is saved or fails. DO NOT sleep, poll, or check whether the file exists while it runs; continue with other work or end your response. After the notice, use the read tool on the path to view the image and mention the path to the user. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." @@ -148,8 +146,9 @@ export const Plugin = { const readImage = Effect.fn("ImageGenerateTool.readImage")(function* ( source: string, context: FileAccess.Invocation, + fetchAllowed: (url: string) => ReturnType, ) { - const bytes = yield* readBytes(source, context) + const bytes = yield* readBytes(source, context, fetchAllowed) const mime = Mime.detect(bytes) if (!INPUT_TYPES.includes(mime)) return yield* new ToolFailure({ message: `${source} is not a PNG, JPEG, WebP or GIF image.` }) @@ -159,10 +158,13 @@ export const Plugin = { const readBytes = Effect.fn("ImageGenerateTool.readBytes")(function* ( source: string, context: FileAccess.Invocation, + fetchAllowed: (url: string) => ReturnType, ) { if (/^https?:\/\//i.test(source)) { if (!source.toLowerCase().startsWith("https://")) return yield* new ToolFailure({ message: `Image URLs must use https: ${source}` }) + // Downloads follow the same rule as the webfetch tool, so denying web access also covers image URLs. + yield* fetchAllowed(source) return yield* Effect.gen(function* () { const response = yield* http .execute(HttpClientRequest.get(source)) @@ -285,7 +287,19 @@ export const Plugin = { }) } const images = input.images - ? yield* Effect.forEach(input.images, (image) => readImage(image, context)) + ? yield* Effect.forEach(input.images, (image) => + readImage(image, context, (url) => + permission.assert({ + action: "webfetch", + resources: [url], + save: ["*"], + metadata: { url }, + sessionID: context.sessionID, + agent: context.agent, + source, + }), + ), + ) : undefined // Everything that needs the user or can fail fast happened above; the paid call runs as a background job. @@ -355,15 +369,13 @@ export const Plugin = { }), } -/** Same provider as the session first, then preferred families, then the newest. */ +/** The newest image model from the session's provider, otherwise the newest available one. */ export function defaultModel( available: readonly Model.Info[], input: { readonly providerID?: string; readonly images: boolean }, ) { const candidates = input.images ? available.filter(acceptsImages) : available - const own = candidates.filter((model) => model.providerID === input.providerID) - const pool = own.length > 0 ? own : candidates - return PREFERRED_FAMILIES.flatMap((family) => pool.find((model) => model.family === family) ?? [])[0] ?? pool[0] + return candidates.find((model) => model.providerID === input.providerID) ?? candidates[0] } // Read per field so a later config that sets other experimental options doesn't hide this one. diff --git a/packages/core/src/tool/plugin/opencode.ts b/packages/core/src/tool/plugin/opencode.ts index 5171ff778590..d18bb45e41d1 100644 --- a/packages/core/src/tool/plugin/opencode.ts +++ b/packages/core/src/tool/plugin/opencode.ts @@ -50,12 +50,11 @@ export const ModelsInput = Schema.Struct({ provider: Schema.optionalKey(Schema.String).annotate({ description: "Provider ID or name to filter by. Try your own provider first.", }), - type: Schema.optionalKey(Schema.Literals(["chat", "image", "decision"])).annotate({ + type: Schema.optionalKey(Schema.Literals(["chat", "image"])).annotate({ description: [ 'Which kind of model to list. Defaults to "chat".', '"chat": a language model (LLM). Powers sessions, agents and subagents, and can call tools.', '"image": generates or edits images from a prompt. It can\'t chat or call tools.', - '"decision": a fast classifier, not an LLM. Given some input and a set of questions, it returns structured answers (a label, a score, or yes/no). It can\'t chat, call tools, or run a subagent.', ].join("\n"), }), all: Schema.optionalKey(Schema.Boolean).annotate({ diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 0c660124a191..b184b02bed2a 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -411,6 +411,23 @@ describe("image_generate", () => { }), ) + it.live("asks webfetch permission before downloading an image URL", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run } = yield* setup() + fixture.deny = "webfetch" + const result = yield* run({ prompt: "x", images: ["https://example.invalid/cat.png"] }) + expect(result.status).toBe("error") + expect(fixture.assertions.map((item) => [item.action, item.resources])).toEqual([ + ["image_generate", ["test/painter"]], + ["webfetch", ["https://example.invalid/cat.png"]], + ]) + expect(fixture.requests).toEqual([]) + }), + ) + it.live("rejects input files that are not images", () => Effect.gen(function* () { yield* imageModel("test", "painter") @@ -469,19 +486,16 @@ describe("defaultModel", () => { return model && `${model.providerID}/${model.id}` } - it.effect("prefers the session's provider, then known families, then the newest", () => + it.effect("prefers the session's provider, then the newest", () => Effect.sync(() => { const newest = info("fresh", "newest") - const flux = info("bfl", "flux-2", "flux") - const gemini = info("google", "nano-banana", "gemini-flash") - const gpt = info("openai", "gpt-image-2", "gpt-image") + const older = info("openai", "gpt-image-2", "gpt-image") const own = info("mine", "own-image") - const all = [newest, flux, gemini, gpt, own] + const all = [newest, older, own] expect(pick(all, "mine")).toBe("mine/own-image") - expect(pick(all, "elsewhere")).toBe("openai/gpt-image-2") - expect(pick([newest, flux, gemini])).toBe("google/nano-banana") - expect(pick([newest, info("x", "unknown", "mystery")])).toBe("fresh/newest") + expect(pick(all, "elsewhere")).toBe("fresh/newest") + expect(pick(all)).toBe("fresh/newest") expect(pick([])).toBeUndefined() }), ) From ef757c4da555c426b4e7727700b62a8f42e875f5 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 19:13:40 -0500 Subject: [PATCH 12/17] feat(core): flatten image config to experimental.image_model --- .../client/src/promise/generated/types.ts | 2 +- packages/core/src/config/normalize.ts | 10 ++--- .../core/src/tool/plugin/image-generate.ts | 19 ++++----- .../core/test/config/normalization.test.ts | 19 ++++----- .../core/test/tool-image-generate.test.ts | 36 +++++----------- packages/protocol/openapi.json | 42 +++++++------------ packages/schema/src/config/experimental.ts | 13 ++---- 7 files changed, 50 insertions(+), 91 deletions(-) diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 32c11e903114..a93cd8c37b84 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -2426,7 +2426,7 @@ export type ConfigEntry = todo?: boolean background_compaction?: boolean media_blob?: boolean - image_generation?: false | { model?: string | { providerID: string; model: string; variant?: string } } + image_model?: string | { providerID: string; model: string; variant?: string } schedules?: boolean subagent_depth?: number location_idle_timeout?: string | false diff --git a/packages/core/src/config/normalize.ts b/packages/core/src/config/normalize.ts index 7163888ce85b..434f45051dcd 100644 --- a/packages/core/src/config/normalize.ts +++ b/packages/core/src/config/normalize.ts @@ -495,14 +495,14 @@ function normalizeExperimental( ) if (value !== undefined) result.media_blob = value } - if (own(experimental, "image_generation")) { + if (own(experimental, "image_model")) { const value = decodeEncoded( - ConfigExperimental.Info.fields.image_generation, - experimental.image_generation, - ["experimental", "image_generation"], + ConfigExperimental.Info.fields.image_model, + experimental.image_model, + ["experimental", "image_model"], diagnostics, ) - if (value !== undefined) result.image_generation = value + if (value !== undefined) result.image_model = value } if (own(experimental, "schedules")) { const value = decodeEncoded( diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index d5ba4aeccb40..e31ad1df3729 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -84,8 +84,7 @@ export const Plugin = { const available = yield* models.available(["image"]) if (!input) { // A configured default is a preference: when it can't serve this call, pick automatically and say so. - const settings = settingsFrom(yield* config.entries()) - const configured = settings ? settings.model : undefined + const configured = configuredModel(yield* config.entries()) const preferred = configured ? available.find((model) => model.providerID === configured.providerID && model.id === configured.model) : undefined @@ -224,11 +223,9 @@ export const Plugin = { Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), ) - // Registered only while an image model is available and config doesn't turn the tool off, re-evaluated - // whenever either changes. Subscribing before the first read keeps a change in between from being missed. - const wanted = Effect.all([config.entries(), models.available(["image"])]).pipe( - Effect.map(([entries, available]) => settingsFrom(entries) !== false && available.length > 0), - ) + // Registered only while an image model is available, re-evaluated whenever the catalog changes. + // Subscribing before the first read keeps a change in between from being missed. + const wanted = models.available(["image"]).pipe(Effect.map((available) => available.length > 0)) let enabled = false const refresh = wanted.pipe( Effect.flatMap((next) => { @@ -237,7 +234,7 @@ export const Plugin = { return ctx.tool.reload() }), ) - yield* ctx.event.subscribe({ types: ["model.updated", "config.updated"] }).pipe( + yield* ctx.event.subscribe({ types: ["model.updated"] }).pipe( Stream.runForEach(() => refresh), Effect.forkScoped({ startImmediately: true }), ) @@ -379,10 +376,10 @@ export function defaultModel( } // Read per field so a later config that sets other experimental options doesn't hide this one. -function settingsFrom(entries: readonly Entry[]) { +function configuredModel(entries: readonly Entry[]) { return entries.findLast( - (entry): entry is Document => entry.type === "document" && entry.info.experimental?.image_generation !== undefined, - )?.info.experimental?.image_generation + (entry): entry is Document => entry.type === "document" && entry.info.experimental?.image_model !== undefined, + )?.info.experimental?.image_model } function acceptsImages(model: Model.Info) { diff --git a/packages/core/test/config/normalization.test.ts b/packages/core/test/config/normalization.test.ts index 423f80ab5ff0..de47941e3185 100644 --- a/packages/core/test/config/normalization.test.ts +++ b/packages/core/test/config/normalization.test.ts @@ -545,7 +545,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, - image_generation: { model: "openai/gpt-image-2" }, + image_model: "openai/gpt-image-2", subagent_depth: 0, location_idle_timeout: "2 hours", policies: [{ action: "provider.use", resource: "custom", effect: "allow" }], @@ -557,7 +557,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, - image_generation: { model: { providerID: "openai", model: "gpt-image-2" } }, + image_model: { providerID: "openai", model: "gpt-image-2" }, subagent_depth: 0, location_idle_timeout: "7200000 millis", }, @@ -571,18 +571,15 @@ describe("ConfigNormalize", () => { }) }) - test("normalizes the image generation setting", () => { - expect(normalized({ experimental: { image_generation: false } }).encoded.experimental).toEqual({ - image_generation: false, + test("normalizes the image model setting", () => { + expect(normalized({ experimental: { image_model: "openai/gpt-image-2#hd" } }).encoded.experimental).toEqual({ + image_model: { providerID: "openai", model: "gpt-image-2", variant: "hd" }, }) - expect(normalized({ experimental: { image_generation: {} } }).encoded.experimental).toEqual({ - image_generation: {}, - }) - for (const value of [true, { model: "no-slash" }, { model: 1 }, "openai/gpt-image-2"]) { - const result = normalized({ experimental: { image_generation: value } }) + for (const value of [false, true, "no-slash", 1, { model: "openai/gpt-image-2" }]) { + const result = normalized({ experimental: { image_model: value } }) expect(result.encoded.experimental).toBeUndefined() expect(result.diagnostics.map((item) => [item.kind, item.path])).toEqual([ - ["invalid", ["experimental", "image_generation"]], + ["invalid", ["experimental", "image_model"]], ]) } }) diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index b184b02bed2a..c675a74dcda4 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -128,13 +128,13 @@ const setup = (providerID = "test") => return { session, run, notice, directory: location.directory } }) -const configure = (image_generation: ConfigExperimental.Info["image_generation"]) => +const configure = (image_model: ConfigExperimental.Info["image_model"]) => Effect.gen(function* () { const config = yield* Config.Test yield* config.setEntries([ new Document({ type: "document", - info: new Info({ experimental: new ConfigExperimental.Info({ image_generation }) }), + info: new Info({ experimental: new ConfigExperimental.Info({ image_model }) }), }), ]) const bus = yield* Bus.Service @@ -142,7 +142,8 @@ const configure = (image_generation: ConfigExperimental.Info["image_generation"] }) const configuredModel = (providerID: string, model: string) => ({ - model: { providerID: Provider.ID.make(providerID), model: Model.ID.make(model) }, + providerID: Provider.ID.make(providerID), + model: Model.ID.make(model), }) const registered = Tool.Service.use((registry) => toolDefinitions(registry)).pipe( @@ -172,19 +173,6 @@ describe("image_generate", () => { }), ) - it.live("is not registered while config turns it off", () => - Effect.gen(function* () { - yield* imageModel("test", "painter") - yield* configure(false) - yield* setup() - expect(yield* registered).toBe(false) - yield* configure({}) - expect(yield* until(true)).toBe(true) - yield* configure(false) - expect(yield* until(false)).toBe(false) - }), - ) - it.live("uses the configured model unless the agent picks one, and falls back with a note", () => Effect.gen(function* () { yield* imageModel("test", "painter", (model) => { @@ -386,11 +374,9 @@ describe("image_generate", () => { ) yield* configure({ - model: { - providerID: Provider.ID.make("test"), - model: Model.ID.make("painter"), - variant: Model.VariantID.make("hd"), - }, + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("hd"), }) const configured = yield* run({ prompt: "x" }) expect(configured.output).toMatchObject({ model: "test/painter#hd" }) @@ -398,11 +384,9 @@ describe("image_generate", () => { expect(fixture.requests.at(-1)?.model.variant).toBe(Model.VariantID.make("hd")) yield* configure({ - model: { - providerID: Provider.ID.make("test"), - model: Model.ID.make("painter"), - variant: Model.VariantID.make("bogus"), - }, + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("bogus"), }) const fallback = yield* run({ prompt: "x" }) expect(text(fallback)).toContain('test/painter#bogus has no variant "bogus"; used ') diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index af6d5b5c4025..5abbbeaed084 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -15790,43 +15790,29 @@ "media_blob": { "type": "boolean" }, - "image_generation": { + "image_model": { "anyOf": [ { - "type": "boolean", - "enum": [false] + "type": "string", + "pattern": "^[^/#]+\\/[^#]+(?:#[^#]+)?$" }, { "type": "object", "properties": { + "providerID": { + "type": "string", + "pattern": "^[^/#]+$" + }, "model": { - "anyOf": [ - { - "type": "string", - "pattern": "^[^/#]+\\/[^#]+(?:#[^#]+)?$" - }, - { - "type": "object", - "properties": { - "providerID": { - "type": "string", - "pattern": "^[^/#]+$" - }, - "model": { - "type": "string", - "pattern": "^[^#]+$" - }, - "variant": { - "type": "string", - "pattern": "^[^#]+$" - } - }, - "required": ["providerID", "model"], - "additionalProperties": false - } - ] + "type": "string", + "pattern": "^[^#]+$" + }, + "variant": { + "type": "string", + "pattern": "^[^#]+$" } }, + "required": ["providerID", "model"], "additionalProperties": false } ] diff --git a/packages/schema/src/config/experimental.ts b/packages/schema/src/config/experimental.ts index f9f2d181dd15..ec463c856d3b 100644 --- a/packages/schema/src/config/experimental.ts +++ b/packages/schema/src/config/experimental.ts @@ -20,15 +20,10 @@ export class Info extends Schema.Class("ConfigExperimental.Info")({ description: "Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.", }), - image_generation: Schema.Union([ - Schema.Literal(false), - Schema.Struct({ model: ConfigModel.Selection.pipe(optional) }), - ]) - .pipe(optional) - .annotate({ - description: - "Image generation tool settings. `model` sets the default image model (providerID/modelID) used when the agent doesn't choose one; set to false to disable the image_generate tool.", - }), + image_model: ConfigModel.Selection.pipe(optional).annotate({ + description: + "Default model for the image_generate tool when the agent doesn't choose one, as providerID/modelID. Falls back to automatic selection when it isn't available.", + }), schedules: Schema.Boolean.pipe(optional).annotate({ description: "Enable experimental scheduled tasks that wake a session with a prompt. Defaults to false.", }), From 8e611c94561e55ddfdfb09ca5beb4f3b1ff6cb86 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 19:44:35 -0500 Subject: [PATCH 13/17] fix(core): count image jobs as background work for idle hooks --- packages/core/src/plugin/goal.ts | 9 ++++--- packages/core/src/session/background.ts | 36 +++++++++++++------------ packages/plugin/src/effect/session.ts | 6 +++++ packages/plugin/src/promise/session.ts | 6 +++++ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/packages/core/src/plugin/goal.ts b/packages/core/src/plugin/goal.ts index bd841405450d..67b3daff8ff5 100644 --- a/packages/core/src/plugin/goal.ts +++ b/packages/core/src/plugin/goal.ts @@ -145,9 +145,12 @@ export const Plugin = define({ if (goal?.status !== "active") return if (goal.boot !== boot()) return yield* settle(sessionID, goal, "paused", "The server restarted while the goal was active") - // The Session's own running subagent wakes it with its report, and the hook runs again then. A grandchild reports - // to its own parent, so it would never wake this Session. - if (background.some((work) => work.kind === "subagent" && work.sessionID === sessionID)) return + // The Session's own running subagent or image job wakes it with its notice, and the hook runs again then. A + // grandchild reports to its own parent, so it would never wake this Session. + if ( + background.some((work) => (work.kind === "subagent" || work.kind === "image") && work.sessionID === sessionID) + ) + return if (goal.rounds >= goal.maxRounds) return yield* settle(sessionID, goal, "paused", `Reached the limit of ${goal.maxRounds} rounds`) diff --git a/packages/core/src/session/background.ts b/packages/core/src/session/background.ts index 1a247a7afa64..df8b38bb0ec3 100644 --- a/packages/core/src/session/background.ts +++ b/packages/core/src/session/background.ts @@ -16,24 +16,26 @@ export const outstanding = Effect.fnUntraced(function* ( sessionID: SessionSchema.ID, ) { const owned = yield* Effect.filter(yield* jobs.runningBackground, (recovery) => - descends(store, recovery.kind === "shell" ? recovery.sessionID : recovery.parentSessionID, sessionID), - ) - return owned.map((recovery) => - recovery.kind === "shell" - ? { - kind: "shell" as const, - sessionID: recovery.sessionID, - shellID: recovery.shellID, - command: recovery.command, - } - : { - kind: "subagent" as const, - sessionID: recovery.parentSessionID, - childSessionID: recovery.childSessionID, - agent: recovery.agent, - description: recovery.description, - }, + descends(store, recovery.kind === "subagent" ? recovery.parentSessionID : recovery.sessionID, sessionID), ) + return owned.map((recovery) => { + if (recovery.kind === "shell") + return { + kind: "shell" as const, + sessionID: recovery.sessionID, + shellID: recovery.shellID, + command: recovery.command, + } + if (recovery.kind === "image") + return { kind: "image" as const, sessionID: recovery.sessionID, path: recovery.path, model: recovery.model } + return { + kind: "subagent" as const, + sessionID: recovery.parentSessionID, + childSessionID: recovery.childSessionID, + agent: recovery.agent, + description: recovery.description, + } + }) }) const descends = Effect.fnUntraced(function* ( diff --git a/packages/plugin/src/effect/session.ts b/packages/plugin/src/effect/session.ts index 920ecf2049d0..c5f04bc673b6 100644 --- a/packages/plugin/src/effect/session.ts +++ b/packages/plugin/src/effect/session.ts @@ -154,6 +154,12 @@ export type SessionIdleBackground = readonly agent: string readonly description: string } + | { + readonly kind: "image" + readonly sessionID: Session.ID + readonly path: string + readonly model: string + } /** * Runs when a Session finishes model work and has no pending input, just before it goes idle. diff --git a/packages/plugin/src/promise/session.ts b/packages/plugin/src/promise/session.ts index 743bd8f7d6d1..1621ce8b30d2 100644 --- a/packages/plugin/src/promise/session.ts +++ b/packages/plugin/src/promise/session.ts @@ -154,6 +154,12 @@ export type SessionIdleBackground = readonly agent: string readonly description: string } + | { + readonly kind: "image" + readonly sessionID: Session.ID + readonly path: string + readonly model: string + } /** * Runs when a Session finishes model work and has no pending input, just before it goes idle. From b9aa4d75ec1edcea1ad56e30b53e4bcc58203aef Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 20:14:02 -0500 Subject: [PATCH 14/17] feat(core): let image_generate overwrite an explicit path --- packages/core/src/tool/plugin/image-generate.ts | 13 ++----------- packages/core/test/tool-image-generate.test.ts | 13 ++++++------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index e31ad1df3729..3ebe369ad91f 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -52,7 +52,7 @@ export const Input = Schema.Struct({ }), path: Schema.optionalKey(Schema.String).annotate({ description: - "Where to save the image, relative to the session directory or absolute. Never overwrites. Defaults to a new file in the session's temp folder.", + "Where to save the image, relative to the session directory or absolute. Overwrites an existing file. Defaults to a new file in the session's temp folder.", }), }) @@ -217,12 +217,6 @@ export const Plugin = { Effect.forkIn(scope, { startImmediately: true }), ) - const exists = (file: string) => - environment.files.stat(file).pipe( - Effect.as(true), - Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), - ) - // Registered only while an image model is available, re-evaluated whenever the catalog changes. // Subscribing before the first read keeps a change in between from being missed. const wanted = models.available(["image"]).pipe(Effect.map((available) => available.length > 0)) @@ -260,8 +254,6 @@ export const Plugin = { const requested = target?.absolute ?? path.join(global.tmp, "images", context.sessionID, `image_${Identifier.create(false)}.png`) - if (yield* exists(requested)) - return yield* new ToolFailure({ message: `${requested} already exists. Choose a new path.` }) yield* permission.assert({ action: name, @@ -323,10 +315,9 @@ export const Plugin = { const bytes = Buffer.from(image.data, "base64") const mime = Mime.detect(bytes) // Providers may return JPEG or WebP and core has no encoder. A default path takes the real extension; - // an explicit path is kept as given, because only that exact path was approved and checked. + // an explicit path is written exactly as given, since that's the path the edit permission covered. const extension = EXTENSIONS[mime] ?? ".png" const file = target || extension === ".png" ? requested : requested.slice(0, -".png".length) + extension - if (yield* exists(file)) return yield* Effect.fail(new Error(`${file} already exists.`)) yield* environment.files.write(file, bytes) const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` return `Image request finished. Saved image: ${file} (${ref}).${note}\nUse the read tool on this path to view the image.` diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index c675a74dcda4..1e0a84c56411 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -203,7 +203,7 @@ describe("image_generate", () => { }), ) - it.live("reads input images, asks permission, and saves the result without overwriting", () => + it.live("reads input images, asks permission, and saves the result, overwriting an existing file", () => Effect.gen(function* () { yield* imageModel("test", "painter", (model) => { model.capabilities.input = ["text", "image"] @@ -244,12 +244,11 @@ describe("image_generate", () => { ]) expect(fixture.assertions[0]?.sessionID).toBe(session.id) - // An existing file is never replaced, and nothing is generated or charged for. - fixture.requests.length = 0 - expect(failure(yield* run({ prompt: "again", path: "out.png" }))).toBe( - `${file} already exists. Choose a new path.`, - ) - expect(fixture.requests).toEqual([]) + // Running again with the same path replaces the file, like the write tool. + const again = yield* run({ prompt: "again", path: "out.png" }) + expect(again.status).toBe("completed") + yield* notice(again) + expect(fixture.requests).toHaveLength(2) }), ) From 58a0c3e63e41f95417b4b8cec8cb1f69dd446c8e Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 20:18:50 -0500 Subject: [PATCH 15/17] feat(core): save generated images with their real extension --- .../core/src/tool/plugin/image-generate.ts | 45 +++++++++++-------- .../core/test/tool-image-generate.test.ts | 31 ++++++++----- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 3ebe369ad91f..b4450819bc98 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -52,12 +52,12 @@ export const Input = Schema.Struct({ }), path: Schema.optionalKey(Schema.String).annotate({ description: - "Where to save the image, relative to the session directory or absolute. Overwrites an existing file. Defaults to a new file in the session's temp folder.", + "Where to save the image, relative to the session directory or absolute. The extension is replaced with the image's real format, and an existing file is overwritten. Defaults to a new file in the session's temp folder.", }), }) export const Output = Schema.Struct({ - path: Schema.String, + path: Schema.String.annotate({ description: "Where the image is saved, without its extension" }), model: Schema.String.annotate({ description: "providerID/modelID" }), }) @@ -251,9 +251,10 @@ export const Plugin = { const variant = resolved.variant const ref = `${model.providerID}/${model.id}${variant ? `#${variant}` : ""}` const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined - const requested = - target?.absolute ?? - path.join(global.tmp, "images", context.sessionID, `image_${Identifier.create(false)}.png`) + // The extension the image is saved with follows its real format, whatever the requested path ends in. + const stem = target + ? target.absolute.slice(0, target.absolute.length - path.extname(target.absolute).length) + : path.join(global.tmp, "images", context.sessionID, `image_${Identifier.create(false)}`) yield* permission.assert({ action: name, @@ -313,38 +314,44 @@ export const Plugin = { const image = result.images[0] if (!image) return yield* Effect.fail(new Error(`${ref} returned no image.`)) const bytes = Buffer.from(image.data, "base64") - const mime = Mime.detect(bytes) - // Providers may return JPEG or WebP and core has no encoder. A default path takes the real extension; - // an explicit path is written exactly as given, since that's the path the edit permission covered. - const extension = EXTENSIONS[mime] ?? ".png" - const file = target || extension === ".png" ? requested : requested.slice(0, -".png".length) + extension + const file = stem + (EXTENSIONS[Mime.detect(bytes)] ?? ".png") + // The edit prompt covered the requested path; a different extension must still not be denied. + if (target && file !== target.absolute) { + const actual = yield* access.resolve({ path: file, kind: "file" }) + yield* permission.assertNotDenied({ + action: "edit", + resources: [actual.resource], + sessionID: context.sessionID, + agent: context.agent, + source, + }) + } yield* environment.files.write(file, bytes) - const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.` - return `Image request finished. Saved image: ${file} (${ref}).${note}\nUse the read tool on this path to view the image.` + return `Image request finished. Saved image: ${file} (${ref}).\nUse the read tool on this path to view the image.` }).pipe( // The job keeps this text, so a restart can still deliver the outcome. - Effect.mapError((error) => new Error(`Image request failed for ${requested}: ${error.message}`)), + Effect.mapError((error) => new Error(`Image request failed for ${stem}: ${error.message}`)), ) const id = `image_${Identifier.create(false)}` const job = yield* jobs.start({ id, type: name, title: input.prompt, - metadata: { sessionID: context.sessionID, path: requested, model: ref }, - recovery: { kind: "image", sessionID: context.sessionID, path: requested, model: ref }, + metadata: { sessionID: context.sessionID, path: stem, model: ref }, + recovery: { kind: "image", sessionID: context.sessionID, path: stem, model: ref }, run, }) yield* jobs.background(job.id) - yield* notifyWhenDone(context.sessionID, job.id, requested) + yield* notifyWhenDone(context.sessionID, job.id, stem) return { - output: { path: requested, model: ref }, + output: { path: stem, model: ref }, content: [ - `Image request started in the background with ${ref}. It will be saved to: ${requested}${resolved.note ?? ""}`, + `Image request started in the background with ${ref}. It will be saved as ${stem} with the extension of its real format (.png, .jpg, .webp or .gif); the notice gives the exact path.${resolved.note ?? ""}`, "You will be notified automatically when it finishes or fails.", "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", "Work on other tasks, or briefly tell the user what you started and end your response.", ].join("\n"), - metadata: { path: requested, model: ref, jobID: job.id }, + metadata: { path: stem, model: ref, jobID: job.id }, } }).pipe( Effect.mapError((error) => diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 1e0a84c56411..0c3faa756a5c 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -33,6 +33,7 @@ const JPEG = const fixture = { requests: [] as ImageRequest[], assertions: [] as Permission.AssertInput[], + checked: [] as Permission.AssertInput[], result: { images: [{ type: "image", data: PNG, mime: "image/png" }] } as ImageResult, // Holds generation until the test releases it, to observe a running job. gate: Effect.void as Effect.Effect, @@ -43,6 +44,7 @@ const fixture = { const it = testEffect( makePluginTestLayer( permissionLayer({ + assertNotDenied: (input) => Effect.sync(() => void fixture.checked.push(input)), assert: (input) => Effect.sync(() => void fixture.assertions.push(input)).pipe( Effect.andThen( @@ -86,6 +88,7 @@ const setup = (providerID = "test") => Effect.gen(function* () { fixture.requests.length = 0 fixture.assertions.length = 0 + fixture.checked.length = 0 fixture.result = { images: [{ type: "image", data: PNG, mime: "image/png" }] } fixture.gate = Effect.void fixture.failure = undefined @@ -213,12 +216,13 @@ describe("image_generate", () => { const result = yield* run({ prompt: "make it blue", images: ["in.png"], path: "out.png" }) const file = path.join(directory, "out.png") - expect(result.output).toEqual({ path: file, model: "test/painter" }) + const stem = path.join(directory, "out") + expect(result.output).toEqual({ path: stem, model: "test/painter" }) expect(result.content).toEqual([ { type: "text", text: [ - `Image request started in the background with test/painter. It will be saved to: ${file}`, + `Image request started in the background with test/painter. It will be saved as ${stem} with the extension of its real format (.png, .jpg, .webp or .gif); the notice gives the exact path.`, "You will be notified automatically when it finishes or fails.", "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", "Work on other tasks, or briefly tell the user what you started and end your response.", @@ -252,26 +256,31 @@ describe("image_generate", () => { }), ) - it.live("saves to the session temp folder by default and handles a non-PNG result", () => + it.live("saves with the extension of the image's real format", () => Effect.gen(function* () { yield* imageModel("test", "painter") const { run, notice, session, directory } = yield* setup() const global = yield* Global.Service fixture.result = { images: [{ type: "image", data: JPEG, mime: "image/jpeg" }] } - // A default path takes the real extension, and the notice gives the final path. + // The default path lives in the session temp folder and takes the real extension. const result = yield* run({ prompt: "a cat" }) expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) - const saved = result.output.path.replace(/\.png$/, ".jpg") + const saved = `${result.output.path}.jpg` expect(yield* notice(result)).toBe( - `Image request finished. Saved image: ${saved} (test/painter). The model returned image/jpeg, so the file is in that format.\nUse the read tool on this path to view the image.`, + `Image request finished. Saved image: ${saved} (test/painter).\nUse the read tool on this path to view the image.`, ) expect(yield* Effect.promise(() => Bun.file(saved).exists())).toBe(true) - // An explicit path is kept exactly, since only that path was approved and checked before paying. + // An explicit path's extension is replaced too, and the notice gives the real path. const explicit = yield* run({ prompt: "a cat", path: "cat.png" }) - expect(explicit.output.path).toBe(path.join(directory, "cat.png")) - expect(yield* notice(explicit)).toContain(`Saved image: ${path.join(directory, "cat.png")} (test/painter).`) + expect(yield* notice(explicit)).toContain(`Saved image: ${path.join(directory, "cat.jpg")} (test/painter).`) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "cat.jpg")).exists())).toBe(true) + // The edit prompt covered cat.png; the file actually written is still checked against deny rules. + expect(fixture.assertions.filter((item) => item.action === "edit").map((item) => item.resources)).toEqual([ + ["cat.png"], + ]) + expect(fixture.checked.map((item) => [item.action, item.resources])).toEqual([["edit", ["cat.jpg"]]]) }), ) @@ -286,7 +295,7 @@ describe("image_generate", () => { }) const result = yield* run({ prompt: "x", path: "out.png" }) expect(yield* notice(result)).toBe( - `Image request failed for ${path.join(directory, "out.png")}: The prompt was rejected`, + `Image request failed for ${path.join(directory, "out")}: The prompt was rejected`, ) expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) }), @@ -302,7 +311,7 @@ describe("image_generate", () => { const id = String(result.metadata?.jobID) expect((yield* jobs.get(id))?.status).toBe("running") yield* jobs.cancel(id) - expect(yield* notice(result)).toBe(`Image request for ${path.join(directory, "out.png")} was cancelled.`) + expect(yield* notice(result)).toBe(`Image request for ${path.join(directory, "out")} was cancelled.`) expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) }), ) From 9789503f1c6d8fe3758fb7fe12d9cd82e07121c0 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 20:29:59 -0500 Subject: [PATCH 16/17] refactor(core): drop redundant checks from image_generate --- .../core/src/tool/plugin/image-generate.ts | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index b4450819bc98..64741bc0ca1a 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -17,6 +17,7 @@ import { Mime } from "../../mime.js" import { Model } from "../../model.js" import { Permission } from "../../permission.js" import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" import { SessionSchema } from "../../session/schema.js" import { collectBoundedResponseBody } from "../http-body.js" @@ -44,7 +45,7 @@ export const Input = Schema.Struct({ Schema.Array(Schema.String.check(Schema.isMinLength(1))).check(Schema.isMinLength(1), Schema.isMaxLength(8)), ).annotate({ description: - "Images to edit or use as references, in order. Local file paths (relative to the session directory or absolute) or https URLs. Omit to create a new image.", + "Images to edit or use as references, in order. Local file paths (relative to the session directory or absolute) or http(s) URLs. Omit to create a new image.", }), model: Schema.optionalKey(Schema.String).annotate({ description: @@ -160,19 +161,12 @@ export const Plugin = { fetchAllowed: (url: string) => ReturnType, ) { if (/^https?:\/\//i.test(source)) { - if (!source.toLowerCase().startsWith("https://")) - return yield* new ToolFailure({ message: `Image URLs must use https: ${source}` }) // Downloads follow the same rule as the webfetch tool, so denying web access also covers image URLs. yield* fetchAllowed(source) return yield* Effect.gen(function* () { const response = yield* http .execute(HttpClientRequest.get(source)) .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)) - const type = (response.headers["content-type"] ?? "").split(";", 1)[0]?.trim().toLowerCase() ?? "" - if (!INPUT_TYPES.includes(type)) - return yield* new ToolFailure({ - message: `${source} is not a PNG, JPEG, WebP or GIF image (${type || "no content type"}).`, - }) return yield* collectBoundedResponseBody( response, MAX_INPUT_BYTES, @@ -197,11 +191,11 @@ export const Plugin = { }) const notifyWhenDone = Effect.fn("ImageGenerateTool.notifyWhenDone")( - function* (sessionID: SessionSchema.ID, id: string, file: string) { + function* (sessionID: SessionSchema.ID, id: string, notificationID: SessionMessage.ID, file: string) { const info = (yield* jobs.wait({ id })).info if (!info || info.status === "running") return yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), + id: notificationID, sessionID, description: "Image generation", text: @@ -212,7 +206,7 @@ export const Plugin = { : `Image request for ${file} was cancelled.`, metadata: { source: name, jobID: id, state: info.status, path: file }, }) - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) + yield* jobs.completeBackground(notificationID) }, Effect.forkIn(scope, { startImmediately: true }), ) @@ -333,8 +327,10 @@ export const Plugin = { Effect.mapError((error) => new Error(`Image request failed for ${stem}: ${error.message}`)), ) const id = `image_${Identifier.create(false)}` + const notificationID = SessionMessage.ID.create() const job = yield* jobs.start({ id, + notificationID, type: name, title: input.prompt, metadata: { sessionID: context.sessionID, path: stem, model: ref }, @@ -342,7 +338,7 @@ export const Plugin = { run, }) yield* jobs.background(job.id) - yield* notifyWhenDone(context.sessionID, job.id, stem) + yield* notifyWhenDone(context.sessionID, job.id, notificationID, stem) return { output: { path: stem, model: ref }, content: [ From 68ab1ddc2d9b880d0ddac27dfd59dbef72d3c4e1 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 28 Oct 2026 20:34:01 -0500 Subject: [PATCH 17/17] fix(core): ask before replacing a file under the image's real extension --- packages/core/src/tool/plugin/image-generate.ts | 12 +++++++++--- packages/core/test/tool-image-generate.test.ts | 11 +++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts index 64741bc0ca1a..1cdaec7aaf89 100644 --- a/packages/core/src/tool/plugin/image-generate.ts +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -309,16 +309,22 @@ export const Plugin = { if (!image) return yield* Effect.fail(new Error(`${ref} returned no image.`)) const bytes = Buffer.from(image.data, "base64") const file = stem + (EXTENSIONS[Mime.detect(bytes)] ?? ".png") - // The edit prompt covered the requested path; a different extension must still not be denied. + // The edit prompt covered the requested path. Replacing a different existing file needs its own + // approval; a new file only has to pass the deny rules. if (target && file !== target.absolute) { const actual = yield* access.resolve({ path: file, kind: "file" }) - yield* permission.assertNotDenied({ + const replaces = yield* environment.files.stat(file).pipe( + Effect.as(true), + Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), + ) + const request = { action: "edit", resources: [actual.resource], sessionID: context.sessionID, agent: context.agent, source, - }) + } + yield* replaces ? permission.assert({ ...request, save: ["*"] }) : permission.assertNotDenied(request) } yield* environment.files.write(file, bytes) return `Image request finished. Saved image: ${file} (${ref}).\nUse the read tool on this path to view the image.` diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts index 0c3faa756a5c..3e4fc5fed5a5 100644 --- a/packages/core/test/tool-image-generate.test.ts +++ b/packages/core/test/tool-image-generate.test.ts @@ -281,6 +281,17 @@ describe("image_generate", () => { ["cat.png"], ]) expect(fixture.checked.map((item) => [item.action, item.resources])).toEqual([["edit", ["cat.jpg"]]]) + + // Replacing an existing file under the real extension asks for its own edit approval. + fixture.assertions.length = 0 + fixture.checked.length = 0 + const again = yield* run({ prompt: "a cat", path: "cat.png" }) + yield* notice(again) + expect(fixture.assertions.filter((item) => item.action === "edit").map((item) => item.resources)).toEqual([ + ["cat.png"], + ["cat.jpg"], + ]) + expect(fixture.checked).toEqual([]) }), )