Skip to content

Commit edd5fc1

Browse files
rekram1-noderekram1-node
committed
fix(ai): accept gpt-image-2 at 4K and name gateway routes in image errors
1 parent 964bab3 commit edd5fc1

6 files changed

Lines changed: 80 additions & 20 deletions

File tree

‎packages/ai/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -694,8 +694,8 @@ this table before swapping only the `model`:
694694

695695
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. "width × height"
696696
computes a size the provider accepts and fails when the request falls outside its limits. ¹ `Image.stream` on OpenAI
697-
generates one image. On OpenAI, `gpt-image-2` and later get a computed size; older models get the nearest of their
698-
fixed sizes. `providerOptions.size` replaces `aspect` and `resolution`, so setting both fails. Vercel AI Gateway
697+
generates one image. On OpenAI, `gpt-image-2` and later get a computed size, and `4K` gets their largest (3840x2160
698+
at 16:9); older models get the nearest of their fixed sizes. `providerOptions.size` replaces `aspect` and `resolution`, so setting both fails. Vercel AI Gateway
699699
forwards `outputFormat` but does not apply it, so the returned bytes keep the upstream model's own format. ² Workers AI
700700
`@cf/black-forest-labs/flux-1-schnell` takes no size, so `aspect` and `resolution` fail there. ³ Non-OpenAI gateway
701701
models (`bfl/flux-2-pro`) get a computed width × height, which the gateway passes through; models with a fixed list,

‎packages/ai/src/protocols/openai-images.ts‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ const sizes = (model: string): ImageSizing.Limits | ReadonlyArray<ImageSizing.Si
164164
{ width: 1024, height: 1536 },
165165
]
166166
if (id.startsWith("gpt-image-"))
167-
return { multiple: 16, maxEdge: 3840, minPixels: 655_360, maxPixels: 8_294_400, maxRatio: 3 }
167+
return { multiple: 16, maxEdge: 3840, minPixels: 655_360, maxPixels: 8_294_400, maxRatio: 3, upToMax: true }
168168
return { multiple: 16 }
169169
}
170170

@@ -190,7 +190,7 @@ export const pixelSize: OpenAIImageSizing = (request, route) => {
190190
).pipe(Effect.map((size) => (size === undefined ? undefined : { size: `${size.width}x${size.height}` })))
191191
}
192192

193-
const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
193+
const streamOptions = (request: MediaProtocol.Addressed<Request>, route: MediaProtocol.Identity) => {
194194
if (request.mode !== "stream") return Effect.succeed(undefined)
195195
if (request.n !== undefined && request.n > 1)
196196
return Effect.fail(
@@ -203,14 +203,15 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
203203

204204
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
205205

206-
const reference = (asset: Media.Asset, provider: ProviderID) =>
206+
const reference = (asset: Media.Asset, provider: ProviderID, route: MediaProtocol.Identity) =>
207207
ProviderShared.mediaReference(asset, provider, route.name).pipe(
208208
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
209209
)
210210

211211
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (
212212
request: MediaProtocol.Addressed<Request>,
213213
sizing: OpenAIImageSizing,
214+
named: MediaProtocol.Identity,
214215
) {
215216
const images = request.images ?? []
216217
const mask = request.providerOptions?.mask
@@ -223,8 +224,8 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (
223224
const fields = mergeJsonRecords(
224225
{
225226
n: request.n,
226-
...(yield* sizing(request, route)),
227-
...(yield* streamOptions(request)),
227+
...(yield* sizing(request, named)),
228+
...(yield* streamOptions(request, named)),
228229
},
229230
nativeOptions(request.providerOptions),
230231
request.http?.body,
@@ -247,8 +248,8 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (
247248
return MediaProtocol.multipart(form)
248249
}
249250

250-
const references = yield* Effect.forEach(images, (image) => reference(image, request.model.provider))
251-
const maskReference = mask === undefined ? undefined : yield* reference(mask, request.model.provider)
251+
const references = yield* Effect.forEach(images, (image) => reference(image, request.model.provider, named))
252+
const maskReference = mask === undefined ? undefined : yield* reference(mask, request.model.provider, named)
252253
return MediaProtocol.json(
253254
mergeJsonRecords(
254255
{
@@ -365,9 +366,10 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
365366
// 7. Protocol and route
366367
// ---------------------------------------------------------------------------
367368

368-
const makeProtocol = (sizing: OpenAIImageSizing) =>
369+
/** `named` labels request-building errors with the host that serves the route, such as OpenRouter. */
370+
const makeProtocol = (sizing: OpenAIImageSizing, named: MediaProtocol.Identity) =>
369371
MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
370-
body: { from: (request) => fromRequest(request, sizing) },
372+
body: { from: (request) => fromRequest(request, sizing, named) },
371373
frames: (bytes, context) =>
372374
context.request.mode === "stream"
373375
? Framing.sse.frame(bytes)
@@ -377,16 +379,28 @@ const makeProtocol = (sizing: OpenAIImageSizing) =>
377379
finish,
378380
})
379381

380-
export const protocol = makeProtocol(pixelSize)
382+
export const protocol = makeProtocol(pixelSize, route)
381383

382384
/**
383385
* `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. `sizing`
384-
* replaces how `aspect` and `resolution` reach the body.
386+
* replaces how `aspect` and `resolution` reach the body, and `name` is the route name request errors use.
385387
*/
386388
export const model = (
387-
input: MediaRoute.ModelInput & { readonly provider?: string; readonly sizing?: OpenAIImageSizing },
389+
input: MediaRoute.ModelInput & {
390+
readonly provider?: string
391+
readonly sizing?: OpenAIImageSizing
392+
readonly name?: string
393+
},
388394
) => {
389-
const base = input.sizing === undefined ? protocol : makeProtocol(input.sizing)
395+
const base =
396+
input.sizing === undefined && input.name === undefined
397+
? protocol
398+
: makeProtocol(
399+
input.sizing ?? pixelSize,
400+
input.name === undefined
401+
? route
402+
: MediaProtocol.identity({ id: route.id, name: input.name, provider: input.provider ?? route.provider }),
403+
)
390404
return ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
391405
{
392406
protocol: input.provider === undefined ? base : { ...base, provider: ProviderID.make(input.provider) },

‎packages/ai/src/protocols/utils/image-sizing.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ export interface Limits {
2222
readonly maxPixels?: number
2323
/** Long edge over short edge. */
2424
readonly maxRatio?: number
25+
/** A request above `maxEdge` or `maxPixels` renders at that maximum instead of failing. */
26+
readonly upToMax?: boolean
2527
}
2628

2729
export const isRequested = (shape: Shape) => shape.aspect !== undefined || shape.resolution !== undefined
@@ -72,8 +74,12 @@ export const nearest = (shape: Shape, sizes: ReadonlyArray<Size>): Size => {
7274
}
7375

7476
/** Free width and height that must honor the shape; a size that misses it fails. */
75-
export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limits) =>
76-
accept(route, shape, fit(shape, limits))
77+
export const exact = (route: MediaProtocol.Identity, shape: Shape, limits: Limits) => {
78+
const size = fit(shape, limits)
79+
// Scaled down to the provider's maximum, the size only has to keep the aspect.
80+
const capped = limits.upToMax === true && size.width * size.height < targetPixels(shape) && isRequested(shape)
81+
return accept(route, capped ? { aspect: shape.aspect ?? "1:1" } : shape, size)
82+
}
7783

7884
/** The size a provider renders for the shape, kept only when it is close enough; a size that misses it fails. */
7985
export const accept = (

‎packages/ai/src/providers/openrouter.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,14 @@ export const configure = (input: Config = {}) => {
226226
})
227227
const media = MediaRoute.deployment({ ...input, baseURL: input.baseURL ?? baseURL }, auth)
228228
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
229-
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
229+
OpenAIImages.model({
230+
...media,
231+
id: modelID,
232+
provider: id,
233+
providerOptions,
234+
sizing: imageSizing,
235+
name: "OpenRouter Images",
236+
})
230237
const decision = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
231238
decisionRoute.model<OpenRouterDecisionOptions>({
232239
id: modelID,

‎packages/ai/src/providers/vercel-ai-gateway.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,14 @@ export const configure = (input: Config = {}) => {
248248
}
249249
const media = MediaRoute.deployment({ ...input, baseURL: configured.endpoint.baseURL }, configured.auth)
250250
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
251-
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
251+
OpenAIImages.model({
252+
...media,
253+
id: modelID,
254+
provider: id,
255+
providerOptions,
256+
sizing: imageSizing,
257+
name: "Vercel AI Gateway Images",
258+
})
252259
const decisionRoute = OpenAIDecisions.route.with({
253260
provider: id,
254261
endpoint: { baseURL: configured.endpoint.baseURL },

‎packages/ai/test/image.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,32 @@ describe("Image", () => {
349349
}).pipe(Effect.provide(sent({ size: "2736x1536" }, OPENAI_STYLE))),
350350
)
351351

352+
for (const [aspect, size] of [
353+
["16:9", "3840x2160"],
354+
["1:1", "2880x2880"],
355+
] as const)
356+
it.effect(`sends gpt-image-2 its largest ${aspect} size for 4K`, () =>
357+
Image.generate({
358+
model: OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" }).image("gpt-image-2"),
359+
prompt,
360+
aspect,
361+
resolution: "4K",
362+
}).pipe(Effect.provide(sent({ size }, OPENAI_STYLE))),
363+
)
364+
365+
it.effect("names the gateway in Vercel AI Gateway sizing errors", () =>
366+
Effect.gen(function* () {
367+
const error = yield* Image.generate({
368+
model: VercelAIGateway.configure({ apiKey: "test" }).image("openai/gpt-image-1"),
369+
prompt,
370+
aspect: "16:9",
371+
}).pipe(Effect.flip)
372+
expect(error.message).toStartWith(
373+
"Vercel AI Gateway Images can't render 16:9; the closest size it takes is 1536x1024 (3:2)",
374+
)
375+
}).pipe(Effect.provide(layer(() => Effect.die("no request should be sent")))),
376+
)
377+
352378
it.effect("sends OpenRouter an aspect ratio and resolution tier instead of pixels", () =>
353379
Effect.gen(function* () {
354380
const openrouter = OpenRouter.configure({ apiKey: "test", baseURL: "https://openrouter.test/v1" })
@@ -361,7 +387,7 @@ describe("Image", () => {
361387
const error = yield* Image.generate({ model: openrouter.image("x/y"), prompt, resolution: "8K" }).pipe(
362388
Effect.flip,
363389
)
364-
expect(error.message).toContain("can't render 8K; the closest resolution it takes is 4K")
390+
expect(error.message).toStartWith("OpenRouter Images can't render 8K; the closest resolution it takes is 4K")
365391
}).pipe(
366392
Effect.provide(
367393
layer((input) => {

0 commit comments

Comments
 (0)