feat(api): add image edit markup overlays (#4210)

## Summary

Adds an optional markup image to guide image edits.

## Changes

- Provide a transparent PNG markup overlay when editing an image.
- Document that markup must match the first image dimensions and cannot
be combined with a mask.
- Keep the Realtime example responsive to Ctrl-C while waiting for
input, without crashing during interpreter shutdown.

On POSIX, the example polls stdin on the main thread instead of using a
background reader. Partial input, Unicode and subsequent lines are
preserved. Windows retains builtin input.

Validation: all eight example input/shutdown scenarios pass under both
Pydantic versions, plus180 repeated signal cases. Tests reject fatal
interpreter errors and invalid exit statuses; the original two-second
deadline is unchanged. The previous daemon-reader implementation fails
the strengthened tests. Formatting and lint/type checks passed. Public
CI is rerunning on this updated candidate.

Author: jbeckwith-oai

.castiron.stats.yml

@@ -1,7 +1,7 @@
 schema_version: 1
-generation_id: 093e3a9d-ca04-4b14-b524-cdefc6807fe1
-openapi_spec_hash: df8d45213a6894a955338882c36a1d86
-openapi_transformed_spec_hash: 050a1c6e01247b5af767e740b8494686
+generation_id: ca0448a4-c5bb-4dae-b9a8-ce0695616efc
+openapi_spec_hash: 8ca9f63bc016670f3fd9c8265f106c33
+openapi_transformed_spec_hash: 2cb5f40bd6fd564475d5b6ef285c9858
 config_hash: 8a684aec06907ff8ced7809753e16225
-codegen_sha: 514ca4482332258ac041a8d62a18c51ad31627e3
-codegen_hash: 319a11170cf75d4fb01e168ed07a00b7eecc51b23d5912ff6d9d8552968d127d
+codegen_sha: 52a6c414b9565e2a8a3f89f87a4df1a5b20deaea
+codegen_hash: 6171d1d87772171c445edcacc04ee274f4d10497fe9da70810d5c778ab57c394

api_reference/openapi.transformed.yml

@@ -44419,6 +44419,8 @@ components:
           example: high
           description: |
             The quality of the image that will be generated for GPT image models. The GPT image models support `low`, `medium`, and `high`. `gpt-image-2.5-sunburst` and `gpt-image-2.5-flare`, including their `2026-09-08` snapshots, also support `xhigh` and `max`. Defaults to `auto`.
+        markup:
+          $ref: '#/components/schemas/markup'
       required:
       - prompt
       - image
@@ -46985,6 +46987,9 @@ components:
           description: Stream partial image results as events.
         partial_images:
           $ref: '#/components/schemas/PartialImages'
+        markup:
+          $ref: '#/components/schemas/ImageRefParam'
+          description: A transparent PNG overlaid on the first image, referenced by file ID, HTTPS URL, or base64 data URL. Must match that image's dimensions after orientation and contain transparent pixels. Supports GPT Image 2.5 and 2.6. Cannot be combined with mask.
       required:
       - images
       - prompt
@@ -111105,6 +111110,10 @@ components:
       - async
       additionalProperties: false
       description: The effective settings for compacting the agent's context.
+    markup:
+      type: string
+      format: binary
+      description: A transparent PNG overlaid on the first image to guide the edit. Must match that image's dimensions after orientation and contain transparent pixels. Supports GPT Image 2.5 and 2.6. Cannot be combined with mask.
     SessionSpendControlResource:
       type: object
       properties:

examples/realtime/realtime.py

@@ -1,4 +1,9 @@
 #!/usr/bin/env -S uv run python
+import os
+import sys
+import codecs
+import select
+
 from openai import OpenAI
 
 # Azure OpenAI Realtime Docs
@@ -8,6 +13,33 @@
 # Entra ID auth: https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity
 
 
+def read_input(prompt: str) -> str:
+    if sys.platform == "win32":
+        # Windows console input does not support select().
+        return input(prompt)
+
+    # SIGINT may reach a WebSocket thread. Poll on the main thread so Python
+    # can run its signal handler, without leaving a stdin reader at shutdown.
+    print(prompt, end="", flush=True)
+    decoder = codecs.getincrementaldecoder(sys.stdin.encoding)(errors=sys.stdin.errors or "strict")
+    characters: list[str] = []
+    while True:
+        readable, _, _ = select.select([sys.stdin], [], [], 0.1)
+        if not readable:
+            continue
+        # Do not read ahead into the next prompt or block on a partial line.
+        data = os.read(sys.stdin.fileno(), 1)
+        character = decoder.decode(data, final=not data)
+        if character == "\n":
+            return "".join(characters).removesuffix("\r")
+        characters.append(character)
+        if not data:
+            text = "".join(characters)
+            if not text:
+                raise EOFError
+            return text
+
+
 def main() -> None:
     """The following example demonstrates how to configure OpenAI to use the Realtime API.
     For an audio example, see push_to_talk_app.py and update the client and model parameter accordingly.
@@ -28,7 +60,7 @@ def main() -> None:
             }
         )
         while True:
-            user_input = input("Enter a message: ")
+            user_input = read_input("Enter a message: ")
             if user_input == "q":
                 break
 

scripts/castiron/generated-files.json

@@ -547,7 +547,7 @@
     },
     "src/openai/resources/images.py": {
       "mode": "100644",
-      "sha256": "f8f5376eeda7b45a90179eb287066447fce19ed761d313de5ab4c59df620b659"
+      "sha256": "211377871db5f2d21633f93122a131023174ea093828c7a957cd0ece49a03b76"
     },
     "src/openai/resources/live/__init__.py": {
       "mode": "100644",
@@ -4543,7 +4543,7 @@
     },
     "src/openai/types/image_edit_params.py": {
       "mode": "100644",
-      "sha256": "e45d0c075029142ad45cbb557962a6e065d299687449b97405390a9534d5b7bf"
+      "sha256": "a08fc3c5d22c14b3eb969fa91891e3d3dd7088f9e46749aa5bd59c4e5c558df3"
     },
     "src/openai/types/image_edit_partial_image_event.py": {
       "mode": "100644",
@@ -7999,7 +7999,7 @@
     },
     "tests/api_resources/test_images.py": {
       "mode": "100644",
-      "sha256": "ebd279e1c7d8c8bd3c8148d1aadbeb7dc1ba43a6e2ee7396653e37563e743f3b"
+      "sha256": "ca9dae5fdc1d35f7a7d1dd2356736628b24d3f11c28e4750f7cea920a6c9804c"
     },
     "tests/api_resources/test_live.py": {
       "mode": "100644",

src/openai/resources/images.py

@@ -144,6 +144,7 @@ def edit(
         prompt: str,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -197,6 +198,10 @@ def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -300,6 +305,7 @@ def edit(
         stream: Literal[True],
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -356,6 +362,10 @@ def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -455,6 +465,7 @@ def edit(
         stream: bool,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -511,6 +522,10 @@ def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -609,6 +624,7 @@ def edit(
         prompt: str,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -635,6 +651,7 @@ def edit(
                 "prompt": prompt,
                 "background": background,
                 "input_fidelity": input_fidelity,
+                "markup": markup,
                 "mask": mask,
                 "model": model,
                 "n": n,
@@ -648,9 +665,11 @@ def edit(
                 "stream": stream,
                 "user": user,
             },
-            [["image"], ["image", "<array>"], ["mask"]],
+            [["image"], ["image", "<array>"], ["markup"], ["mask"]],
+        )
+        files = extract_files(
+            cast(Mapping[str, object], body), paths=[["image"], ["image", "<array>"], ["markup"], ["mask"]]
         )
-        files = extract_files(cast(Mapping[str, object], body), paths=[["image"], ["image", "<array>"], ["mask"]])
         # It should be noted that the actual Content-Type header that will be
         # sent to the server will contain a `boundary` parameter, e.g.
         # multipart/form-data; boundary=---abc--
@@ -1301,6 +1320,7 @@ async def edit(
         prompt: str,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -1354,6 +1374,10 @@ async def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -1457,6 +1481,7 @@ async def edit(
         stream: Literal[True],
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -1513,6 +1538,10 @@ async def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -1612,6 +1641,7 @@ async def edit(
         stream: bool,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -1668,6 +1698,10 @@ async def edit(
               `gpt-image-1.5`; `gpt-image-1-mini` supports only `low`. For `gpt-image-2`, omit
               this parameter. Defaults to `low` on supported models.
 
+          markup: A transparent PNG overlaid on the first image to guide the edit. Must match that
+              image's dimensions after orientation and contain transparent pixels. Supports
+              GPT Image 2.5 and 2.6. Cannot be combined with mask.
+
           mask: An additional image whose fully transparent areas (e.g. where alpha is zero)
               indicate where `image` should be edited. If there are multiple images provided,
               the mask will be applied on the first image. Must be a valid PNG file, less than
@@ -1766,6 +1800,7 @@ async def edit(
         prompt: str,
         background: Optional[Literal["transparent", "opaque", "auto"]] | Omit = omit,
         input_fidelity: Optional[Literal["high", "low"]] | Omit = omit,
+        markup: FileTypes | Omit = omit,
         mask: FileTypes | Omit = omit,
         model: Union[str, ImageModel, None] | Omit = omit,
         n: Optional[int] | Omit = omit,
@@ -1792,6 +1827,7 @@ async def edit(
                 "prompt": prompt,
                 "background": background,
                 "input_fidelity": input_fidelity,
+                "markup": markup,
                 "mask": mask,
                 "model": model,
                 "n": n,
@@ -1805,9 +1841,11 @@ async def edit(
                 "stream": stream,
                 "user": user,
             },
-            [["image"], ["image", "<array>"], ["mask"]],
+            [["image"], ["image", "<array>"], ["markup"], ["mask"]],
+        )
+        files = extract_files(
+            cast(Mapping[str, object], body), paths=[["image"], ["image", "<array>"], ["markup"], ["mask"]]
         )
-        files = extract_files(cast(Mapping[str, object], body), paths=[["image"], ["image", "<array>"], ["mask"]])
         # It should be noted that the actual Content-Type header that will be
         # sent to the server will contain a `boundary` parameter, e.g.
         # multipart/form-data; boundary=---abc--

src/openai/types/image_edit_params.py

@@ -51,6 +51,13 @@ class ImageEditParamsBase(TypedDict, total=False):
     this parameter. Defaults to `low` on supported models.
     """
 
+    markup: FileTypes
+    """A transparent PNG overlaid on the first image to guide the edit.
+
+    Must match that image's dimensions after orientation and contain transparent
+    pixels. Supports GPT Image 2.5 and 2.6. Cannot be combined with mask.
+    """
+
     mask: FileTypes
     """An additional image whose fully transparent areas (e.g.
 

tests/api_resources/test_images.py

@@ -81,6 +81,7 @@ def test_method_edit_with_all_params_overload_1(self, client: OpenAI) -> None:
             prompt="A cute baby sea otter wearing a beret",
             background="transparent",
             input_fidelity="high",
+            markup=b"Example data",
             mask=b"Example data",
             model="gpt-image-1.5",
             n=1,
@@ -139,6 +140,7 @@ def test_method_edit_with_all_params_overload_2(self, client: OpenAI) -> None:
             stream=True,
             background="transparent",
             input_fidelity="high",
+            markup=b"Example data",
             mask=b"Example data",
             model="gpt-image-1.5",
             n=1,
@@ -360,6 +362,7 @@ async def test_method_edit_with_all_params_overload_1(self, async_client: AsyncO
             prompt="A cute baby sea otter wearing a beret",
             background="transparent",
             input_fidelity="high",
+            markup=b"Example data",
             mask=b"Example data",
             model="gpt-image-1.5",
             n=1,
@@ -418,6 +421,7 @@ async def test_method_edit_with_all_params_overload_2(self, async_client: AsyncO
             stream=True,
             background="transparent",
             input_fidelity="high",
+            markup=b"Example data",
             mask=b"Example data",
             model="gpt-image-1.5",
             n=1,