fix(scrape): preserve successful formats when other outputs fail (#1263)

Stainless-Generated-From: b3de6c4

Author: stlc-bot

src/context/dev/resources/web.py

@@ -418,14 +418,20 @@ def scrape(
         extraction. Cached outputs can come from different visits within maxAgeMs; use 0
         for a fresh capture. HTML-only requests use the existing fast acquisition path.
         Highlights return the plain-text passages most relevant to
-        highlightsParams.query. One credit per request, including cache hits and missing
-        pages, or two with browser actions; highlights add 3 credits when passages are
-        returned; JSON extraction adds four credits and runs an LLM over the page
-        Markdown on every request that has text to extract; PDF OCR adds one credit per
-        recovered page on fresh extraction; the product output adds one credit, plus six
-        more when the specialized model is used. Original response bytes and screenshots
-        are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
-        browser capture to 60 MiB.
+        highlightsParams.query. Requests with at least one successful output cost one
+        base credit, including cache hits, or two with browser actions. All-failed
+        responses are unbilled except missing pages, which retain the base price and the
+        one-credit product charge when product was requested. Highlights add 3 credits
+        when passages are returned. JSON extraction runs an LLM over nonempty page
+        Markdown and adds four credits only when its result is returned successfully.
+        PDF OCR adds one credit per recovered page on fresh extraction. Product adds one
+        credit when its successful result is returned, plus six if that result used the
+        specialized model. Original response bytes and screenshots are limited to 20 MiB
+        each, screenshots to 40 megapixels, and the combined response to 60 MiB. An
+        oversized output has success: false and data: null. If the combined response
+        exceeds its limit, the largest outputs are marked failed until the remaining
+        outputs fit. Valid captured pieces may still be cached when omitted to meet the
+        response size limit.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
@@ -457,12 +463,16 @@ def scrape(
           tags: Labels for tracking request usage. Not retained when zdr is enabled.
 
           timeout_opts: Total deadline, including navigation, actions, waiting, and all outputs.
-              Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
-              the current page state and return captured images if image processing cannot
-              finish before the deadline; these responses set isPartial and are not cached.
-              Every requested format must still be available. Fixed waits must fit before a
-              response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
-              when using return-partial.
+              Defaults to 60000 milliseconds with behavior fail. Individual outputs have
+              internal deadlines that reserve time to return completed outputs; timed-out
+              outputs have success: false and data: null under either behavior. The overall
+              request deadline remains enforced: fail returns an error if that deadline is
+              reached. Use return-partial to allow the current page state and available
+              outputs when the page is still loading. Partial responses set isPartial. Failed
+              retrievals and incomplete captures are not cached; valid captured pieces may be
+              cached independently. Fixed waits must fit before a response reserve of up to
+              5000 milliseconds (at most one quarter of the timeout) when using
+              return-partial.
 
           zdr: Zero data retention. Bypasses caches and uploads; excludes request/response
               content and tags from logs. Must be enabled for your organization.
@@ -1913,14 +1923,20 @@ async def scrape(
         extraction. Cached outputs can come from different visits within maxAgeMs; use 0
         for a fresh capture. HTML-only requests use the existing fast acquisition path.
         Highlights return the plain-text passages most relevant to
-        highlightsParams.query. One credit per request, including cache hits and missing
-        pages, or two with browser actions; highlights add 3 credits when passages are
-        returned; JSON extraction adds four credits and runs an LLM over the page
-        Markdown on every request that has text to extract; PDF OCR adds one credit per
-        recovered page on fresh extraction; the product output adds one credit, plus six
-        more when the specialized model is used. Original response bytes and screenshots
-        are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
-        browser capture to 60 MiB.
+        highlightsParams.query. Requests with at least one successful output cost one
+        base credit, including cache hits, or two with browser actions. All-failed
+        responses are unbilled except missing pages, which retain the base price and the
+        one-credit product charge when product was requested. Highlights add 3 credits
+        when passages are returned. JSON extraction runs an LLM over nonempty page
+        Markdown and adds four credits only when its result is returned successfully.
+        PDF OCR adds one credit per recovered page on fresh extraction. Product adds one
+        credit when its successful result is returned, plus six if that result used the
+        specialized model. Original response bytes and screenshots are limited to 20 MiB
+        each, screenshots to 40 megapixels, and the combined response to 60 MiB. An
+        oversized output has success: false and data: null. If the combined response
+        exceeds its limit, the largest outputs are marked failed until the remaining
+        outputs fit. Valid captured pieces may still be cached when omitted to meet the
+        response size limit.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
@@ -1952,12 +1968,16 @@ async def scrape(
           tags: Labels for tracking request usage. Not retained when zdr is enabled.
 
           timeout_opts: Total deadline, including navigation, actions, waiting, and all outputs.
-              Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
-              the current page state and return captured images if image processing cannot
-              finish before the deadline; these responses set isPartial and are not cached.
-              Every requested format must still be available. Fixed waits must fit before a
-              response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
-              when using return-partial.
+              Defaults to 60000 milliseconds with behavior fail. Individual outputs have
+              internal deadlines that reserve time to return completed outputs; timed-out
+              outputs have success: false and data: null under either behavior. The overall
+              request deadline remains enforced: fail returns an error if that deadline is
+              reached. Use return-partial to allow the current page state and available
+              outputs when the page is still loading. Partial responses set isPartial. Failed
+              retrievals and incomplete captures are not cached; valid captured pieces may be
+              cached independently. Fixed waits must fit before a response reserve of up to
+              5000 milliseconds (at most one quarter of the timeout) when using
+              return-partial.
 
           zdr: Zero data retention. Bypasses caches and uploads; excludes request/response
               content and tags from logs. Must be enabled for your organization.

src/context/dev/types/web_scrape_params.py

@@ -84,12 +84,16 @@ class WebScrapeParams(TypedDict, total=False):
     timeout_opts: Annotated[TimeoutOpts, PropertyInfo(alias="timeoutOpts")]
     """Total deadline, including navigation, actions, waiting, and all outputs.
 
-    Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
-    the current page state and return captured images if image processing cannot
-    finish before the deadline; these responses set isPartial and are not cached.
-    Every requested format must still be available. Fixed waits must fit before a
-    response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
-    when using return-partial.
+    Defaults to 60000 milliseconds with behavior fail. Individual outputs have
+    internal deadlines that reserve time to return completed outputs; timed-out
+    outputs have success: false and data: null under either behavior. The overall
+    request deadline remains enforced: fail returns an error if that deadline is
+    reached. Use return-partial to allow the current page state and available
+    outputs when the page is still loading. Partial responses set isPartial. Failed
+    retrievals and incomplete captures are not cached; valid captured pieces may be
+    cached independently. Fixed waits must fit before a response reserve of up to
+    5000 milliseconds (at most one quarter of the timeout) when using
+    return-partial.
     """
 
     zdr: Literal["enabled", "disabled"]
@@ -109,7 +113,7 @@ class Formats(TypedDict, total=False):
     highlights: bool
     """
     Relevant passages for your question or topic, with headings included when needed
-    for context. Adds 3 credits.
+    for context. Adds 3 credits when passages are returned.
     """
 
     html: bool
@@ -119,7 +123,10 @@ class Formats(TypedDict, total=False):
     """Images found on the page."""
 
     json: bool
-    """Page data extracted using your schema. Adds 4 credits."""
+    """Page data extracted using your schema.
+
+    Adds 4 credits when extraction succeeds and its result is returned.
+    """
 
     markdown: bool
     """Page content as Markdown."""
@@ -128,7 +135,11 @@ class Formats(TypedDict, total=False):
     """Fields selected by parseParams.rules."""
 
     product: bool
-    """Product details such as name, price, and availability. Adds 1 credit."""
+    """Product details such as name, price, and availability.
+
+    Adds 1 credit when its successful result is returned or the target page is
+    missing.
+    """
 
     screenshot: bool
     """An inline image of the page."""
@@ -216,9 +227,10 @@ class ProductParams(TypedDict, total=False):
     use_ai_fallback: Annotated[bool, PropertyInfo(alias="useAIFallback")]
     """
     Extract the product with a specialized model when the page has no structured
-    product data. Adds six credits when the model returns a verdict. If the fallback
-    fails, returns a partial response with the deterministic result and no fallback
-    charge. Request deadlines and client disconnects still apply.
+    product data. Adds six credits when the model verdict is returned successfully.
+    If the fallback fails, the product output has success: false and data: null with
+    no fallback charge; other outputs remain available. Request deadlines and client
+    disconnects still apply.
     """
 
 
@@ -387,7 +399,7 @@ class SharedParams(TypedDict, total=False):
 class TimeoutOpts(TypedDict, total=False):
     """Total deadline, including navigation, actions, waiting, and all outputs.
 
-    Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture the current page state and return captured images if image processing cannot finish before the deadline; these responses set isPartial and are not cached. Every requested format must still be available. Fixed waits must fit before a response reserve of up to 5000 milliseconds (at most one quarter of the timeout) when using return-partial.
+    Defaults to 60000 milliseconds with behavior fail. Individual outputs have internal deadlines that reserve time to return completed outputs; timed-out outputs have success: false and data: null under either behavior. The overall request deadline remains enforced: fail returns an error if that deadline is reached. Use return-partial to allow the current page state and available outputs when the page is still loading. Partial responses set isPartial. Failed retrievals and incomplete captures are not cached; valid captured pieces may be cached independently. Fixed waits must fit before a response reserve of up to 5000 milliseconds (at most one quarter of the timeout) when using return-partial.
     """
 
     milliseconds: Required[int]

src/context/dev/types/web_scrape_response.py

@@ -51,6 +51,9 @@ class Bytes(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class CacheMetadata(BaseModel):
     """Cache outcome for this response.
@@ -78,6 +81,9 @@ class Highlights(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class HTML(BaseModel):
     """Rendered HTML after content filters."""
@@ -86,6 +92,9 @@ class HTML(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class ImagesData(BaseModel):
     alt: Optional[str] = None
@@ -116,6 +125,9 @@ class Images(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class Json(BaseModel):
     """Page data extracted using your schema."""
@@ -124,6 +136,9 @@ class Json(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class Markdown(BaseModel):
     """Markdown after content filters."""
@@ -132,6 +147,9 @@ class Markdown(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class MetadataAlternate(BaseModel):
     href: str
@@ -223,6 +241,9 @@ class Parsed(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class ProductDataProductVariant(BaseModel):
     attributes: Dict[str, str]
@@ -315,6 +336,9 @@ class Product(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class Screenshot(BaseModel):
     """An image data URL. Use directly as an image src."""
@@ -323,6 +347,9 @@ class Screenshot(BaseModel):
 
     requested: bool
 
+    success: Optional[bool] = None
+    """True when retrieved, false when retrieval failed, and null when not requested."""
+
 
 class KeyMetadata(BaseModel):
     """Credit usage, included whenever a valid API key is provided."""
@@ -390,10 +417,11 @@ class WebScrapeResponse(BaseModel):
 
     is_partial: Optional[Literal[True]] = FieldInfo(alias="isPartial", default=None)
     """
-    Present when return-partial captures a page that is still loading, returns
-    images before image processing finishes, or cuts product AI extraction short.
-    Also present if the optional product AI fallback fails. Partial responses are
-    not cached.
+    Present when a requested output fails, capture returns a page that is still
+    loading, images return before processing finishes, or the optional product AI
+    fallback fails or is cut short. Check each output's success field for its
+    result. Valid captured pieces may be cached independently; failed retrievals and
+    incomplete captures are not cached.
     """
 
     key_metadata: Optional[KeyMetadata] = None