feat(scrape): add highlights format to POST /web/scrape (#1210)

Stainless-Generated-From: 76b2be5

Author: stlc-bot

src/context/dev/resources/web.py

@@ -391,6 +391,7 @@ def scrape(
         *,
         formats: web_scrape_params.Formats,
         url: str,
+        highlights_params: web_scrape_params.HighlightsParams | Omit = omit,
         image_params: web_scrape_params.ImageParams | Omit = omit,
         json_params: web_scrape_params.JsonParams | Omit = omit,
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
@@ -412,20 +413,24 @@ def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can
-        come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only
-        requests use the existing fast acquisition path. One credit per request,
-        including cache hits and missing pages, or two with browser actions; JSON
-        extraction adds four credits and runs an LLM over the page Markdown on every
-        request that has text to extract; PDF OCR adds one credit per recovered page on
-        fresh extraction. Original response bytes and screenshots are limited to 20 MiB
-        each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+        is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached
+        outputs can come from different visits within maxAgeMs; use 0 for a fresh
+        capture. HTML-only requests use the existing fast acquisition path. Highlights
+        return the plain-text passages most relevant to highlightsParams.query. One
+        credit per request, including cache hits and missing pages, or two with browser
+        actions; highlights add 3 credits when passages are returned; JSON extraction
+        adds four credits and runs an LLM over the page Markdown on every request that
+        has text to extract; PDF OCR adds one credit per recovered page on fresh
+        extraction. Original response bytes and screenshots are limited to 20 MiB each,
+        screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
 
           url: The URL to scrape.
 
+          highlights_params: Highlight options. Requires formats.highlights: true.
+
           image_params: Image options. Requires formats.images: true.
 
           json_params: Required when formats.json is true.
@@ -455,7 +460,8 @@ def scrape(
               when using return-partial.
 
           zdr: Zero data retention. Bypasses caches and uploads; excludes request/response
-              content and tags from logs. Must be enabled for your organization.
+              content and tags from logs. Must be enabled for your organization. Not available
+              with the highlights output.
 
           extra_headers: Send extra headers
 
@@ -471,6 +477,7 @@ def scrape(
                 {
                     "formats": formats,
                     "url": url,
+                    "highlights_params": highlights_params,
                     "image_params": image_params,
                     "json_params": json_params,
                     "markdown_params": markdown_params,
@@ -1874,6 +1881,7 @@ async def scrape(
         *,
         formats: web_scrape_params.Formats,
         url: str,
+        highlights_params: web_scrape_params.HighlightsParams | Omit = omit,
         image_params: web_scrape_params.ImageParams | Omit = omit,
         json_params: web_scrape_params.JsonParams | Omit = omit,
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
@@ -1895,20 +1903,24 @@ async def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can
-        come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only
-        requests use the existing fast acquisition path. One credit per request,
-        including cache hits and missing pages, or two with browser actions; JSON
-        extraction adds four credits and runs an LLM over the page Markdown on every
-        request that has text to extract; PDF OCR adds one credit per recovered page on
-        fresh extraction. Original response bytes and screenshots are limited to 20 MiB
-        each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+        is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached
+        outputs can come from different visits within maxAgeMs; use 0 for a fresh
+        capture. HTML-only requests use the existing fast acquisition path. Highlights
+        return the plain-text passages most relevant to highlightsParams.query. One
+        credit per request, including cache hits and missing pages, or two with browser
+        actions; highlights add 3 credits when passages are returned; JSON extraction
+        adds four credits and runs an LLM over the page Markdown on every request that
+        has text to extract; PDF OCR adds one credit per recovered page on fresh
+        extraction. Original response bytes and screenshots are limited to 20 MiB each,
+        screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
 
           url: The URL to scrape.
 
+          highlights_params: Highlight options. Requires formats.highlights: true.
+
           image_params: Image options. Requires formats.images: true.
 
           json_params: Required when formats.json is true.
@@ -1938,7 +1950,8 @@ async def scrape(
               when using return-partial.
 
           zdr: Zero data retention. Bypasses caches and uploads; excludes request/response
-              content and tags from logs. Must be enabled for your organization.
+              content and tags from logs. Must be enabled for your organization. Not available
+              with the highlights output.
 
           extra_headers: Send extra headers
 
@@ -1954,6 +1967,7 @@ async def scrape(
                 {
                     "formats": formats,
                     "url": url,
+                    "highlights_params": highlights_params,
                     "image_params": image_params,
                     "json_params": json_params,
                     "markdown_params": markdown_params,

src/context/dev/types/web_scrape_params.py

@@ -11,6 +11,7 @@
 __all__ = [
     "WebScrapeParams",
     "Formats",
+    "HighlightsParams",
     "ImageParams",
     "JsonParams",
     "MarkdownParams",
@@ -41,6 +42,9 @@ class WebScrapeParams(TypedDict, total=False):
     url: Required[str]
     """The URL to scrape."""
 
+    highlights_params: Annotated[HighlightsParams, PropertyInfo(alias="highlightsParams")]
+    """Highlight options. Requires formats.highlights: true."""
+
     image_params: Annotated[ImageParams, PropertyInfo(alias="imageParams")]
     """Image options. Requires formats.images: true."""
 
@@ -88,7 +92,8 @@ class WebScrapeParams(TypedDict, total=False):
     """Zero data retention.
 
     Bypasses caches and uploads; excludes request/response content and tags from
-    logs. Must be enabled for your organization.
+    logs. Must be enabled for your organization. Not available with the highlights
+    output.
     """
 
 
@@ -98,6 +103,13 @@ class Formats(TypedDict, total=False):
     bytes: bool
     """The original HTTP response body."""
 
+    highlights: bool
+    """
+    Plain-text passages from the page that are most relevant to
+    highlightsParams.query, each prefixed with its section heading. Adds 3 credits.
+    Not available with zdr enabled.
+    """
+
     html: bool
     """Rendered HTML."""
 
@@ -122,6 +134,16 @@ class Formats(TypedDict, total=False):
     """An inline image of the page."""
 
 
+class HighlightsParams(TypedDict, total=False):
+    """Highlight options. Requires formats.highlights: true."""
+
+    query: Required[str]
+    """The question or topic to find passages for."""
+
+    max_characters: Annotated[int, PropertyInfo(alias="maxCharacters")]
+    """Maximum combined length of the returned passages, in characters."""
+
+
 class ImageParams(TypedDict, total=False):
     """Image options. Requires formats.images: true."""
 

src/context/dev/types/web_scrape_response.py

@@ -12,6 +12,7 @@
     "Bytes",
     "BytesData",
     "CacheMetadata",
+    "Highlights",
     "HTML",
     "Images",
     "ImagesData",
@@ -63,6 +64,16 @@ class CacheMetadata(BaseModel):
     """
 
 
+class Highlights(BaseModel):
+    """
+    Plain-text passages relevant to highlightsParams.query, in page order, each prefixed with its section heading in square brackets. Empty when the page has no text.
+    """
+
+    data: Optional[List[str]] = None
+
+    requested: bool
+
+
 class HTML(BaseModel):
     """Rendered HTML after content filters."""
 
@@ -243,6 +254,13 @@ class WebScrapeResponse(BaseModel):
     to the output was a hit; age_ms is the oldest contributing hit.
     """
 
+    highlights: Highlights
+    """
+    Plain-text passages relevant to highlightsParams.query, in page order, each
+    prefixed with its section heading in square brackets. Empty when the page has no
+    text.
+    """
+
     html: HTML
     """Rendered HTML after content filters."""
 

tests/api_resources/test_web.py

@@ -242,6 +242,7 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
         web = client.web.scrape(
             formats={
                 "bytes": True,
+                "highlights": True,
                 "html": True,
                 "images": True,
                 "json": True,
@@ -250,6 +251,10 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
                 "screenshot": True,
             },
             url="https://example.com",
+            highlights_params={
+                "query": "x",
+                "max_characters": 100,
+            },
             image_params={
                 "dedupe": "none",
                 "enrich": ["dimensions"],
@@ -772,6 +777,7 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
         web = await async_client.web.scrape(
             formats={
                 "bytes": True,
+                "highlights": True,
                 "html": True,
                 "images": True,
                 "json": True,
@@ -780,6 +786,10 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
                 "screenshot": True,
             },
             url="https://example.com",
+            highlights_params={
+                "query": "x",
+                "max_characters": 100,
+            },
             image_params={
                 "dedupe": "none",
                 "enrich": ["dimensions"],