Skip to content

Commit 65fa0fe

Browse files
author
stlc-bot
committed
feat(scrape): support waitForMs for byte downloads (#1170)
Stainless-Generated-From: 08a3b13ed89a10fe51cbddd6ace4409087095d99
1 parent fc642b8 commit 65fa0fe

3 files changed

Lines changed: 43 additions & 9 deletions

File tree

‎src/context/dev/resources/web.py‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1732,6 +1732,7 @@ def web_scrape_bytes(
17321732
headers: Dict[str, str] | Omit = omit,
17331733
tags: SequenceNotStr[str] | Omit = omit,
17341734
timeout_opts: web_web_scrape_bytes_params.TimeoutOpts | Omit = omit,
1735+
wait_for_ms: Optional[int] | Omit = omit,
17351736
zdr: Literal["enabled", "disabled"] | Omit = omit,
17361737
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
17371738
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -1742,10 +1743,12 @@ def web_scrape_bytes(
17421743
) -> WebWebScrapeBytesResponse:
17431744
"""Downloads a resource and returns its bytes as base64.
17441745
1745-
Supports images, PDFs,
1746-
HTML pages, and any other content type without image conversion, text
1747-
extraction, or character-encoding changes. HTTP compression is decoded before
1748-
base64 encoding. HTML is the original HTTP response; JavaScript is not rendered.
1746+
Without waitForMs, returns
1747+
the original HTTP response without image conversion, text extraction, or
1748+
character-encoding changes. HTTP compression is decoded before base64 encoding.
1749+
Supply waitForMs to render HTML with JavaScript in the browser and return the
1750+
resulting HTML as UTF-8 bytes after the wait. Non-HTML resources, including
1751+
images and PDFs, keep their original bytes and do not incur a browser wait.
17491752
Follows public redirects and retries failed downloads through ISP and
17501753
residential proxies, with a direct fallback. When country is specified, only a
17511754
residential proxy in that country is used. Supply headers such as Referer for
@@ -1772,6 +1775,13 @@ def web_scrape_bytes(
17721775
timeoutOpts[milliseconds]=30000&timeoutOpts[behavior]=fail or a JSON-encoded
17731776
timeoutOpts object.
17741777
1778+
wait_for_ms: Optional browser wait time after initial page load, in milliseconds (0–30000; 0
1779+
uses 500). When supplied, HTML is rendered with JavaScript and returned as UTF-8
1780+
bytes. Other resources keep their original bytes without a browser wait. Omit to
1781+
download the original HTTP response. When combined with timeoutOpts,
1782+
timeoutOpts.milliseconds must be at least waitForMs + 10000 ms; a shorter
1783+
deadline is rejected with 400 TIMEOUT_TOO_SHORT_FOR_WAIT.
1784+
17751785
zdr: Set to enabled to bypass shared caches and omit request and response content
17761786
from retained usage logs. Asset uploads are skipped, so hosted image URLs are
17771787
omitted. Requires zero data retention to be enabled for your organization
@@ -1800,6 +1810,7 @@ def web_scrape_bytes(
18001810
"headers": headers,
18011811
"tags": tags,
18021812
"timeout_opts": timeout_opts,
1813+
"wait_for_ms": wait_for_ms,
18031814
"zdr": zdr,
18041815
},
18051816
web_web_scrape_bytes_params.WebWebScrapeBytesParams,
@@ -4743,6 +4754,7 @@ async def web_scrape_bytes(
47434754
headers: Dict[str, str] | Omit = omit,
47444755
tags: SequenceNotStr[str] | Omit = omit,
47454756
timeout_opts: web_web_scrape_bytes_params.TimeoutOpts | Omit = omit,
4757+
wait_for_ms: Optional[int] | Omit = omit,
47464758
zdr: Literal["enabled", "disabled"] | Omit = omit,
47474759
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
47484760
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -4753,10 +4765,12 @@ async def web_scrape_bytes(
47534765
) -> WebWebScrapeBytesResponse:
47544766
"""Downloads a resource and returns its bytes as base64.
47554767
4756-
Supports images, PDFs,
4757-
HTML pages, and any other content type without image conversion, text
4758-
extraction, or character-encoding changes. HTTP compression is decoded before
4759-
base64 encoding. HTML is the original HTTP response; JavaScript is not rendered.
4768+
Without waitForMs, returns
4769+
the original HTTP response without image conversion, text extraction, or
4770+
character-encoding changes. HTTP compression is decoded before base64 encoding.
4771+
Supply waitForMs to render HTML with JavaScript in the browser and return the
4772+
resulting HTML as UTF-8 bytes after the wait. Non-HTML resources, including
4773+
images and PDFs, keep their original bytes and do not incur a browser wait.
47604774
Follows public redirects and retries failed downloads through ISP and
47614775
residential proxies, with a direct fallback. When country is specified, only a
47624776
residential proxy in that country is used. Supply headers such as Referer for
@@ -4783,6 +4797,13 @@ async def web_scrape_bytes(
47834797
timeoutOpts[milliseconds]=30000&timeoutOpts[behavior]=fail or a JSON-encoded
47844798
timeoutOpts object.
47854799
4800+
wait_for_ms: Optional browser wait time after initial page load, in milliseconds (0–30000; 0
4801+
uses 500). When supplied, HTML is rendered with JavaScript and returned as UTF-8
4802+
bytes. Other resources keep their original bytes without a browser wait. Omit to
4803+
download the original HTTP response. When combined with timeoutOpts,
4804+
timeoutOpts.milliseconds must be at least waitForMs + 10000 ms; a shorter
4805+
deadline is rejected with 400 TIMEOUT_TOO_SHORT_FOR_WAIT.
4806+
47864807
zdr: Set to enabled to bypass shared caches and omit request and response content
47874808
from retained usage logs. Asset uploads are skipped, so hosted image URLs are
47884809
omitted. Requires zero data retention to be enabled for your organization
@@ -4811,6 +4832,7 @@ async def web_scrape_bytes(
48114832
"headers": headers,
48124833
"tags": tags,
48134834
"timeout_opts": timeout_opts,
4835+
"wait_for_ms": wait_for_ms,
48144836
"zdr": zdr,
48154837
},
48164838
web_web_scrape_bytes_params.WebWebScrapeBytesParams,

‎src/context/dev/types/web_web_scrape_bytes_params.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from __future__ import annotations
44

5-
from typing import Dict
5+
from typing import Dict, Optional
66
from typing_extensions import Literal, Required, Annotated, TypedDict
77

88
from .._types import SequenceNotStr
@@ -248,6 +248,16 @@ class WebWebScrapeBytesParams(TypedDict, total=False):
248248
or a JSON-encoded timeoutOpts object.
249249
"""
250250

251+
wait_for_ms: Annotated[Optional[int], PropertyInfo(alias="waitForMs")]
252+
"""
253+
Optional browser wait time after initial page load, in milliseconds (0–30000; 0
254+
uses 500). When supplied, HTML is rendered with JavaScript and returned as UTF-8
255+
bytes. Other resources keep their original bytes without a browser wait. Omit to
256+
download the original HTTP response. When combined with timeoutOpts,
257+
timeoutOpts.milliseconds must be at least waitForMs + 10000 ms; a shorter
258+
deadline is rejected with 400 TIMEOUT_TOO_SHORT_FOR_WAIT.
259+
"""
260+
251261
zdr: Literal["enabled", "disabled"]
252262
"""
253263
Set to enabled to bypass shared caches and omit request and response content

‎tests/api_resources/test_web.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,7 @@ def test_method_web_scrape_bytes_with_all_params(self, client: ContextDev) -> No
534534
"milliseconds": 1,
535535
"behavior": "fail",
536536
},
537+
wait_for_ms=0,
537538
zdr="enabled",
538539
)
539540
assert_matches_type(WebWebScrapeBytesResponse, web, path=["response"])
@@ -1393,6 +1394,7 @@ async def test_method_web_scrape_bytes_with_all_params(self, async_client: Async
13931394
"milliseconds": 1,
13941395
"behavior": "fail",
13951396
},
1397+
wait_for_ms=0,
13961398
zdr="enabled",
13971399
)
13981400
assert_matches_type(WebWebScrapeBytesResponse, web, path=["response"])

0 commit comments

Comments
 (0)