feat(sitemap): add subdomain discovery

Stainless-Generated-From: 583748d

Author: stlc-bot

src/context/dev/resources/web.py

@@ -2207,6 +2207,7 @@ def web_scrape_sitemap(
         *,
         domain: str,
         headers: Dict[str, str] | Omit = omit,
+        include_subdomains: bool | Omit = omit,
         max_links: int | Omit = omit,
         search: str | Omit = omit,
         sitemap_url: str | Omit = omit,
@@ -2223,11 +2224,12 @@ def web_scrape_sitemap(
     ) -> WebWebScrapeSitemapResponse:
         """Crawl an entire website's sitemap and return all discovered page URLs.
 
-        Pass
-        `search` to have the crawled sitemap filtered down to the pages about a phrase
-        (for example `pricing and plans` or `api authentication docs`), most relevant
-        first — a searched crawl scans the whole sitemap and costs 2 credits instead
-        of 1.
+        Set
+        `includeSubdomains=true` to also discover public pages and sitemaps on child
+        hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have
+        the discovered URLs filtered down to the pages about a phrase (for example
+        `pricing and plans` or `api authentication docs`), most relevant first — a
+        searched crawl scans the whole sitemap and costs 2 credits instead of 1.
 
         Args:
           domain: Domain to build a sitemap for
@@ -2236,6 +2238,9 @@ def web_scrape_sitemap(
               deep-object query params such as headers[X-Custom]=value. When provided, caching
               is bypassed: the result is neither read from nor written to cache.
 
+          include_subdomains: When true, discover and include public pages and sitemaps on subdomains of the
+              requested domain. Defaults to false.
+
           max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
               Minimum is 1, maximum is 100,000.
 
@@ -2281,6 +2286,7 @@ def web_scrape_sitemap(
                     {
                         "domain": domain,
                         "headers": headers,
+                        "include_subdomains": include_subdomains,
                         "max_links": max_links,
                         "search": search,
                         "sitemap_url": sitemap_url,
@@ -4457,6 +4463,7 @@ async def web_scrape_sitemap(
         *,
         domain: str,
         headers: Dict[str, str] | Omit = omit,
+        include_subdomains: bool | Omit = omit,
         max_links: int | Omit = omit,
         search: str | Omit = omit,
         sitemap_url: str | Omit = omit,
@@ -4473,11 +4480,12 @@ async def web_scrape_sitemap(
     ) -> WebWebScrapeSitemapResponse:
         """Crawl an entire website's sitemap and return all discovered page URLs.
 
-        Pass
-        `search` to have the crawled sitemap filtered down to the pages about a phrase
-        (for example `pricing and plans` or `api authentication docs`), most relevant
-        first — a searched crawl scans the whole sitemap and costs 2 credits instead
-        of 1.
+        Set
+        `includeSubdomains=true` to also discover public pages and sitemaps on child
+        hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have
+        the discovered URLs filtered down to the pages about a phrase (for example
+        `pricing and plans` or `api authentication docs`), most relevant first — a
+        searched crawl scans the whole sitemap and costs 2 credits instead of 1.
 
         Args:
           domain: Domain to build a sitemap for
@@ -4486,6 +4494,9 @@ async def web_scrape_sitemap(
               deep-object query params such as headers[X-Custom]=value. When provided, caching
               is bypassed: the result is neither read from nor written to cache.
 
+          include_subdomains: When true, discover and include public pages and sitemaps on subdomains of the
+              requested domain. Defaults to false.
+
           max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
               Minimum is 1, maximum is 100,000.
 
@@ -4531,6 +4542,7 @@ async def web_scrape_sitemap(
                     {
                         "domain": domain,
                         "headers": headers,
+                        "include_subdomains": include_subdomains,
                         "max_links": max_links,
                         "search": search,
                         "sitemap_url": sitemap_url,

src/context/dev/types/web_web_scrape_sitemap_params.py

@@ -22,6 +22,12 @@ class WebWebScrapeSitemapParams(TypedDict, total=False):
     is bypassed: the result is neither read from nor written to cache.
     """
 
+    include_subdomains: Annotated[bool, PropertyInfo(alias="includeSubdomains")]
+    """
+    When true, discover and include public pages and sitemaps on subdomains of the
+    requested domain. Defaults to false.
+    """
+
     max_links: Annotated[int, PropertyInfo(alias="maxLinks")]
     """Maximum number of links to return from the sitemap crawl.
 

tests/api_resources/test_web.py

@@ -640,6 +640,7 @@ def test_method_web_scrape_sitemap_with_all_params(self, client: ContextDev) ->
         web = client.web.web_scrape_sitemap(
             domain="xxx",
             headers={"foo": "J!"},
+            include_subdomains=True,
             max_links=1,
             search="help center and troubleshooting articles",
             sitemap_url="https://example.com",
@@ -1293,6 +1294,7 @@ async def test_method_web_scrape_sitemap_with_all_params(self, async_client: Asy
         web = await async_client.web.web_scrape_sitemap(
             domain="xxx",
             headers={"foo": "J!"},
+            include_subdomains=True,
             max_links=1,
             search="help center and troubleshooting articles",
             sitemap_url="https://example.com",