feat(batches): let sitemap crawls choose which paths count as non-pages (#2796)

Stainless-Generated-From: 6cc6ef98459a02e9c8578225ccd921ea8292ffa6

Author: stlc-bot

src/context/dev/types/batch_submit_params.py

@@ -1530,6 +1530,14 @@ class InputCrawlDataMarkdownSourceSitemapControls(TypedDict, total=False):
     A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here.
     """
 
+    exclude_paths: Annotated[SequenceNotStr[str], PropertyInfo(alias="excludePaths")]
+    """Path fragments that skip a sitemap URL in a crawl.
+
+    Works separately from any regex inclusion or exclusion you define. Defaults to
+    ["/feed","/rss","/atom","/api/","/cdn-cgi/","/_next/","/__"]. Pass `[]` to keep
+    every listed URL.
+    """
+
     exclude_regex: Annotated[str, PropertyInfo(alias="excludeRegex")]
     """
     Regex pattern; sitemap URLs matching it are not scraped, even when they match
@@ -1993,6 +2001,14 @@ class InputCrawlDataHTMLSourceSitemapControls(TypedDict, total=False):
     A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here.
     """
 
+    exclude_paths: Annotated[SequenceNotStr[str], PropertyInfo(alias="excludePaths")]
+    """Path fragments that skip a sitemap URL in a crawl.
+
+    Works separately from any regex inclusion or exclusion you define. Defaults to
+    ["/feed","/rss","/atom","/api/","/cdn-cgi/","/_next/","/__"]. Pass `[]` to keep
+    every listed URL.
+    """
+
     exclude_regex: Annotated[str, PropertyInfo(alias="excludeRegex")]
     """
     Regex pattern; sitemap URLs matching it are not scraped, even when they match
@@ -2468,6 +2484,14 @@ class InputCrawlDataBatchUnifiedCrawlDataSourceSitemapControls(TypedDict, total=
     A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here.
     """
 
+    exclude_paths: Annotated[SequenceNotStr[str], PropertyInfo(alias="excludePaths")]
+    """Path fragments that skip a sitemap URL in a crawl.
+
+    Works separately from any regex inclusion or exclusion you define. Defaults to
+    ["/feed","/rss","/atom","/api/","/cdn-cgi/","/_next/","/__"]. Pass `[]` to keep
+    every listed URL.
+    """
+
     exclude_regex: Annotated[str, PropertyInfo(alias="excludeRegex")]
     """
     Regex pattern; sitemap URLs matching it are not scraped, even when they match