Skip to content

Commit 9a47f59

Browse files
author
stlc-bot
committed
feat(batch): follow only chosen languages in start_url crawls (#2003)
Stainless-Generated-From: 935acbd7115390c06bb086c0c2f44bc89044a3a7
1 parent bbe2ad6 commit 9a47f59

2 files changed

Lines changed: 177 additions & 1 deletion

File tree

‎src/context/dev/types/batch_submit_params.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,94 @@ class InputCrawlDataSourceStartURLControls(TypedDict, total=False):
711711
follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")]
712712
"""Follow links to subdomains."""
713713

714+
languages: Union[
715+
Literal["same_as_start_url"],
716+
List[
717+
Literal[
718+
"am",
719+
"ar",
720+
"az",
721+
"be",
722+
"bg",
723+
"bn",
724+
"ca",
725+
"ceb",
726+
"cs",
727+
"cy",
728+
"da",
729+
"de",
730+
"el",
731+
"en",
732+
"es",
733+
"et",
734+
"eu",
735+
"fa",
736+
"fi",
737+
"fr",
738+
"gu",
739+
"ha",
740+
"haw",
741+
"he",
742+
"hi",
743+
"hr",
744+
"hu",
745+
"hy",
746+
"id",
747+
"is",
748+
"it",
749+
"ja",
750+
"ka",
751+
"kk",
752+
"kn",
753+
"ko",
754+
"ku",
755+
"ky",
756+
"lo",
757+
"lt",
758+
"lv",
759+
"mk",
760+
"ml",
761+
"mn",
762+
"mr",
763+
"ms",
764+
"ne",
765+
"nl",
766+
"no",
767+
"or",
768+
"pa",
769+
"pl",
770+
"ps",
771+
"pt",
772+
"ro",
773+
"ru",
774+
"sk",
775+
"sl",
776+
"so",
777+
"sq",
778+
"sr",
779+
"sv",
780+
"sw",
781+
"ta",
782+
"te",
783+
"th",
784+
"tl",
785+
"tr",
786+
"uk",
787+
"ur",
788+
"uz",
789+
"vi",
790+
"yo",
791+
"zh",
792+
]
793+
],
794+
]
795+
"""Languages whose pages are followed.
796+
797+
`same_as_start_url` follows only the start page's language; a list of language
798+
codes such as `["en"]` follows those languages. Links whose language cannot be
799+
determined are followed. When omitted all urls are crawled.
800+
"""
801+
714802
max_depth: Annotated[int, PropertyInfo(alias="maxDepth")]
715803
"""Maximum link depth. Source pages are depth 0. No limit when omitted."""
716804

‎src/context/dev/types/crawl_controls.py‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
22

3-
from typing import Union, Optional
3+
from typing import List, Union, Optional
44
from typing_extensions import Literal, TypeAlias
55

66
from .._models import BaseModel
@@ -38,6 +38,94 @@ class CrawlControls(BaseModel):
3838
follow_subdomains: bool
3939
"""Whether links to subdomains were followed. Always false for a sitemap crawl."""
4040

41+
languages: Union[
42+
Literal["same_as_start_url"],
43+
List[
44+
Literal[
45+
"am",
46+
"ar",
47+
"az",
48+
"be",
49+
"bg",
50+
"bn",
51+
"ca",
52+
"ceb",
53+
"cs",
54+
"cy",
55+
"da",
56+
"de",
57+
"el",
58+
"en",
59+
"es",
60+
"et",
61+
"eu",
62+
"fa",
63+
"fi",
64+
"fr",
65+
"gu",
66+
"ha",
67+
"haw",
68+
"he",
69+
"hi",
70+
"hr",
71+
"hu",
72+
"hy",
73+
"id",
74+
"is",
75+
"it",
76+
"ja",
77+
"ka",
78+
"kk",
79+
"kn",
80+
"ko",
81+
"ku",
82+
"ky",
83+
"lo",
84+
"lt",
85+
"lv",
86+
"mk",
87+
"ml",
88+
"mn",
89+
"mr",
90+
"ms",
91+
"ne",
92+
"nl",
93+
"no",
94+
"or",
95+
"pa",
96+
"pl",
97+
"ps",
98+
"pt",
99+
"ro",
100+
"ru",
101+
"sk",
102+
"sl",
103+
"so",
104+
"sq",
105+
"sr",
106+
"sv",
107+
"sw",
108+
"ta",
109+
"te",
110+
"th",
111+
"tl",
112+
"tr",
113+
"uk",
114+
"ur",
115+
"uz",
116+
"vi",
117+
"yo",
118+
"zh",
119+
]
120+
],
121+
None,
122+
] = None
123+
"""
124+
Languages whose pages were followed: `same_as_start_url` or a list of language
125+
codes. Null when every language was followed, which is always the case for a
126+
sitemap crawl.
127+
"""
128+
41129
max_depth: Optional[int] = None
42130
"""Link depth limit.
43131

0 commit comments

Comments
 (0)