Skip to content

Commit 7ae20fc

Browse files
author
stlc-bot
committed
feat(batch): follow only chosen languages in start_url crawls (#2003)
Stainless-Generated-From: 54d95a25963712129c1828b884f3707f208a12d3
1 parent b23dcf2 commit 7ae20fc

6 files changed

Lines changed: 1570 additions & 2 deletions

File tree

‎lib/context_dev/models/batch_submit_params.rb‎

Lines changed: 135 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1854,6 +1854,16 @@ class Controls < ContextDev::Internal::Type::BaseModel
18541854
ContextDev::Internal::Type::Boolean,
18551855
api_name: :followSubdomains
18561856

1857+
# @!attribute languages
1858+
# Languages whose pages are followed. `same_as_start_url` follows only the start
1859+
# page's language; a list of language codes such as `["en"]` follows those
1860+
# languages. Links whose language cannot be determined are followed. When omitted
1861+
# all urls are crawled.
1862+
#
1863+
# @return [Symbol, Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1>, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages, nil]
1864+
optional :languages,
1865+
union: -> { ContextDev::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages }
1866+
18571867
# @!attribute max_depth
18581868
# Maximum link depth. Source pages are depth 0. No limit when omitted.
18591869
#
@@ -1872,7 +1882,7 @@ class Controls < ContextDev::Internal::Type::BaseModel
18721882
# @return [String, nil]
18731883
optional :regex, String
18741884

1875-
# @!method initialize(exclude_regex: nil, follow_subdomains: nil, max_depth: nil, max_urls: nil, regex: nil)
1885+
# @!method initialize(exclude_regex: nil, follow_subdomains: nil, languages: nil, max_depth: nil, max_urls: nil, regex: nil)
18761886
# Some parameter documentations has been truncated, see
18771887
# {ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls}
18781888
# for more details.
@@ -1883,11 +1893,135 @@ class Controls < ContextDev::Internal::Type::BaseModel
18831893
#
18841894
# @param follow_subdomains [Boolean] Follow links to subdomains.
18851895
#
1896+
# @param languages [Symbol, Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1>, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages] Languages whose pages are followed. `same_as_start_url` follows only the start p
1897+
#
18861898
# @param max_depth [Integer] Maximum link depth. Source pages are depth 0. No limit when omitted.
18871899
#
18881900
# @param max_urls [Integer] Maximum pages to fetch. Unused reserved credits are refunded. Maximum 100000.
18891901
#
18901902
# @param regex [String] RE2 pattern for URLs to include. The `start_url` itself is always included.
1903+
1904+
# Languages whose pages are followed. `same_as_start_url` follows only the start
1905+
# page's language; a list of language codes such as `["en"]` follows those
1906+
# languages. Links whose language cannot be determined are followed. When omitted
1907+
# all urls are crawled.
1908+
#
1909+
# @see ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls#languages
1910+
module Languages
1911+
extend ContextDev::Internal::Type::Union
1912+
1913+
variant const: -> { ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::SAME_AS_START_URL }
1914+
1915+
variant -> { ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1Array }
1916+
1917+
module UnionMember1
1918+
extend ContextDev::Internal::Type::Enum
1919+
1920+
AM = :am
1921+
AR = :ar
1922+
AZ = :az
1923+
BE = :be
1924+
BG = :bg
1925+
BN = :bn
1926+
CA = :ca
1927+
CEB = :ceb
1928+
CS = :cs
1929+
CY = :cy
1930+
DA = :da
1931+
DE = :de
1932+
EL = :el
1933+
EN = :en
1934+
ES = :es
1935+
ET = :et
1936+
EU = :eu
1937+
FA = :fa
1938+
FI = :fi
1939+
FR = :fr
1940+
GU = :gu
1941+
HA = :ha
1942+
HAW = :haw
1943+
HE = :he
1944+
HI = :hi
1945+
HR = :hr
1946+
HU = :hu
1947+
HY = :hy
1948+
ID = :id
1949+
IS = :is
1950+
IT = :it
1951+
JA = :ja
1952+
KA = :ka
1953+
KK = :kk
1954+
KN = :kn
1955+
KO = :ko
1956+
KU = :ku
1957+
KY = :ky
1958+
LO = :lo
1959+
LT = :lt
1960+
LV = :lv
1961+
MK = :mk
1962+
ML = :ml
1963+
MN = :mn
1964+
MR = :mr
1965+
MS = :ms
1966+
NE = :ne
1967+
NL = :nl
1968+
NO = :no
1969+
OR = :or
1970+
PA = :pa
1971+
PL = :pl
1972+
PS = :ps
1973+
PT = :pt
1974+
RO = :ro
1975+
RU = :ru
1976+
SK = :sk
1977+
SL = :sl
1978+
SO = :so
1979+
SQ = :sq
1980+
SR = :sr
1981+
SV = :sv
1982+
SW = :sw
1983+
TA = :ta
1984+
TE = :te
1985+
TH = :th
1986+
TL = :tl
1987+
TR = :tr
1988+
UK = :uk
1989+
UR = :ur
1990+
UZ = :uz
1991+
VI = :vi
1992+
YO = :yo
1993+
ZH = :zh
1994+
1995+
# @!method self.values
1996+
# @return [Array<Symbol>]
1997+
end
1998+
1999+
# @!method self.variants
2000+
# @return [Array(Symbol, Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1>)]
2001+
2002+
define_sorbet_constant!(:Variants) do
2003+
T.type_alias do
2004+
T.any(
2005+
ContextDev::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::TaggedSymbol,
2006+
T::Array[
2007+
ContextDev::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1::TaggedSymbol
2008+
]
2009+
)
2010+
end
2011+
end
2012+
2013+
# @!group
2014+
2015+
SAME_AS_START_URL = :same_as_start_url
2016+
2017+
# @!endgroup
2018+
2019+
# @type [ContextDev::Internal::Type::Converter]
2020+
UnionMember1Array =
2021+
ContextDev::Internal::Type::ArrayOf[
2022+
enum: -> { ContextDev::BatchSubmitParams::Input::Crawl::Data::Source::StartURL::Controls::Languages::UnionMember1 }
2023+
]
2024+
end
18912025
end
18922026
end
18932027

‎lib/context_dev/models/crawl_controls.rb‎

Lines changed: 128 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ class CrawlControls < ContextDev::Internal::Type::BaseModel
1515
# @return [Boolean]
1616
required :follow_subdomains, ContextDev::Internal::Type::Boolean
1717

18+
# @!attribute languages
19+
# Languages whose pages were followed: `same_as_start_url` or a list of language
20+
# codes. Null when every language was followed, which is always the case for a
21+
# sitemap crawl.
22+
#
23+
# @return [Symbol, Array<Symbol, ContextDev::Models::CrawlControls::Languages::UnionMember1>, ContextDev::Models::CrawlControls::Languages, nil]
24+
required :languages, union: -> { ContextDev::CrawlControls::Languages }, nil?: true
25+
1826
# @!attribute max_depth
1927
# Link depth limit. Always 0 for a sitemap crawl, which never follows links off
2028
# its URLs; null when a `start_url` crawl set no limit.
@@ -41,7 +49,7 @@ class CrawlControls < ContextDev::Internal::Type::BaseModel
4149
# @return [String, nil]
4250
required :url_pattern, String, nil?: true
4351

44-
# @!method initialize(exclude_url_pattern:, follow_subdomains:, max_depth:, max_pages:, source:, url_pattern:)
52+
# @!method initialize(exclude_url_pattern:, follow_subdomains:, languages:, max_depth:, max_pages:, source:, url_pattern:)
4553
# Some parameter documentations has been truncated, see
4654
# {ContextDev::Models::CrawlControls} for more details.
4755
#
@@ -51,6 +59,8 @@ class CrawlControls < ContextDev::Internal::Type::BaseModel
5159
#
5260
# @param follow_subdomains [Boolean] Whether links to subdomains were followed. Always false for a sitemap crawl.
5361
#
62+
# @param languages [Symbol, Array<Symbol, ContextDev::Models::CrawlControls::Languages::UnionMember1>, ContextDev::Models::CrawlControls::Languages, nil] Languages whose pages were followed: `same_as_start_url` or a list of language c
63+
#
5464
# @param max_depth [Integer, nil] Link depth limit. Always 0 for a sitemap crawl, which never follows links off it
5565
#
5666
# @param max_pages [Integer] The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs it
@@ -59,6 +69,123 @@ class CrawlControls < ContextDev::Internal::Type::BaseModel
5969
#
6070
# @param url_pattern [String, nil] RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
6171

72+
# Languages whose pages were followed: `same_as_start_url` or a list of language
73+
# codes. Null when every language was followed, which is always the case for a
74+
# sitemap crawl.
75+
#
76+
# @see ContextDev::Models::CrawlControls#languages
77+
module Languages
78+
extend ContextDev::Internal::Type::Union
79+
80+
variant const: -> { ContextDev::Models::CrawlControls::Languages::SAME_AS_START_URL }
81+
82+
variant -> { ContextDev::Models::CrawlControls::Languages::UnionMember1Array }
83+
84+
module UnionMember1
85+
extend ContextDev::Internal::Type::Enum
86+
87+
AM = :am
88+
AR = :ar
89+
AZ = :az
90+
BE = :be
91+
BG = :bg
92+
BN = :bn
93+
CA = :ca
94+
CEB = :ceb
95+
CS = :cs
96+
CY = :cy
97+
DA = :da
98+
DE = :de
99+
EL = :el
100+
EN = :en
101+
ES = :es
102+
ET = :et
103+
EU = :eu
104+
FA = :fa
105+
FI = :fi
106+
FR = :fr
107+
GU = :gu
108+
HA = :ha
109+
HAW = :haw
110+
HE = :he
111+
HI = :hi
112+
HR = :hr
113+
HU = :hu
114+
HY = :hy
115+
ID = :id
116+
IS = :is
117+
IT = :it
118+
JA = :ja
119+
KA = :ka
120+
KK = :kk
121+
KN = :kn
122+
KO = :ko
123+
KU = :ku
124+
KY = :ky
125+
LO = :lo
126+
LT = :lt
127+
LV = :lv
128+
MK = :mk
129+
ML = :ml
130+
MN = :mn
131+
MR = :mr
132+
MS = :ms
133+
NE = :ne
134+
NL = :nl
135+
NO = :no
136+
OR = :or
137+
PA = :pa
138+
PL = :pl
139+
PS = :ps
140+
PT = :pt
141+
RO = :ro
142+
RU = :ru
143+
SK = :sk
144+
SL = :sl
145+
SO = :so
146+
SQ = :sq
147+
SR = :sr
148+
SV = :sv
149+
SW = :sw
150+
TA = :ta
151+
TE = :te
152+
TH = :th
153+
TL = :tl
154+
TR = :tr
155+
UK = :uk
156+
UR = :ur
157+
UZ = :uz
158+
VI = :vi
159+
YO = :yo
160+
ZH = :zh
161+
162+
# @!method self.values
163+
# @return [Array<Symbol>]
164+
end
165+
166+
# @!method self.variants
167+
# @return [Array(Symbol, Array<Symbol, ContextDev::Models::CrawlControls::Languages::UnionMember1>)]
168+
169+
define_sorbet_constant!(:Variants) do
170+
T.type_alias do
171+
T.any(
172+
ContextDev::CrawlControls::Languages::TaggedSymbol,
173+
T::Array[ContextDev::CrawlControls::Languages::UnionMember1::TaggedSymbol]
174+
)
175+
end
176+
end
177+
178+
# @!group
179+
180+
SAME_AS_START_URL = :same_as_start_url
181+
182+
# @!endgroup
183+
184+
# @type [ContextDev::Internal::Type::Converter]
185+
UnionMember1Array =
186+
ContextDev::Internal::Type::ArrayOf[enum: -> { ContextDev::CrawlControls::Languages::UnionMember1 }]
187+
end
188+
62189
# Where the crawl started.
63190
#
64191
# @see ContextDev::Models::CrawlControls#source

0 commit comments

Comments
 (0)