You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit d48939f
Browse filesBrowse the repository at this point in the historyBrowse files
stlc-bot
committed
docs(openapi): unify regex wording and shorten sourcePreference descriptions (#2051)
# RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
45
+
# Regex pattern URLs had to match to be crawled. Null when the crawl set none.
48
46
#
49
47
# @return [String, nil]
50
48
required:url_pattern,String,nil?: true
@@ -55,23 +53,21 @@ class CrawlControls < ContextDev::Internal::Type::BaseModel
55
53
#
56
54
# Crawl settings as submitted.
57
55
#
58
-
# @param exclude_url_pattern [String, nil] RE2 pattern URLs must not match to be crawled. Null when the crawl set none.
56
+
# @param exclude_url_pattern [String, nil] Regex pattern URLs must not match to be crawled. Null when the crawl set none.
59
57
#
60
58
# @param follow_subdomains [Boolean] Whether links to subdomains were followed. Always false for a sitemap crawl.
61
59
#
62
-
# @param languages [Symbol, Array<Symbol, ContextDev::Models::CrawlControls::Languages::UnionMember1>, ContextDev::Models::CrawlControls::Languages, nil] Languages whose pages were followed: `same_as_start_url` or a list of language c
60
+
# @param languages [Symbol, Array<Symbol, ContextDev::Models::CrawlControls::Languages::UnionMember1>, ContextDev::Models::CrawlControls::Languages, nil] Languages whose pages were followed; null when all languages were.
63
61
#
64
62
# @param max_depth [Integer, nil] Link depth limit. Always 0 for a sitemap crawl, which never follows links off it
65
63
#
66
64
# @param max_pages [Integer] The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs it
67
65
#
68
66
# @param source [ContextDev::Models::CrawlControls::Source::StartURL, ContextDev::Models::CrawlControls::Source::Sitemap] Where the crawl started.
69
67
#
70
-
# @param url_pattern [String, nil] RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
68
+
# @param url_pattern [String, nil] Regex pattern URLs had to match to be crawled. Null when the crawl set none.
71
69
72
-
# Languages whose pages were followed: `same_as_start_url` or a list of language
73
-
# codes. Null when every language was followed, which is always the case for a
74
-
# sitemap crawl.
70
+
# Languages whose pages were followed; null when all languages were.
@@ -348,16 +338,15 @@ class MarkdownParams < ContextDev::Internal::Type::BaseModel
348
338
#
349
339
# Markdown options. Requires `formats.markdown`.
350
340
#
351
-
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text. Only a
341
+
# @param include_images [Boolean] Include images in the Markdown (`html-conversion` only).
352
342
#
353
-
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs. Only
343
+
# @param include_links [Boolean] Keep link URLs in the Markdown (`html-conversion` only).
354
344
#
355
-
# @param inline_images [Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::InlineImages] How base64 images appear: `placeholder` (default) or `preserve`. Requires `inclu
345
+
# @param inline_images [Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::InlineImages] How base64 images appear when `includeImages` is set (`html-conversion` only).
356
346
#
357
-
# @param source_preference [Array<Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
347
+
# @param source_preference [Array<Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::SourcePreference>] Markdown sources in order of preference: `html-conversion` converts the page HTM
358
348
359
-
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
360
-
# `includeImages`. Only applies to `html-conversion` extractions.
349
+
# How base64 images appear when `includeImages` is set (`html-conversion` only).
@@ -187,15 +178,15 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
187
178
#
188
179
# @param exclude_selectors [Array<String>, nil] Remove matching elements after inclusions. Exclusions take precedence.
189
180
#
190
-
# @param exclude_url_regex [String] RE2 regex pattern. URLs matching this pattern are never followed or scraped, eve
181
+
# @param exclude_url_regex [String] Regex pattern. URLs matching this pattern are never followed or scraped, even wh
191
182
#
192
183
# @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
193
184
#
194
185
# @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown for each crawled pag
195
186
#
196
-
# @param include_images [Boolean] Include image references in the Markdown output Only applies to `html-conversion
187
+
# @param include_images [Boolean] Include image references in the Markdown output (`html-conversion` only).
197
188
#
198
-
# @param include_links [Boolean] Preserve hyperlinks in the Markdown output Only applies to `html-conversion` ext
189
+
# @param include_links [Boolean] Preserve hyperlinks in the Markdown output (`html-conversion` only).
199
190
#
200
191
# @param include_selectors [Array<String>, nil] Keep matching HTML subtrees before converting each page to Markdown.
201
192
#
@@ -209,9 +200,9 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
209
200
#
210
201
# @param settle_animations [Boolean] Wait briefly for CSS animations and transitions to settle before reading each pa
211
202
#
212
-
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output Only applies to `html-
203
+
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output (`html-conversion` onl
213
204
#
214
-
# @param source_preference [Array<Symbol, ContextDev::Models::WebWebCrawlMdParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
205
+
# @param source_preference [Array<Symbol, ContextDev::Models::WebWebCrawlMdParams::SourcePreference>] Markdown sources in order of preference: `html-conversion` converts the page HTM
215
206
#
216
207
# @param stop_after_ms [Integer] Soft crawl deadline in milliseconds. Returns pages collected before the next dea
# @param timeout_opts [ContextDev::Models::WebMapURLsParams::TimeoutOpts] Request deadline and what to return when it passes.
145
145
#
146
-
# @param url_regex [String] Optional RE2-compatible regex pattern. Only URLs matching this pattern are retur
146
+
# @param url_regex [String] Regex pattern. Only URLs matching this pattern are returned and counted against
147
147
#
148
148
# @param zdr [Symbol, ContextDev::Models::WebMapURLsParams::Zdr] `enabled` turns on zero data retention. Returns 403 `ZDR_NOT_ENABLED` unless you
149
149
#
@@ -355,15 +355,15 @@ def search(params)
355
355
#
356
356
# @param exclude_selectors [Array<String>, nil] Remove matching elements after inclusions. Exclusions take precedence.
357
357
#
358
-
# @param exclude_url_regex [String] RE2 regex pattern. URLs matching this pattern are never followed or scraped, eve
358
+
# @param exclude_url_regex [String] Regex pattern. URLs matching this pattern are never followed or scraped, even wh
359
359
#
360
360
# @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
361
361
#
362
362
# @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown for each crawled pag
363
363
#
364
-
# @param include_images [Boolean] Include image references in the Markdown output Only applies to `html-conversion
364
+
# @param include_images [Boolean] Include image references in the Markdown output (`html-conversion` only).
365
365
#
366
-
# @param include_links [Boolean] Preserve hyperlinks in the Markdown output Only applies to `html-conversion` ext
366
+
# @param include_links [Boolean] Preserve hyperlinks in the Markdown output (`html-conversion` only).
367
367
#
368
368
# @param include_selectors [Array<String>, nil] Keep matching HTML subtrees before converting each page to Markdown.
369
369
#
@@ -377,9 +377,9 @@ def search(params)
377
377
#
378
378
# @param settle_animations [Boolean] Wait briefly for CSS animations and transitions to settle before reading each pa
379
379
#
380
-
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output Only applies to `html-
380
+
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output (`html-conversion` onl
381
381
#
382
-
# @param source_preference [Array<Symbol, ContextDev::Models::WebWebCrawlMdParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
382
+
# @param source_preference [Array<Symbol, ContextDev::Models::WebWebCrawlMdParams::SourcePreference>] Markdown sources in order of preference: `html-conversion` converts the page HTM
383
383
#
384
384
# @param stop_after_ms [Integer] Soft crawl deadline in milliseconds. Returns pages collected before the next dea
0 commit comments