Skip to content

Commit 21d9ba7

Browse files
author
stlc-bot
committed
feat(scrape): add sourcePreference to choose between HTML conversion and published Markdown (#2010)
Stainless-Generated-From: 7d5f59458ef0592eb23544e143c464695514cafc
1 parent e7c529a commit 21d9ba7

13 files changed

Lines changed: 653 additions & 81 deletions

File tree

‎Gemfile.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ GIT
1111
PATH
1212
remote: .
1313
specs:
14-
context.dev (2.34.2)
14+
context.dev (2.35.0)
1515
cgi
1616
connection_pool
1717

‎lib/context_dev/models/batch_submit_params.rb‎

Lines changed: 78 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -446,20 +446,22 @@ class JsonParams < ContextDev::Internal::Type::BaseModel
446446
# @see ContextDev::Models::BatchSubmitParams::Input::Scrape::Data#markdown_params
447447
class MarkdownParams < ContextDev::Internal::Type::BaseModel
448448
# @!attribute include_images
449-
# Include images in the Markdown using image syntax with URLs and alt text.
449+
# Include images in the Markdown using image syntax with URLs and alt text. Only
450+
# applies to `html-conversion` extractions.
450451
#
451452
# @return [Boolean, nil]
452453
optional :include_images, ContextDev::Internal::Type::Boolean, api_name: :includeImages
453454

454455
# @!attribute include_links
455-
# Keep link URLs in the Markdown. Set false to return link text without URLs.
456+
# Keep link URLs in the Markdown. Set false to return link text without URLs. Only
457+
# applies to `html-conversion` extractions.
456458
#
457459
# @return [Boolean, nil]
458460
optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks
459461

460462
# @!attribute inline_images
461463
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
462-
# `includeImages`.
464+
# `includeImages`. Only applies to `html-conversion` extractions.
463465
#
464466
# @return [Symbol, ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams::InlineImages, nil]
465467
optional :inline_images,
@@ -468,21 +470,41 @@ class MarkdownParams < ContextDev::Internal::Type::BaseModel
468470
},
469471
api_name: :inlineImages
470472

471-
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil)
473+
# @!attribute source_preference
474+
# Where the Markdown should come from, in order of preference; the first usable
475+
# source wins. `html-conversion` converts the page HTML, and counts as unusable
476+
# when the conversion has almost no content (under 200 characters).
477+
# `markdown-direct` uses the official Markdown version the page publishes through,
478+
# returned as the site serves it. Defaults to
479+
# `["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
480+
# that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
481+
# Requests with include or exclude selectors never use `markdown-direct`, so they
482+
# must include `html-conversion`.
483+
#
484+
# @return [Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams::SourcePreference>, nil]
485+
optional :source_preference,
486+
-> {
487+
ContextDev::Internal::Type::ArrayOf[enum: ContextDev::BatchSubmitParams::Input::Scrape::Data::MarkdownParams::SourcePreference]
488+
},
489+
api_name: :sourcePreference
490+
491+
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil, source_preference: nil)
472492
# Some parameter documentations has been truncated, see
473493
# {ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams} for
474494
# more details.
475495
#
476496
# Markdown options. Requires `formats.markdown`.
477497
#
478-
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text.
498+
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text. Only a
479499
#
480-
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs.
500+
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs. Only
481501
#
482502
# @param inline_images [Symbol, ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams::InlineImages] How base64 images appear: `placeholder` (default) or `preserve`. Requires `inclu
503+
#
504+
# @param source_preference [Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
483505

484506
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
485-
# `includeImages`.
507+
# `includeImages`. Only applies to `html-conversion` extractions.
486508
#
487509
# @see ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::MarkdownParams#inline_images
488510
module InlineImages
@@ -494,6 +516,16 @@ module InlineImages
494516
# @!method self.values
495517
# @return [Array<Symbol>]
496518
end
519+
520+
module SourcePreference
521+
extend ContextDev::Internal::Type::Enum
522+
523+
HTML_CONVERSION = :"html-conversion"
524+
MARKDOWN_DIRECT = :"markdown-direct"
525+
526+
# @!method self.values
527+
# @return [Array<Symbol>]
528+
end
497529
end
498530

499531
# Requires `formats.parse: true`; required when it is set.
@@ -2200,20 +2232,22 @@ class JsonParams < ContextDev::Internal::Type::BaseModel
22002232
# @see ContextDev::Models::BatchSubmitParams::Input::Crawl::Data#markdown_params
22012233
class MarkdownParams < ContextDev::Internal::Type::BaseModel
22022234
# @!attribute include_images
2203-
# Include images in the Markdown using image syntax with URLs and alt text.
2235+
# Include images in the Markdown using image syntax with URLs and alt text. Only
2236+
# applies to `html-conversion` extractions.
22042237
#
22052238
# @return [Boolean, nil]
22062239
optional :include_images, ContextDev::Internal::Type::Boolean, api_name: :includeImages
22072240

22082241
# @!attribute include_links
2209-
# Keep link URLs in the Markdown. Set false to return link text without URLs.
2242+
# Keep link URLs in the Markdown. Set false to return link text without URLs. Only
2243+
# applies to `html-conversion` extractions.
22102244
#
22112245
# @return [Boolean, nil]
22122246
optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks
22132247

22142248
# @!attribute inline_images
22152249
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
2216-
# `includeImages`.
2250+
# `includeImages`. Only applies to `html-conversion` extractions.
22172251
#
22182252
# @return [Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams::InlineImages, nil]
22192253
optional :inline_images,
@@ -2222,21 +2256,41 @@ class MarkdownParams < ContextDev::Internal::Type::BaseModel
22222256
},
22232257
api_name: :inlineImages
22242258

2225-
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil)
2259+
# @!attribute source_preference
2260+
# Where the Markdown should come from, in order of preference; the first usable
2261+
# source wins. `html-conversion` converts the page HTML, and counts as unusable
2262+
# when the conversion has almost no content (under 200 characters).
2263+
# `markdown-direct` uses the official Markdown version the page publishes through,
2264+
# returned as the site serves it. Defaults to
2265+
# `["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
2266+
# that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
2267+
# Requests with include or exclude selectors never use `markdown-direct`, so they
2268+
# must include `html-conversion`.
2269+
#
2270+
# @return [Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams::SourcePreference>, nil]
2271+
optional :source_preference,
2272+
-> {
2273+
ContextDev::Internal::Type::ArrayOf[enum: ContextDev::BatchSubmitParams::Input::Crawl::Data::MarkdownParams::SourcePreference]
2274+
},
2275+
api_name: :sourcePreference
2276+
2277+
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil, source_preference: nil)
22262278
# Some parameter documentations has been truncated, see
22272279
# {ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams} for
22282280
# more details.
22292281
#
22302282
# Markdown options. Requires `formats.markdown`.
22312283
#
2232-
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text.
2284+
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text. Only a
22332285
#
2234-
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs.
2286+
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs. Only
22352287
#
22362288
# @param inline_images [Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams::InlineImages] How base64 images appear: `placeholder` (default) or `preserve`. Requires `inclu
2289+
#
2290+
# @param source_preference [Array<Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
22372291

22382292
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
2239-
# `includeImages`.
2293+
# `includeImages`. Only applies to `html-conversion` extractions.
22402294
#
22412295
# @see ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::MarkdownParams#inline_images
22422296
module InlineImages
@@ -2248,6 +2302,16 @@ module InlineImages
22482302
# @!method self.values
22492303
# @return [Array<Symbol>]
22502304
end
2305+
2306+
module SourcePreference
2307+
extend ContextDev::Internal::Type::Enum
2308+
2309+
HTML_CONVERSION = :"html-conversion"
2310+
MARKDOWN_DIRECT = :"markdown-direct"
2311+
2312+
# @!method self.values
2313+
# @return [Array<Symbol>]
2314+
end
22512315
end
22522316

22532317
# Requires `formats.parse: true`; required when it is set.

‎lib/context_dev/models/web_scrape_params.rb‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -302,40 +302,62 @@ class JsonParams < ContextDev::Internal::Type::BaseModel
302302

303303
class MarkdownParams < ContextDev::Internal::Type::BaseModel
304304
# @!attribute include_images
305-
# Include images in the Markdown using image syntax with URLs and alt text.
305+
# Include images in the Markdown using image syntax with URLs and alt text. Only
306+
# applies to `html-conversion` extractions.
306307
#
307308
# @return [Boolean, nil]
308309
optional :include_images, ContextDev::Internal::Type::Boolean, api_name: :includeImages
309310

310311
# @!attribute include_links
311-
# Keep link URLs in the Markdown. Set false to return link text without URLs.
312+
# Keep link URLs in the Markdown. Set false to return link text without URLs. Only
313+
# applies to `html-conversion` extractions.
312314
#
313315
# @return [Boolean, nil]
314316
optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks
315317

316318
# @!attribute inline_images
317319
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
318-
# `includeImages`.
320+
# `includeImages`. Only applies to `html-conversion` extractions.
319321
#
320322
# @return [Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::InlineImages, nil]
321323
optional :inline_images,
322324
enum: -> { ContextDev::WebScrapeParams::MarkdownParams::InlineImages },
323325
api_name: :inlineImages
324326

325-
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil)
327+
# @!attribute source_preference
328+
# Where the Markdown should come from, in order of preference; the first usable
329+
# source wins. `html-conversion` converts the page HTML, and counts as unusable
330+
# when the conversion has almost no content (under 200 characters).
331+
# `markdown-direct` uses the official Markdown version the page publishes through,
332+
# returned as the site serves it. Defaults to
333+
# `["html-conversion", "markdown-direct"]`. With only `markdown-direct`, a page
334+
# that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed.
335+
# Requests with include or exclude selectors never use `markdown-direct`, so they
336+
# must include `html-conversion`.
337+
#
338+
# @return [Array<Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::SourcePreference>, nil]
339+
optional :source_preference,
340+
-> {
341+
ContextDev::Internal::Type::ArrayOf[enum: ContextDev::WebScrapeParams::MarkdownParams::SourcePreference]
342+
},
343+
api_name: :sourcePreference
344+
345+
# @!method initialize(include_images: nil, include_links: nil, inline_images: nil, source_preference: nil)
326346
# Some parameter documentations has been truncated, see
327347
# {ContextDev::Models::WebScrapeParams::MarkdownParams} for more details.
328348
#
329349
# Markdown options. Requires `formats.markdown`.
330350
#
331-
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text.
351+
# @param include_images [Boolean] Include images in the Markdown using image syntax with URLs and alt text. Only a
332352
#
333-
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs.
353+
# @param include_links [Boolean] Keep link URLs in the Markdown. Set false to return link text without URLs. Only
334354
#
335355
# @param inline_images [Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::InlineImages] How base64 images appear: `placeholder` (default) or `preserve`. Requires `inclu
356+
#
357+
# @param source_preference [Array<Symbol, ContextDev::Models::WebScrapeParams::MarkdownParams::SourcePreference>] Where the Markdown should come from, in order of preference; the first usable so
336358

337359
# How base64 images appear: `placeholder` (default) or `preserve`. Requires
338-
# `includeImages`.
360+
# `includeImages`. Only applies to `html-conversion` extractions.
339361
#
340362
# @see ContextDev::Models::WebScrapeParams::MarkdownParams#inline_images
341363
module InlineImages
@@ -347,6 +369,16 @@ module InlineImages
347369
# @!method self.values
348370
# @return [Array<Symbol>]
349371
end
372+
373+
module SourcePreference
374+
extend ContextDev::Internal::Type::Enum
375+
376+
HTML_CONVERSION = :"html-conversion"
377+
MARKDOWN_DIRECT = :"markdown-direct"
378+
379+
# @!method self.values
380+
# @return [Array<Symbol>]
381+
end
350382
end
351383

352384
# Requires `formats.parse: true`; required when it is set.

0 commit comments

Comments
 (0)