feat(scrape): add sourcePreference to choose between HTML conversion and published Markdown (#2010)

Stainless-Generated-From: 95d1d3475e3ebdd10577b3497d9eb2b71bdc3def

Author: stlc-bot

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/charmbracelet/bubbletea v1.3.6
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/charmbracelet/x/term v0.2.1
-	github.com/context-dot-dev/context-go-sdk/v2 v2.31.3
+	github.com/context-dot-dev/context-go-sdk/v2 v2.32.0
 	github.com/goccy/go-yaml v1.18.0
 	github.com/itchyny/json2yaml v0.1.4
 	github.com/muesli/reflow v0.3.0

go.sum

@@ -18,8 +18,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR
 github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
 github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
 github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
-github.com/context-dot-dev/context-go-sdk/v2 v2.31.3 h1:GKW1xWJfGtqmjVidIWMLEszE5XOF7n7VWvKzxhup7kw=
-github.com/context-dot-dev/context-go-sdk/v2 v2.31.3/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8=
+github.com/context-dot-dev/context-go-sdk/v2 v2.32.0 h1:rF7kVjfdI4QsHyRuZySwllnUpdrlWqMgvk6bdR1vk0k=
+github.com/context-dot-dev/context-go-sdk/v2 v2.32.0/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

pkg/cmd/batch_test.go

@@ -84,7 +84,7 @@ func TestBatchSubmit(t *testing.T) {
 			t,
 			"--api-key", "string",
 			"batch", "submit",
-			"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
+			"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
 			"--tag", "docs",
 			"--tag", "competitor",
 			"--webhook", "{url: https://example.com, retry: {delays_seconds: [10, 60, 300, 1800, 7200, 21600, 57600]}}",
@@ -102,7 +102,7 @@ func TestBatchSubmit(t *testing.T) {
 			t,
 			"--api-key", "string",
 			"batch", "submit",
-			"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
+			"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
 			"--tag", "docs",
 			"--tag", "competitor",
 			"--webhook.url", "https://example.com",
@@ -154,6 +154,8 @@ func TestBatchSubmit(t *testing.T) {
 			"      includeImages: true\n" +
 			"      includeLinks: true\n" +
 			"      inlineImages: placeholder\n" +
+			"      sourcePreference:\n" +
+			"        - html-conversion\n" +
 			"    maxAgeMs: 0\n" +
 			"    parseParams:\n" +
 			"      rules:\n" +

pkg/cmd/web.go

@@ -502,13 +502,13 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{
 		},
 		&requestflag.Flag[bool]{
 			Name:     "include-images",
-			Usage:    "Include image references in the Markdown output",
+			Usage:    "Include image references in the Markdown output Only applies to `html-conversion` extractions.",
 			Default:  false,
 			BodyPath: "includeImages",
 		},
 		&requestflag.Flag[bool]{
 			Name:     "include-links",
-			Usage:    "Preserve hyperlinks in the Markdown output",
+			Usage:    "Preserve hyperlinks in the Markdown output Only applies to `html-conversion` extractions.",
 			Default:  true,
 			BodyPath: "includeLinks",
 		},
@@ -548,10 +548,16 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{
 		},
 		&requestflag.Flag[bool]{
 			Name:     "shorten-base64-images",
-			Usage:    "Truncate base64-encoded image data in the Markdown output",
+			Usage:    "Truncate base64-encoded image data in the Markdown output Only applies to `html-conversion` extractions.",
 			Default:  true,
 			BodyPath: "shortenBase64Images",
 		},
+		&requestflag.Flag[[]string]{
+			Name:     "source-preference",
+			Usage:    "Where the Markdown should come from, in order of preference; the first usable source wins. `html-conversion` converts the page HTML, and counts as unusable when the conversion has almost no content (under 200 characters). `markdown-direct` uses the official Markdown version the page publishes through, returned as the site serves it. Defaults to `[\"html-conversion\", \"markdown-direct\"]`. With only `markdown-direct`, a page that publishes no Markdown fails with `NO_PUBLISHED_MARKDOWN` and is not billed. Requests with include or exclude selectors never use `markdown-direct`, so they must include `html-conversion`.",
+			Default:  []string{"html-conversion", "markdown-direct"},
+			BodyPath: "sourcePreference",
+		},
 		&requestflag.Flag[int64]{
 			Name:     "stop-after-ms",
 			Usage:    "Soft crawl deadline in milliseconds. Returns pages collected before the next deadline check.",

pkg/cmd/web_test.go

@@ -337,6 +337,7 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"--pdf", "{end: 1, ocr: true, shouldParse: true, start: 1}",
 			"--settle-animations=true",
 			"--shorten-base64-images=true",
+			"--source-preference", "html-conversion",
 			"--stop-after-ms", "10000",
 			"--tag", "production",
 			"--tag", "team-alpha",
@@ -375,6 +376,7 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"--pdf.start", "1",
 			"--settle-animations=true",
 			"--shorten-base64-images=true",
+			"--source-preference", "html-conversion",
 			"--stop-after-ms", "10000",
 			"--tag", "production",
 			"--tag", "team-alpha",
@@ -411,6 +413,8 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"  start: 1\n" +
 			"settleAnimations: true\n" +
 			"shortenBase64Images: true\n" +
+			"sourcePreference:\n" +
+			"  - html-conversion\n" +
 			"stopAfterMs: 10000\n" +
 			"tags:\n" +
 			"  - production\n" +