Skip to content

Commit 403f0b2

Browse files
author
stlc-bot
committed
feat(batch): support Scrape Anything batches up to 100k URLs (#1883)
Stainless-Generated-From: 0b4d8ca007a3349b1c05080c46f1e8431549e770
1 parent fc43a74 commit 403f0b2

2 files changed

Lines changed: 85 additions & 20 deletions

File tree

‎pkg/cmd/batch.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ var batchList = cli.Command{
5656
Usage: "`prefix` for as-you-type prefix matching (default), `exact` for full-token matching.",
5757
QueryPath: "search_type",
5858
},
59+
&requestflag.Flag[bool]{
60+
Name: "should-include-unified",
61+
Usage: "Include batches submitted with Scrape Anything formats. Defaults to false.",
62+
Default: false,
63+
QueryPath: "shouldIncludeUnified",
64+
},
5965
&requestflag.Flag[string]{
6066
Name: "status",
6167
Usage: "Filter by status.",
@@ -105,7 +111,7 @@ var batchCancel = cli.Command{
105111

106112
var batchGetResults = cli.Command{
107113
Name: "get-results",
108-
Usage: "Page through a finished batch’s results as JSON. Results remain available for\n180 days.",
114+
Usage: "Page through a finished batch’s results as JSON. Each record contains result\nwith the same outputs and per-format errors as POST /web/scrape. Results remain\navailable for 180 days.",
109115
Suggest: true,
110116
Flags: []cli.Flag{
111117
&requestflag.Flag[string]{
@@ -132,7 +138,7 @@ var batchGetResults = cli.Command{
132138

133139
var batchSubmit = requestflag.WithInnerFlags(cli.Command{
134140
Name: "submit",
135-
Usage: "Scrape up to 25,000 URLs, or crawl a site, asynchronously. Poll the batch ID or\nreceive a webhook when it finishes.",
141+
Usage: "Scrape up to 100,000 URLs, or crawl a site, asynchronously using the same\nformats and scrape parameters as POST /web/scrape. Choose Markdown, HTML,\nscreenshots, images, bytes, parsed fields, products, highlights, and JSON. Poll\nthe batch ID or receive a webhook when it finishes.",
136142
Suggest: true,
137143
Flags: []cli.Flag{
138144
&requestflag.Flag[map[string]any]{

‎pkg/cmd/batch_test.go‎

Lines changed: 77 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func TestBatchList(t *testing.T) {
3232
"--limit", "1",
3333
"--q", "batch_1a2b",
3434
"--search-type", "exact",
35+
"--should-include-unified=true",
3536
"--status", "queued",
3637
"--tags", "docs,competitor",
3738
)
@@ -83,7 +84,7 @@ func TestBatchSubmit(t *testing.T) {
8384
t,
8485
"--api-key", "string",
8586
"batch", "submit",
86-
"--input", "{data: {format: markdown, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], options: {country: de, excludeSelectors: [x], includeHTML: true, includeImages: true, includeLinks: true, includeSelectors: [x], maxAgeMs: 0, pdf: {end: 1, ocr: true, shouldParse: true, start: 1}, settleAnimations: true, shortenBase64Images: true, useMainContentOnly: true, waitForMs: 0}}, mode: scrape}",
87+
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
8788
"--tag", "docs",
8889
"--tag", "competitor",
8990
"--webhook", "{url: https://example.com, retry: {delays_seconds: [10, 60, 300, 1800, 7200, 21600, 57600]}}",
@@ -101,7 +102,7 @@ func TestBatchSubmit(t *testing.T) {
101102
t,
102103
"--api-key", "string",
103104
"batch", "submit",
104-
"--input", "{data: {format: markdown, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], options: {country: de, excludeSelectors: [x], includeHTML: true, includeImages: true, includeLinks: true, includeSelectors: [x], maxAgeMs: 0, pdf: {end: 1, ocr: true, shouldParse: true, start: 1}, settleAnimations: true, shortenBase64Images: true, useMainContentOnly: true, waitForMs: 0}}, mode: scrape}",
105+
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
105106
"--tag", "docs",
106107
"--tag", "competitor",
107108
"--webhook.url", "https://example.com",
@@ -116,7 +117,16 @@ func TestBatchSubmit(t *testing.T) {
116117
pipeData := []byte("" +
117118
"input:\n" +
118119
" data:\n" +
119-
" format: markdown\n" +
120+
" formats:\n" +
121+
" bytes: true\n" +
122+
" highlights: true\n" +
123+
" html: true\n" +
124+
" images: true\n" +
125+
" json: true\n" +
126+
" markdown: true\n" +
127+
" parse: true\n" +
128+
" product: true\n" +
129+
" screenshot: true\n" +
120130
" urls:\n" +
121131
" - url: https://example.com/products/anvil\n" +
122132
" itemId: sku-1\n" +
@@ -126,25 +136,74 @@ func TestBatchSubmit(t *testing.T) {
126136
" itemId: sku-2\n" +
127137
" meta:\n" +
128138
" foo: bar\n" +
129-
" options:\n" +
130-
" country: de\n" +
131-
" excludeSelectors:\n" +
132-
" - x\n" +
133-
" includeHTML: true\n" +
139+
" highlightsParams:\n" +
140+
" query: x\n" +
141+
" maxCharacters: 100\n" +
142+
" imageParams:\n" +
143+
" dedupe: none\n" +
144+
" enrich:\n" +
145+
" - dimensions\n" +
146+
" jsonParams:\n" +
147+
" schema:\n" +
148+
" type: bar\n" +
149+
" properties: bar\n" +
150+
" required: bar\n" +
151+
" additionalProperties: bar\n" +
152+
" instructions: instructions\n" +
153+
" markdownParams:\n" +
134154
" includeImages: true\n" +
135155
" includeLinks: true\n" +
156+
" inlineImages: placeholder\n" +
157+
" maxAgeMs: 0\n" +
158+
" parseParams:\n" +
159+
" rules:\n" +
160+
" title: h1\n" +
161+
" links:\n" +
162+
" selector: a\n" +
163+
" output: '@href'\n" +
164+
" type: list\n" +
165+
" type: custom\n" +
166+
" productParams:\n" +
167+
" dedupeImages: true\n" +
168+
" screenshotParams:\n" +
169+
" area: viewport\n" +
170+
" format: png\n" +
171+
" sharedParams:\n" +
172+
" actions:\n" +
173+
" - action: Click the product details tab\n" +
174+
" type: perform\n" +
175+
" country: US\n" +
176+
" dismissCookies: true\n" +
177+
" dismissPopups: true\n" +
178+
" excludeSelectors:\n" +
179+
" - P\n" +
180+
" headers:\n" +
181+
" Accept-Language: en-US\n" +
182+
" includeFrames: true\n" +
136183
" includeSelectors:\n" +
137-
" - x\n" +
138-
" maxAgeMs: 0\n" +
139-
" pdf:\n" +
140-
" end: 1\n" +
141-
" ocr: true\n" +
142-
" shouldParse: true\n" +
143-
" start: 1\n" +
184+
" - P\n" +
185+
" mainContentOnly: true\n" +
186+
" parsers:\n" +
187+
" pdf:\n" +
188+
" endPage: 1\n" +
189+
" ocr: 'off'\n" +
190+
" startPage: 1\n" +
191+
" profile:\n" +
192+
" profileId: prof_210b9798eb53baa4e69d31c1071cf03d\n" +
193+
" type: existing\n" +
144194
" settleAnimations: true\n" +
145-
" shortenBase64Images: true\n" +
146-
" useMainContentOnly: true\n" +
147-
" waitForMs: 0\n" +
195+
" theme: light\n" +
196+
" viewport:\n" +
197+
" height: 240\n" +
198+
" width: 240\n" +
199+
" waitFor: 500\n" +
200+
" tags:\n" +
201+
" - production\n" +
202+
" - team-alpha\n" +
203+
" timeoutOpts:\n" +
204+
" milliseconds: 1\n" +
205+
" behavior: fail\n" +
206+
" zdr: disabled\n" +
148207
" mode: scrape\n" +
149208
"tags:\n" +
150209
" - docs\n" +

0 commit comments

Comments
 (0)