Skip to content

Commit 94f185d

Browse files
author
stlc-bot
committed
feat(batch): accept lightweight submissions and queue pending work (#2683)
Stainless-Generated-From: de2b398a1c49ed57ff98d246ac7c3d271b53a058
1 parent c60a6f0 commit 94f185d

14 files changed

Lines changed: 185 additions & 44 deletions

‎lib/context_dev/models/batch_list_params.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ module SearchType
8888
module Status
8989
extend ContextDev::Internal::Type::Enum
9090

91+
PENDING = :pending
9192
QUEUED = :queued
9293
RUNNING = :running
9394
CANCELLING = :cancelling

‎lib/context_dev/models/batch_list_response.rb‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,9 @@ class Data < ContextDev::Internal::Type::BaseModel
114114
required :results, -> { ContextDev::Models::BatchListResponse::Data::Results }, nil?: true
115115

116116
# @!attribute status
117-
# Current state. `completed`, `cancelled`, and `failed` are final.
117+
# Current state. `pending` means waiting for execution capacity or background
118+
# validation. Capacity-limited batches remain pending until a slot opens.
119+
# `completed`, `cancelled`, and `failed` are final.
118120
#
119121
# @return [Symbol, ContextDev::Models::BatchListResponse::Data::Status]
120122
required :status, enum: -> { ContextDev::Models::BatchListResponse::Data::Status }
@@ -163,7 +165,7 @@ class Data < ContextDev::Internal::Type::BaseModel
163165
#
164166
# @param results [ContextDev::Models::BatchListResponse::Data::Results, nil] Result download links; null until the batch finishes. Files are deleted 180 days
165167
#
166-
# @param status [Symbol, ContextDev::Models::BatchListResponse::Data::Status] Current state. `completed`, `cancelled`, and `failed` are final.
168+
# @param status [Symbol, ContextDev::Models::BatchListResponse::Data::Status] Current state. `pending` means waiting for execution capacity or background vali
167169
#
168170
# @param tags [Array<String>] Tags stored on the batch at submission.
169171
#
@@ -192,12 +194,16 @@ class Credits < ContextDev::Internal::Type::BaseModel
192194
required :refunded, Integer
193195

194196
# @!attribute reserved
195-
# Credits held when the batch was accepted.
197+
# Credits held by background preparation before scraping starts. Zero while
198+
# pending.
196199
#
197200
# @return [Integer]
198201
required :reserved, Integer
199202

200203
# @!method initialize(net:, ocr_charged:, refunded:, reserved:)
204+
# Some parameter documentations has been truncated, see
205+
# {ContextDev::Models::BatchListResponse::Data::Credits} for more details.
206+
#
201207
# Batch credit usage and settlement.
202208
#
203209
# @param net [Integer] `reserved` minus `refunded` plus `ocr_charged`.
@@ -206,7 +212,7 @@ class Credits < ContextDev::Internal::Type::BaseModel
206212
#
207213
# @param refunded [Integer] Credits returned for unsuccessful pages when the batch settles.
208214
#
209-
# @param reserved [Integer] Credits held when the batch was accepted.
215+
# @param reserved [Integer] Credits held by background preparation before scraping starts. Zero while pendin
210216
end
211217

212218
# @see ContextDev::Models::BatchListResponse::Data#formats
@@ -389,12 +395,15 @@ class File < ContextDev::Internal::Type::BaseModel
389395
end
390396
end
391397

392-
# Current state. `completed`, `cancelled`, and `failed` are final.
398+
# Current state. `pending` means waiting for execution capacity or background
399+
# validation. Capacity-limited batches remain pending until a slot opens.
400+
# `completed`, `cancelled`, and `failed` are final.
393401
#
394402
# @see ContextDev::Models::BatchListResponse::Data#status
395403
module Status
396404
extend ContextDev::Internal::Type::Enum
397405

406+
PENDING = :pending
398407
QUEUED = :queued
399408
RUNNING = :running
400409
CANCELLING = :cancelling

‎lib/context_dev/models/batch_retrieve_response.rb‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,9 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel
8282
required :results, -> { ContextDev::Models::BatchRetrieveResponse::Results }, nil?: true
8383

8484
# @!attribute status
85-
# Current state. `completed`, `cancelled`, and `failed` are final.
85+
# Current state. `pending` means waiting for execution capacity or background
86+
# validation. Capacity-limited batches remain pending until a slot opens.
87+
# `completed`, `cancelled`, and `failed` are final.
8688
#
8789
# @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Status]
8890
required :status, enum: -> { ContextDev::Models::BatchRetrieveResponse::Status }
@@ -145,7 +147,7 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel
145147
#
146148
# @param results [ContextDev::Models::BatchRetrieveResponse::Results, nil] Result download links; null until the batch finishes. Files are deleted 180 days
147149
#
148-
# @param status [Symbol, ContextDev::Models::BatchRetrieveResponse::Status] Current state. `completed`, `cancelled`, and `failed` are final.
150+
# @param status [Symbol, ContextDev::Models::BatchRetrieveResponse::Status] Current state. `pending` means waiting for execution capacity or background vali
149151
#
150152
# @param tags [Array<String>] Tags stored on the batch at submission.
151153
#
@@ -178,12 +180,16 @@ class Credits < ContextDev::Internal::Type::BaseModel
178180
required :refunded, Integer
179181

180182
# @!attribute reserved
181-
# Credits held when the batch was accepted.
183+
# Credits held by background preparation before scraping starts. Zero while
184+
# pending.
182185
#
183186
# @return [Integer]
184187
required :reserved, Integer
185188

186189
# @!method initialize(net:, ocr_charged:, refunded:, reserved:)
190+
# Some parameter documentations has been truncated, see
191+
# {ContextDev::Models::BatchRetrieveResponse::Credits} for more details.
192+
#
187193
# Batch credit usage and settlement.
188194
#
189195
# @param net [Integer] `reserved` minus `refunded` plus `ocr_charged`.
@@ -192,7 +198,7 @@ class Credits < ContextDev::Internal::Type::BaseModel
192198
#
193199
# @param refunded [Integer] Credits returned for unsuccessful pages when the batch settles.
194200
#
195-
# @param reserved [Integer] Credits held when the batch was accepted.
201+
# @param reserved [Integer] Credits held by background preparation before scraping starts. Zero while pendin
196202
end
197203

198204
# @see ContextDev::Models::BatchRetrieveResponse#formats
@@ -394,12 +400,15 @@ class File < ContextDev::Internal::Type::BaseModel
394400
end
395401
end
396402

397-
# Current state. `completed`, `cancelled`, and `failed` are final.
403+
# Current state. `pending` means waiting for execution capacity or background
404+
# validation. Capacity-limited batches remain pending until a slot opens.
405+
# `completed`, `cancelled`, and `failed` are final.
398406
#
399407
# @see ContextDev::Models::BatchRetrieveResponse#status
400408
module Status
401409
extend ContextDev::Internal::Type::Enum
402410

411+
PENDING = :pending
403412
QUEUED = :queued
404413
RUNNING = :running
405414
CANCELLING = :cancelling

‎lib/context_dev/models/batch_submit_response.rb‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class BatchSubmitResponse < ContextDev::Internal::Type::BaseModel
2929
required :created_at, String
3030

3131
# @!attribute credits
32-
# What accepting this batch cost.
32+
# Credits are reserved after background validation, before scraping starts.
3333
#
3434
# @return [ContextDev::Models::BatchSubmitResponse::Credits]
3535
required :credits, -> { ContextDev::Models::BatchSubmitResponse::Credits }
@@ -67,7 +67,9 @@ class BatchSubmitResponse < ContextDev::Internal::Type::BaseModel
6767
required :request_id, String
6868

6969
# @!attribute status
70-
# Always `queued`. An accepted batch has not started yet.
70+
# Current state. `pending` means waiting for execution capacity or background
71+
# validation. Capacity-limited batches remain pending until a slot opens.
72+
# `completed`, `cancelled`, and `failed` are final.
7173
#
7274
# @return [Symbol, ContextDev::Models::BatchSubmitResponse::Status]
7375
required :status, enum: -> { ContextDev::Models::BatchSubmitResponse::Status }
@@ -102,7 +104,7 @@ class BatchSubmitResponse < ContextDev::Internal::Type::BaseModel
102104
#
103105
# @param created_at [String] When the batch was created.
104106
#
105-
# @param credits [ContextDev::Models::BatchSubmitResponse::Credits] What accepting this batch cost.
107+
# @param credits [ContextDev::Models::BatchSubmitResponse::Credits] Credits are reserved after background validation, before scraping starts.
106108
#
107109
# @param formats [ContextDev::Models::BatchSubmitResponse::Formats] Outputs requested for each page. Matches input.data.formats on submission.
108110
#
@@ -114,7 +116,7 @@ class BatchSubmitResponse < ContextDev::Internal::Type::BaseModel
114116
#
115117
# @param request_id [String] Unique ID of this request, also in `X-Request-Id`. Include it when contacting su
116118
#
117-
# @param status [Symbol, ContextDev::Models::BatchSubmitResponse::Status] Always `queued`. An accepted batch has not started yet.
119+
# @param status [Symbol, ContextDev::Models::BatchSubmitResponse::Status] Current state. `pending` means waiting for execution capacity or background vali
118120
#
119121
# @param tags [Array<String>] Tags stored on the batch.
120122
#
@@ -166,15 +168,15 @@ module Status
166168
# @see ContextDev::Models::BatchSubmitResponse#credits
167169
class Credits < ContextDev::Internal::Type::BaseModel
168170
# @!attribute reserved
169-
# Credits held at submission.
171+
# Credits reserved by background preparation. Zero while pending.
170172
#
171173
# @return [Integer]
172174
required :reserved, Integer
173175

174176
# @!method initialize(reserved:)
175-
# What accepting this batch cost.
177+
# Credits are reserved after background validation, before scraping starts.
176178
#
177-
# @param reserved [Integer] Credits held at submission.
179+
# @param reserved [Integer] Credits reserved by background preparation. Zero while pending.
178180
end
179181

180182
# @see ContextDev::Models::BatchSubmitResponse#formats
@@ -291,13 +293,21 @@ module Mode
291293
# @return [Array<Symbol>]
292294
end
293295

294-
# Always `queued`. An accepted batch has not started yet.
296+
# Current state. `pending` means waiting for execution capacity or background
297+
# validation. Capacity-limited batches remain pending until a slot opens.
298+
# `completed`, `cancelled`, and `failed` are final.
295299
#
296300
# @see ContextDev::Models::BatchSubmitResponse#status
297301
module Status
298302
extend ContextDev::Internal::Type::Enum
299303

304+
PENDING = :pending
300305
QUEUED = :queued
306+
RUNNING = :running
307+
CANCELLING = :cancelling
308+
COMPLETED = :completed
309+
CANCELLED = :cancelled
310+
FAILED = :failed
301311

302312
# @!method self.values
303313
# @return [Array<Symbol>]

‎lib/context_dev/resources/batch.rb‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,15 @@ def get_results(batch_id, params = {})
142142
#
143143
# Scrape up to 100,000 URLs, or crawl a site, asynchronously using the same
144144
# formats and scrape parameters as POST /web/scrape. Choose Markdown, HTML,
145-
# screenshots, images, bytes, parsed fields, products, highlights, and JSON. Poll
146-
# the batch ID or receive a webhook when it finishes.
145+
# screenshots, images, bytes, parsed fields, products, highlights, and JSON.
146+
# Submission persists the input and returns immediately with status pending.
147+
# Organizations with fewer than 10 available credits and no overage are rejected
148+
# with 402 up front. Each organization can submit up to 1,000 batches per minute,
149+
# independently of its polling budget. Batches wait pending when execution
150+
# capacity is full. Validation, sitemap discovery, the full credit quote, and
151+
# shard scheduling happen afterward. Poll the batch ID or receive a webhook when
152+
# it finishes. Deferred rejections appear as status failed with a failure code and
153+
# message.
147154
#
148155
# @overload submit(input:, policies: nil, tags: nil, webhook: nil, webhook_url: nil, idempotency_key: nil, request_options: {})
149156
#

‎rbi/context_dev/models/batch_list_params.rbi‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,8 @@ module ContextDev
153153
T.type_alias { T.all(Symbol, ContextDev::BatchListParams::Status) }
154154
OrSymbol = T.type_alias { T.any(Symbol, String) }
155155

156+
PENDING =
157+
T.let(:pending, ContextDev::BatchListParams::Status::TaggedSymbol)
156158
QUEUED =
157159
T.let(:queued, ContextDev::BatchListParams::Status::TaggedSymbol)
158160
RUNNING =

‎rbi/context_dev/models/batch_list_response.rbi‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,9 @@ module ContextDev
198198
end
199199
attr_writer :results
200200

201-
# Current state. `completed`, `cancelled`, and `failed` are final.
201+
# Current state. `pending` means waiting for execution capacity or background
202+
# validation. Capacity-limited batches remain pending until a slot opens.
203+
# `completed`, `cancelled`, and `failed` are final.
202204
sig do
203205
returns(
204206
ContextDev::Models::BatchListResponse::Data::Status::TaggedSymbol
@@ -277,7 +279,9 @@ module ContextDev
277279
# Result download links; null until the batch finishes. Files are deleted 180 days
278280
# after the batch finishes.
279281
results:,
280-
# Current state. `completed`, `cancelled`, and `failed` are final.
282+
# Current state. `pending` means waiting for execution capacity or background
283+
# validation. Capacity-limited batches remain pending until a slot opens.
284+
# `completed`, `cancelled`, and `failed` are final.
281285
status:,
282286
# Tags stored on the batch at submission.
283287
tags:,
@@ -334,7 +338,8 @@ module ContextDev
334338
sig { returns(Integer) }
335339
attr_accessor :refunded
336340

337-
# Credits held when the batch was accepted.
341+
# Credits held by background preparation before scraping starts. Zero while
342+
# pending.
338343
sig { returns(Integer) }
339344
attr_accessor :reserved
340345

@@ -354,7 +359,8 @@ module ContextDev
354359
ocr_charged:,
355360
# Credits returned for unsuccessful pages when the batch settles.
356361
refunded:,
357-
# Credits held when the batch was accepted.
362+
# Credits held by background preparation before scraping starts. Zero while
363+
# pending.
358364
reserved:
359365
)
360366
end
@@ -684,7 +690,9 @@ module ContextDev
684690
end
685691
end
686692

687-
# Current state. `completed`, `cancelled`, and `failed` are final.
693+
# Current state. `pending` means waiting for execution capacity or background
694+
# validation. Capacity-limited batches remain pending until a slot opens.
695+
# `completed`, `cancelled`, and `failed` are final.
688696
module Status
689697
extend ContextDev::Internal::Type::Enum
690698

@@ -694,6 +702,11 @@ module ContextDev
694702
end
695703
OrSymbol = T.type_alias { T.any(Symbol, String) }
696704

705+
PENDING =
706+
T.let(
707+
:pending,
708+
ContextDev::Models::BatchListResponse::Data::Status::TaggedSymbol
709+
)
697710
QUEUED =
698711
T.let(
699712
:queued,

‎rbi/context_dev/models/batch_retrieve_response.rbi‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,9 @@ module ContextDev
109109
end
110110
attr_writer :results
111111

112-
# Current state. `completed`, `cancelled`, and `failed` are final.
112+
# Current state. `pending` means waiting for execution capacity or background
113+
# validation. Capacity-limited batches remain pending until a slot opens.
114+
# `completed`, `cancelled`, and `failed` are final.
113115
sig do
114116
returns(ContextDev::Models::BatchRetrieveResponse::Status::TaggedSymbol)
115117
end
@@ -217,7 +219,9 @@ module ContextDev
217219
# Result download links; null until the batch finishes. Files are deleted 180 days
218220
# after the batch finishes.
219221
results:,
220-
# Current state. `completed`, `cancelled`, and `failed` are final.
222+
# Current state. `pending` means waiting for execution capacity or background
223+
# validation. Capacity-limited batches remain pending until a slot opens.
224+
# `completed`, `cancelled`, and `failed` are final.
221225
status:,
222226
# Tags stored on the batch at submission.
223227
tags:,
@@ -283,7 +287,8 @@ module ContextDev
283287
sig { returns(Integer) }
284288
attr_accessor :refunded
285289

286-
# Credits held when the batch was accepted.
290+
# Credits held by background preparation before scraping starts. Zero while
291+
# pending.
287292
sig { returns(Integer) }
288293
attr_accessor :reserved
289294

@@ -303,7 +308,8 @@ module ContextDev
303308
ocr_charged:,
304309
# Credits returned for unsuccessful pages when the batch settles.
305310
refunded:,
306-
# Credits held when the batch was accepted.
311+
# Credits held by background preparation before scraping starts. Zero while
312+
# pending.
307313
reserved:
308314
)
309315
end
@@ -660,7 +666,9 @@ module ContextDev
660666
end
661667
end
662668

663-
# Current state. `completed`, `cancelled`, and `failed` are final.
669+
# Current state. `pending` means waiting for execution capacity or background
670+
# validation. Capacity-limited batches remain pending until a slot opens.
671+
# `completed`, `cancelled`, and `failed` are final.
664672
module Status
665673
extend ContextDev::Internal::Type::Enum
666674

@@ -670,6 +678,11 @@ module ContextDev
670678
end
671679
OrSymbol = T.type_alias { T.any(Symbol, String) }
672680

681+
PENDING =
682+
T.let(
683+
:pending,
684+
ContextDev::Models::BatchRetrieveResponse::Status::TaggedSymbol
685+
)
673686
QUEUED =
674687
T.let(
675688
:queued,

0 commit comments

Comments
 (0)