feat(parse): support WARC archives in parsing and scraping (#2761)

Stainless-Generated-From: cbff604d9a9dd49f3b0ee9d1f64da1867400fc56

Author: stlc-bot

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/charmbracelet/bubbletea v1.3.6
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/charmbracelet/x/term v0.2.1
-	github.com/context-dot-dev/context-go-sdk/v2 v2.45.0
+	github.com/context-dot-dev/context-go-sdk/v2 v2.46.0
 	github.com/goccy/go-yaml v1.18.0
 	github.com/itchyny/json2yaml v0.1.4
 	github.com/muesli/reflow v0.3.0

go.sum

@@ -18,8 +18,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR
 github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
 github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
 github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
-github.com/context-dot-dev/context-go-sdk/v2 v2.45.0 h1:qB8a9EhMOwzeRqwhJIfe4L924vtmMN7QF9/ajA3zexk=
-github.com/context-dot-dev/context-go-sdk/v2 v2.45.0/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8=
+github.com/context-dot-dev/context-go-sdk/v2 v2.46.0 h1:eoNQ/cNKkDtUut9xvRbAWfFZCzLlrgbdwJlBQ+ca0N4=
+github.com/context-dot-dev/context-go-sdk/v2 v2.46.0/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

pkg/cmd/parse.go

@@ -18,7 +18,7 @@ import (
 
 var parseHandle = requestflag.WithInnerFlags(cli.Command{
 	Name:    "handle",
-	Usage:   "Convert uploaded file bytes into Markdown and optional HTML.",
+	Usage:   "Convert uploaded file bytes into Markdown and optional HTML. WARC and WARC.GZ\narchives combine HTTP response, resource, and conversion records in archive\norder. Revisit and segmented records are skipped. Decoded WARC content is\nlimited to 50 MiB.",
 	Suggest: true,
 	Flags: []cli.Flag{
 		&requestflag.Flag[string]{
@@ -34,7 +34,7 @@ var parseHandle = requestflag.WithInnerFlags(cli.Command{
 		},
 		&requestflag.Flag[string]{
 			Name:      "extension",
-			Usage:     "Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, md, py, rtf, jpg, png, or txt.",
+			Usage:     "Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, md, py, rtf, jpg, png, txt, warc, or warc.gz.",
 			QueryPath: "extension",
 		},
 		&requestflag.Flag[bool]{

pkg/cmd/parse_test.go

@@ -18,7 +18,7 @@ func TestParseHandle(t *testing.T) {
 			"parse", "handle",
 			"--body", mocktest.TestFile(t, "Example data"),
 			"--client", "x",
-			"--extension", "txt",
+			"--extension", "warc",
 			"--include-images=true",
 			"--include-links=true",
 			"--include-numbered-pages=true",
@@ -43,7 +43,7 @@ func TestParseHandle(t *testing.T) {
 			"parse", "handle",
 			"--body", mocktest.TestFile(t, "Example data"),
 			"--client", "x",
-			"--extension", "txt",
+			"--extension", "warc",
 			"--include-images=true",
 			"--include-links=true",
 			"--include-numbered-pages=true",
@@ -66,7 +66,7 @@ func TestParseHandle(t *testing.T) {
 			"--api-key", "string",
 			"parse", "handle",
 			"--client", "x",
-			"--extension", "txt",
+			"--extension", "warc",
 			"--include-images=true",
 			"--include-links=true",
 			"--include-numbered-pages=true",