Merge pull request #2362 from unclecode/fix/cli-json-output-2334

Fail early when `-o json` has no extraction strategy

Author: ntohidi

crawl4ai/cli.py

@@ -175,7 +175,7 @@ def show_examples():
     crwl https://example.com -o markdown
 
     # Verbose JSON output with cache bypass
-    crwl https://example.com -o json -v --bypass-cache
+    crwl https://example.com -o all -v --bypass-cache
 
 2️⃣  Using Config Files:
     # Using browser and crawler configs
@@ -516,7 +516,7 @@ async def crawl_with_profile_cli(profile_path, url):
     # Ask for output format
     output_format = Prompt.ask(
         "[cyan]Output format[/cyan]",
-        choices=["all", "json", "markdown", "md", "title"],
+        choices=["all", "markdown", "md", "title"],
         default="markdown"
     )
 
@@ -1204,6 +1204,12 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
         else:
             ensure_ascii = config.get("JSON_ENSURE_ASCII", USER_SETTINGS["JSON_ENSURE_ASCII"]["default"])
 
+        if output == "json" and not question and crawler_cfg.extraction_strategy is None:
+            raise click.UsageError(
+                "-o json needs an extraction strategy (-j or -e). "
+                "Use -o all for the full crawl result."
+            )
+
         # Run crawler
         result : CrawlResult = anyio.run(
             run_crawler,
@@ -1242,7 +1248,6 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
                 else:
                     click.echo(json.dumps(main_result.model_dump(), indent=2, ensure_ascii=ensure_ascii))
             elif output == "json":
-                print(main_result.extracted_content)
                 extracted_items = json.loads(main_result.extracted_content)
                 click.echo(json.dumps(extracted_items, indent=2, ensure_ascii=ensure_ascii))
 
@@ -1292,6 +1297,8 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
                     else:
                         f.write(main_result.markdown.fit_markdown)
 
+    except click.ClickException:
+        raise
     except Exception as e:
         raise click.ClickException(str(e))
 

tests/cli/test_cli.py

@@ -253,5 +253,20 @@ def test_single_crawl_markdown_output_unchanged(self, runner):
             assert 'Content here' in result.output
 
 
+def test_json_output_without_extractor_fails_early(runner):
+    with patch("crawl4ai.cli.anyio.run") as run:
+        result = runner.invoke(cli, ["crawl", "https://example.com", "-o", "json"])
+    assert result.exit_code == 2
+    assert "-o json needs an extraction strategy" in result.output
+    run.assert_not_called()  # browser never started
+
+
+def test_json_output_with_question_is_allowed(runner):
+    with patch("crawl4ai.cli.anyio.run") as run, \
+         patch("crawl4ai.cli.setup_llm_config", return_value=("openai/gpt-4o", "x")):
+        runner.invoke(cli, ["crawl", "https://example.com", "-o", "json", "-q", "hi"])
+    run.assert_called()  # no early error
+
+
 if __name__ == '__main__':
     pytest.main(['-v', '-s', '--tb=native', __file__])