Skip to content

Commit 3e12a96

Browse files
authored
Merge pull request #2359 from unclecode/fix/2288-per-seed-refusal
fix(docker): report a refused seed as a failed result, not a dead batch
2 parents 7e3518e + 350a141 commit 3e12a96

2 files changed

Lines changed: 113 additions & 9 deletions

File tree

‎deploy/docker/api.py‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
BrowserConfig,
2424
MemoryAdaptiveDispatcher,
2525
RateLimiter,
26-
LLMConfig
26+
LLMConfig,
27+
CrawlResult,
2728
)
2829
from crawl4ai.async_configs import Provenance, UntrustedConfigError
2930
from hook_registry import build_declarative_hooks, HookValidationError
@@ -647,14 +648,26 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
647648
await _dispose_crawler(crawler)
648649

649650

650-
def _normalize_and_validate_seeds(urls: List[str]) -> List[str]:
651+
def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:
651652
"""Prefix bare hosts with https:// and SSRF-validate every seed URL's
652653
destination. Shared by the streaming and non-streaming crawl handlers so a
653-
new entry point cannot silently skip the destination check."""
654+
new entry point cannot silently skip the destination check.
655+
656+
Returns (allowed, refused): a refused seed becomes a failed CrawlResult so
657+
one bad seed does not sink the batch. If no seed is allowed, the first
658+
refusal is raised (400), as before."""
654659
urls = [('https://' + url) if not url.startswith(('http://', 'https://')) and not url.startswith(("raw:", "raw://")) else url for url in urls]
660+
allowed, refused, first_error = [], [], None
655661
for url in urls:
656-
validate_url_destination(url)
657-
return urls
662+
try:
663+
validate_url_destination(url)
664+
allowed.append(url)
665+
except HTTPException as e:
666+
first_error = first_error or e
667+
refused.append(CrawlResult(url=url, html="", success=False, error_message=e.detail))
668+
if first_error and not allowed:
669+
raise first_error
670+
return allowed, refused
658671

659672

660673
async def handle_crawl_request(
@@ -684,7 +697,7 @@ async def handle_crawl_request(
684697
peak_mem_mb = start_mem_mb
685698

686699
try:
687-
urls = _normalize_and_validate_seeds(urls)
700+
urls, refused = _normalize_and_validate_seeds(urls)
688701
browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)
689702
crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
690703
from egress_broker import enforce_egress
@@ -762,6 +775,7 @@ async def handle_crawl_request(
762775
# Ensure results is always a list
763776
if not isinstance(results, list):
764777
results = [results]
778+
results = refused + results
765779

766780
end_mem_mb = _get_memory_mb() # <--- Get memory after
767781
end_time = time.time()
@@ -886,6 +900,12 @@ async def handle_crawl_request(
886900
else:
887901
await release_crawler(crawler)
888902

903+
async def _prepend_results(first: List[CrawlResult], results_gen: AsyncGenerator) -> AsyncGenerator:
904+
for result in first:
905+
yield result
906+
async for result in results_gen:
907+
yield result
908+
889909
async def handle_stream_crawl_request(
890910
urls: List[str],
891911
browser_config: dict,
@@ -900,7 +920,7 @@ async def handle_stream_crawl_request(
900920
# SSRF guard: validate every seed URL's destination before fetching,
901921
# mirroring handle_crawl_request. The streaming path previously skipped
902922
# this, leaving /crawl/stream (and /crawl with stream=true) unguarded.
903-
urls = _normalize_and_validate_seeds(urls)
923+
urls, refused = _normalize_and_validate_seeds(urls)
904924
browser_config = BrowserConfig.load(
905925
browser_config, provenance=Provenance.UNTRUSTED
906926
)
@@ -918,12 +938,12 @@ async def handle_stream_crawl_request(
918938
crawler_config.stream = True
919939

920940
# Deep crawl streaming supports exactly one start URL
921-
if crawler_config.deep_crawl_strategy is not None and len(urls) != 1:
941+
if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1:
922942
raise HTTPException(
923943
status_code=status.HTTP_400_BAD_REQUEST,
924944
detail=(
925945
"Deep crawling with stream currently supports exactly one URL per request. "
926-
f"Received {len(urls)} URLs."
946+
f"Received {len(urls) + len(refused)} URLs."
927947
),
928948
)
929949

@@ -968,6 +988,9 @@ async def handle_stream_crawl_request(
968988
dispatcher=dispatcher
969989
)
970990

991+
if refused:
992+
results_gen = _prepend_results(refused, results_gen)
993+
971994
return crawler, results_gen, hooks_info
972995

973996
except (UntrustedConfigError, HookValidationError) as e:
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
"""#2288: one refused seed must not sink the whole /crawl or /crawl/stream batch."""
2+
import json
3+
4+
import pytest
5+
from fastapi import HTTPException
6+
7+
from crawl4ai import CrawlResult
8+
9+
GOOD = "https://example.com/"
10+
BAD = "https://no-such-host.example/"
11+
12+
13+
@pytest.fixture
14+
def crawled(monkeypatch):
15+
"""Refuse BAD, fake the crawler, and record which URLs reached it."""
16+
import api
17+
import crawler_pool
18+
19+
seen = []
20+
21+
def validate(url):
22+
if url == BAD:
23+
raise HTTPException(400, "URL blocked (SSRF protection): URL blocked")
24+
25+
class FakeCrawler:
26+
async def arun(self, url, **kwargs):
27+
seen.append(url)
28+
return CrawlResult(url=url, html="ok", success=True)
29+
30+
async def arun_many(self, urls, config=None, **kwargs):
31+
seen.extend(urls)
32+
results = [CrawlResult(url=u, html="ok", success=True) for u in urls]
33+
if not config.stream:
34+
return results
35+
36+
async def gen():
37+
for r in results:
38+
yield r
39+
return gen()
40+
41+
async def get_crawler(*args, **kwargs):
42+
return FakeCrawler()
43+
44+
async def release_crawler(*args, **kwargs):
45+
return None
46+
47+
monkeypatch.setattr(api, "validate_url_destination", validate)
48+
monkeypatch.setattr(crawler_pool, "get_crawler", get_crawler)
49+
monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)
50+
return seen
51+
52+
53+
def _auth():
54+
from auth import create_access_token
55+
return {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
56+
57+
58+
def test_batch_crawls_allowed_seeds_and_reports_refused(stock_client, crawled):
59+
r = stock_client.post("/crawl", json={"urls": [GOOD, BAD]}, headers=_auth())
60+
assert r.status_code == 200, r.text
61+
by_url = {res["url"]: res for res in r.json()["results"]}
62+
assert by_url[GOOD]["success"] is True
63+
assert by_url[BAD]["success"] is False
64+
assert by_url[BAD]["error_message"] == "URL blocked (SSRF protection): URL blocked"
65+
assert crawled == [GOOD]
66+
67+
68+
def test_stream_crawls_allowed_seeds_and_reports_refused(stock_client, crawled):
69+
r = stock_client.post("/crawl/stream", json={"urls": [GOOD, BAD, GOOD]}, headers=_auth())
70+
assert r.status_code == 200, r.text
71+
lines = [json.loads(line) for line in r.text.splitlines() if line]
72+
results = [l for l in lines if "url" in l]
73+
assert [(l["url"], l["success"]) for l in results] == [(BAD, False), (GOOD, True), (GOOD, True)]
74+
assert crawled == [GOOD, GOOD]
75+
76+
77+
@pytest.mark.parametrize("path", ["/crawl", "/crawl/stream"])
78+
def test_all_refused_is_still_400(stock_client, crawled, path):
79+
r = stock_client.post(path, json={"urls": [BAD, BAD]}, headers=_auth())
80+
assert r.status_code == 400
81+
assert crawled == []

0 commit comments

Comments
 (0)