2323 BrowserConfig ,
2424 MemoryAdaptiveDispatcher ,
2525 RateLimiter ,
26- LLMConfig
26+ LLMConfig ,
27+ CrawlResult ,
2728)
2829from crawl4ai .async_configs import Provenance , UntrustedConfigError
2930from hook_registry import build_declarative_hooks , HookValidationError
@@ -647,14 +648,26 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
647648 await _dispose_crawler (crawler )
648649
649650
650- def _normalize_and_validate_seeds (urls : List [str ]) -> List [str ]:
651+ def _normalize_and_validate_seeds (urls : List [str ]) -> Tuple [ List [str ], List [ CrawlResult ] ]:
651652 """Prefix bare hosts with https:// and SSRF-validate every seed URL's
652653 destination. Shared by the streaming and non-streaming crawl handlers so a
653- new entry point cannot silently skip the destination check."""
654+ new entry point cannot silently skip the destination check.
655+
656+ Returns (allowed, refused): a refused seed becomes a failed CrawlResult so
657+ one bad seed does not sink the batch. If no seed is allowed, the first
658+ refusal is raised (400), as before."""
654659 urls = [('https://' + url ) if not url .startswith (('http://' , 'https://' )) and not url .startswith (("raw:" , "raw://" )) else url for url in urls ]
660+ allowed , refused , first_error = [], [], None
655661 for url in urls :
656- validate_url_destination (url )
657- return urls
662+ try :
663+ validate_url_destination (url )
664+ allowed .append (url )
665+ except HTTPException as e :
666+ first_error = first_error or e
667+ refused .append (CrawlResult (url = url , html = "" , success = False , error_message = e .detail ))
668+ if first_error and not allowed :
669+ raise first_error
670+ return allowed , refused
658671
659672
660673async def handle_crawl_request (
@@ -684,7 +697,7 @@ async def handle_crawl_request(
684697 peak_mem_mb = start_mem_mb
685698
686699 try :
687- urls = _normalize_and_validate_seeds (urls )
700+ urls , refused = _normalize_and_validate_seeds (urls )
688701 browser_config = BrowserConfig .load (browser_config , provenance = Provenance .UNTRUSTED )
689702 crawler_config = CrawlerRunConfig .load (crawler_config , provenance = Provenance .UNTRUSTED )
690703 from egress_broker import enforce_egress
@@ -762,6 +775,7 @@ async def handle_crawl_request(
762775 # Ensure results is always a list
763776 if not isinstance (results , list ):
764777 results = [results ]
778+ results = refused + results
765779
766780 end_mem_mb = _get_memory_mb () # <--- Get memory after
767781 end_time = time .time ()
@@ -886,6 +900,12 @@ async def handle_crawl_request(
886900 else :
887901 await release_crawler (crawler )
888902
903+ async def _prepend_results (first : List [CrawlResult ], results_gen : AsyncGenerator ) -> AsyncGenerator :
904+ for result in first :
905+ yield result
906+ async for result in results_gen :
907+ yield result
908+
889909async def handle_stream_crawl_request (
890910 urls : List [str ],
891911 browser_config : dict ,
@@ -900,7 +920,7 @@ async def handle_stream_crawl_request(
900920 # SSRF guard: validate every seed URL's destination before fetching,
901921 # mirroring handle_crawl_request. The streaming path previously skipped
902922 # this, leaving /crawl/stream (and /crawl with stream=true) unguarded.
903- urls = _normalize_and_validate_seeds (urls )
923+ urls , refused = _normalize_and_validate_seeds (urls )
904924 browser_config = BrowserConfig .load (
905925 browser_config , provenance = Provenance .UNTRUSTED
906926 )
@@ -918,12 +938,12 @@ async def handle_stream_crawl_request(
918938 crawler_config .stream = True
919939
920940 # Deep crawl streaming supports exactly one start URL
921- if crawler_config .deep_crawl_strategy is not None and len (urls ) != 1 :
941+ if crawler_config .deep_crawl_strategy is not None and len (urls ) + len ( refused ) != 1 :
922942 raise HTTPException (
923943 status_code = status .HTTP_400_BAD_REQUEST ,
924944 detail = (
925945 "Deep crawling with stream currently supports exactly one URL per request. "
926- f"Received { len (urls )} URLs."
946+ f"Received { len (urls ) + len ( refused ) } URLs."
927947 ),
928948 )
929949
@@ -968,6 +988,9 @@ async def handle_stream_crawl_request(
968988 dispatcher = dispatcher
969989 )
970990
991+ if refused :
992+ results_gen = _prepend_results (refused , results_gen )
993+
971994 return crawler , results_gen , hooks_info
972995
973996 except (UntrustedConfigError , HookValidationError ) as e :
0 commit comments