Merge pull request #2290 from talelboussetta/bugfix/crawler-configs-single-url-and-stream

fix(docker): apply crawler_configs to single-URL and streaming requests

Author: ntohidi

deploy/docker/README.md

@@ -658,6 +658,29 @@ async def test_stream_crawl(token: str = None): # Made token optional
 # asyncio.run(test_stream_crawl())
 ```
 
+#### Per-URL Configs (`crawler_configs`)
+
+To crawl different URLs with different settings in one request, send `crawler_configs`: a list of `CrawlerRunConfig` payloads, each with a `url_matcher`. Every URL uses the first entry whose `url_matcher` matches it, and an entry without a `url_matcher` matches every URL.
+
+```python
+payload = {
+    "urls": ["https://docs.example.com/guide", "https://example.com/"],
+    "crawler_configs": [
+        {
+            "type": "CrawlerRunConfig",
+            "params": {"url_matcher": "*docs.example.com*", "css_selector": "main"},
+        },
+        # Catch-all: decides every URL the entries above did not match
+        {"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}},
+    ],
+}
+```
+
+- The list is used for one URL or many, on both `/crawl` and `/crawl/stream`.
+- When a list is sent, `crawler_config` is ignored. It is not a fallback: a URL that no entry matches comes back as a failed result (`No matching configuration found for URL: ...`), so end the list with a catch-all entry (`"url_matcher": "*"`).
+- The server's `crawler.base_config` defaults from `config.yml` are applied to each entry on both endpoints.
+- A list that mixes `PDFContentScrapingStrategy` entries with other scraping strategies is refused with `400`, because one crawler serves every URL of a request.
+
 ### Asynchronous Jobs with Webhooks
 
 For long-running crawls or when you want to avoid keeping connections open, use the job queue endpoints. Instead of polling for results, configure a webhook to receive notifications when jobs complete.

deploy/docker/api.py

@@ -648,6 +648,57 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
             await _dispose_crawler(crawler)
 
 
+def _load_crawler_configs(
+    crawler_configs: List[dict],
+    base_config: Optional[dict] = None,
+    stream: bool = False,
+) -> List[CrawlerRunConfig]:
+    """Deserialize a per-URL crawler_configs list under the untrusted gate.
+    Shared by the streaming and non-streaming crawl handlers so both apply the
+    same boundary and wire the PDF URL validator into every entry."""
+    from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+    config_list = [
+        CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED)
+        for cc in crawler_configs
+    ]
+    for cfg in config_list:
+        for key, value in (base_config or {}).items():
+            if hasattr(cfg, key):
+                current_value = getattr(cfg, key)
+                if current_value is None or current_value == "":
+                    setattr(cfg, key, value)
+        # SSRF: per-URL PDF strategies need the validator wired too
+        if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
+            cfg.scraping_strategy.url_validator = validate_url_destination
+        if stream:
+            # arun_many takes the stream flag from the first config of a list.
+            cfg.stream = True
+    return config_list
+
+
+def _needs_pdf_crawler(
+    crawler_config: CrawlerRunConfig,
+    config_list: Optional[List[CrawlerRunConfig]] = None,
+) -> bool:
+    """Whether the crawl runs on PDFCrawlerStrategy. A per-URL list decides when
+    present, since it is what arun_many runs. One crawler serves every URL of a
+    request, so a list mixing PDF and browser entries is refused rather than run
+    on the wrong one."""
+    from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+    configs = config_list or [crawler_config]
+    pdf = [
+        isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs
+    ]
+    if any(pdf) and not all(pdf):
+        raise HTTPException(
+            status_code=400,
+            detail="crawler_configs cannot mix PDFContentScrapingStrategy with other scraping strategies in one request",
+        )
+    return all(pdf)
+
+
 def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:
     """Prefix bare hosts with https:// and SSRF-validate every seed URL's
     destination. Shared by the streaming and non-streaming crawl handlers so a
@@ -712,42 +763,39 @@ async def handle_crawl_request(
             ) if config["crawler"]["rate_limiter"]["enabled"] else None
         )
 
+        base_config = config["crawler"]["base_config"]
+        # Per-URL config list: deserialize each and apply base_config. Loaded
+        # before a crawler is chosen, because the list decides which one runs.
+        config_list = (
+            _load_crawler_configs(crawler_configs, base_config)
+            if crawler_configs
+            else None
+        )
+
         from crawler_pool import get_crawler, release_crawler
         from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
-        is_pdf_crawl = isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy)
+        is_pdf_crawl = _needs_pdf_crawler(crawler_config, config_list)
         if is_pdf_crawl:
             if hooks_config:
                 # PDFCrawlerStrategy has no browser page, so hooks can't attach to it
                 raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
             # SSRF protection: vet the PDF download URL and every redirect hop
-            crawler_config.scraping_strategy.url_validator = validate_url_destination
+            if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+                crawler_config.scraping_strategy.url_validator = validate_url_destination
             # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
             crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
             await crawler.start()
         else:
             crawler = await get_crawler(browser_config)
-        
+
         # Attach declarative hooks if provided
         hooks_status = {}
         if hooks_config:
             hooks_status = _attach_declarative_hooks(crawler, hooks_config)
             logger.info(f"Hooks attachment status: {hooks_status['status']}")
-        
-        base_config = config["crawler"]["base_config"]
 
         # Build the config(s) to pass to arun/arun_many
-        if crawler_configs and len(urls) > 1:
-            # Per-URL config list: deserialize each and apply base_config
-            config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
-            for cfg in config_list:
-                for key, value in base_config.items():
-                    if hasattr(cfg, key):
-                        current_value = getattr(cfg, key)
-                        if current_value is None or current_value == "":
-                            setattr(cfg, key, value)
-                # SSRF: per-URL PDF strategies need the validator wired too
-                if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
-                    cfg.scraping_strategy.url_validator = validate_url_destination
+        if config_list:
             effective_config = config_list
         else:
             # Single config (original behavior)
@@ -759,9 +807,12 @@ async def handle_crawl_request(
             effective_config = crawler_config
 
         results = []
-        func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
+        # arun takes one config, so a per-URL list goes to arun_many even for
+        # a single URL; arun_many matches each URL against the list.
+        use_many = len(urls) > 1 or isinstance(effective_config, list)
+        func = getattr(crawler, "arun_many" if use_many else "arun")
         partial_func = partial(func,
-                                urls[0] if len(urls) == 1 else urls,
+                                urls if use_many else urls[0],
                                 config=effective_config,
                                 dispatcher=dispatcher)
         # Optional per-crawl wall-clock deadline (config limits.wall_clock_s; 0 = none).
@@ -911,7 +962,8 @@ async def handle_stream_crawl_request(
     browser_config: dict,
     crawler_config: dict,
     config: dict,
-    hooks_config: Optional[dict] = None
+    hooks_config: Optional[dict] = None,
+    crawler_configs: Optional[List[dict]] = None,
 ) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]:
     """Handle streaming crawl requests with optional hooks."""
     hooks_info = None
@@ -936,6 +988,13 @@ async def handle_stream_crawl_request(
 
         clamp_deep_crawl(crawler_config)
         crawler_config.stream = True
+        config_list = (
+            _load_crawler_configs(
+                crawler_configs, config["crawler"]["base_config"], stream=True
+            )
+            if crawler_configs
+            else None
+        )
 
         # Deep crawl streaming supports exactly one start URL
         if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1:
@@ -949,12 +1008,13 @@ async def handle_stream_crawl_request(
 
         from crawler_pool import get_crawler
         from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
-        if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+        if _needs_pdf_crawler(crawler_config, config_list):
             if hooks_config:
                 # PDFCrawlerStrategy has no browser page, so hooks can't attach to it
                 raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
             # SSRF protection: vet the PDF download URL and every redirect hop
-            crawler_config.scraping_strategy.url_validator = validate_url_destination
+            if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+                crawler_config.scraping_strategy.url_validator = validate_url_destination
             # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
             crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
             await crawler.start()
@@ -984,7 +1044,7 @@ async def handle_stream_crawl_request(
             )
             results_gen = await crawler.arun_many(
                 urls=urls,
-                config=crawler_config,
+                config=config_list or crawler_config,
                 dispatcher=dispatcher
             )
 

deploy/docker/server.py

@@ -1032,7 +1032,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
         browser_config=crawl_request.browser_config,
         crawler_config=crawl_request.crawler_config,
         config=config,
-        hooks_config=hooks_config
+        hooks_config=hooks_config,
+        crawler_configs=crawl_request.crawler_configs,
     )
 
     # Add hooks info to response headers if available