From 81efcc35f00a9d63416bc5d8a97c5c70ee7cfbe3 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Fri, 25 Sep 2026 10:29:47 +0100 Subject: [PATCH 1/5] fix(docker): apply crawler_configs to single-URL and streaming requests The per-URL config list (#1837) was only honoured when a non-streaming request carried two or more URLs. With one URL the handler called arun(), which takes a single config, and /crawl/stream (and /crawl with stream=true) never passed the list to handle_stream_crawl_request. Both requests ran with crawler_config alone and still answered 200. A request that carries a list now goes to arun_many whatever its URL count, and the streaming handler takes and applies the list. Loading moves into _load_crawler_configs(), shared by both handlers the way _normalize_and_validate_seeds() is, so both apply the same trust boundary and wire the PDF URL validator into every entry. On the streaming path each entry gets stream=True, since arun_many reads the stream flag from the first config of a list. The legacy source-grep test that pinned the single-URL path to arun() is updated to the new routing; behavioural tests are in deploy/docker/tests/test_crawler_configs_routing.py. --- deploy/docker/api.py | 55 +++++--- deploy/docker/server.py | 3 +- .../tests/test_crawler_configs_routing.py | 131 ++++++++++++++++++ tests/test_issue_1837_config_list.py | 14 +- 4 files changed, 180 insertions(+), 23 deletions(-) create mode 100644 deploy/docker/tests/test_crawler_configs_routing.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index b03841e7b..ab1f1aac1 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -648,6 +648,32 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) await _dispose_crawler(crawler) +def _load_crawler_configs( + crawler_configs: List[dict], + base_config: Optional[dict] = None, + stream: bool = False, +) -> List[CrawlerRunConfig]: + """Deserialize a per-URL crawler_configs list under the untrusted gate. + Shared by the streaming and non-streaming crawl handlers so both apply the + same boundary and wire the PDF URL validator into every entry.""" + from crawl4ai.processors.pdf import PDFContentScrapingStrategy + + config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs] + for cfg in config_list: + for key, value in (base_config or {}).items(): + if hasattr(cfg, key): + current_value = getattr(cfg, key) + if current_value is None or current_value == "": + setattr(cfg, key, value) + # SSRF: per-URL PDF strategies need the validator wired too + if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy): + cfg.scraping_strategy.url_validator = validate_url_destination + if stream: + # arun_many takes the stream flag from the first config of a list. + cfg.stream = True + return config_list + + def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]: """Prefix bare hosts with https:// and SSRF-validate every seed URL's destination. Shared by the streaming and non-streaming crawl handlers so a @@ -736,19 +762,9 @@ async def handle_crawl_request( base_config = config["crawler"]["base_config"] # Build the config(s) to pass to arun/arun_many - if crawler_configs and len(urls) > 1: + if crawler_configs: # Per-URL config list: deserialize each and apply base_config - config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs] - for cfg in config_list: - for key, value in base_config.items(): - if hasattr(cfg, key): - current_value = getattr(cfg, key) - if current_value is None or current_value == "": - setattr(cfg, key, value) - # SSRF: per-URL PDF strategies need the validator wired too - if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy): - cfg.scraping_strategy.url_validator = validate_url_destination - effective_config = config_list + effective_config = _load_crawler_configs(crawler_configs, base_config) else: # Single config (original behavior) for key, value in base_config.items(): @@ -759,9 +775,12 @@ async def handle_crawl_request( effective_config = crawler_config results = [] - func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many") + # arun takes one config, so a per-URL list goes to arun_many even for + # a single URL; arun_many matches each URL against the list. + use_many = len(urls) > 1 or isinstance(effective_config, list) + func = getattr(crawler, "arun_many" if use_many else "arun") partial_func = partial(func, - urls[0] if len(urls) == 1 else urls, + urls if use_many else urls[0], config=effective_config, dispatcher=dispatcher) # Optional per-crawl wall-clock deadline (config limits.wall_clock_s; 0 = none). @@ -911,7 +930,8 @@ async def handle_stream_crawl_request( browser_config: dict, crawler_config: dict, config: dict, - hooks_config: Optional[dict] = None + hooks_config: Optional[dict] = None, + crawler_configs: Optional[List[dict]] = None, ) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]: """Handle streaming crawl requests with optional hooks.""" hooks_info = None @@ -936,6 +956,9 @@ async def handle_stream_crawl_request( clamp_deep_crawl(crawler_config) crawler_config.stream = True + config_list = ( + _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None + ) # Deep crawl streaming supports exactly one start URL if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1: @@ -984,7 +1007,7 @@ async def handle_stream_crawl_request( ) results_gen = await crawler.arun_many( urls=urls, - config=crawler_config, + config=config_list or crawler_config, dispatcher=dispatcher ) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 3d71a307f..5f08bae44 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -1032,7 +1032,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks): browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, - hooks_config=hooks_config + hooks_config=hooks_config, + crawler_configs=crawl_request.crawler_configs, ) # Add hooks info to response headers if available diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.py new file mode 100644 index 000000000..b97093dfc --- /dev/null +++ b/deploy/docker/tests/test_crawler_configs_routing.py @@ -0,0 +1,131 @@ +"""crawler_configs must reach arun_many for one URL and on the streaming path. + +The per-URL config list (#1837) was only applied when a non-streaming request +carried two or more URLs. With one URL the handler called arun(), which takes a +single config, and /crawl/stream never passed the list down, so both requests +ran with crawler_config alone and still answered HTTP 200. +""" + +import asyncio + +from crawl4ai import CrawlerRunConfig +from crawl4ai.models import CrawlResult + +PER_URL = [ + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*example.com*", "word_count_threshold": 7}, + }, +] + + +class RecordingCrawler: + def __init__(self): + self.calls = [] + + async def arun(self, url, config=None, **kwargs): + self.calls.append(("arun", url, config)) + return CrawlResult(url=url, html="", success=True) + + async def arun_many(self, urls, config=None, **kwargs): + self.calls.append(("arun_many", urls, config)) + return [CrawlResult(url=url, html="", success=True) for url in urls] + + +def _install(monkeypatch, crawler): + import api + import crawler_pool + + async def get_crawler(*args, **kwargs): + return crawler + + async def release_crawler(*args, **kwargs): + return None + + # Offline: the seed check resolves DNS. + monkeypatch.setattr(api, "validate_url_destination", lambda url: None) + monkeypatch.setattr(crawler_pool, "get_crawler", get_crawler) + monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler) + return api + + +def test_single_url_uses_the_config_list(server_module, monkeypatch): + crawler = RecordingCrawler() + api = _install(monkeypatch, crawler) + + asyncio.run( + api.handle_crawl_request( + urls=["https://example.com/"], + browser_config={}, + crawler_config={}, + config=server_module.config, + crawler_configs=PER_URL, + ) + ) + + [(method, urls, config)] = crawler.calls + assert method == "arun_many" + assert urls == ["https://example.com/"] + assert [c.word_count_threshold for c in config] == [7] + + +def test_single_url_without_a_list_still_uses_arun(server_module, monkeypatch): + crawler = RecordingCrawler() + api = _install(monkeypatch, crawler) + + asyncio.run( + api.handle_crawl_request( + urls=["https://example.com/"], + browser_config={}, + crawler_config={}, + config=server_module.config, + ) + ) + + [(method, url, config)] = crawler.calls + assert method == "arun" + assert url == "https://example.com/" + assert isinstance(config, CrawlerRunConfig) + + +def test_stream_passes_the_config_list(server_module, monkeypatch): + crawler = RecordingCrawler() + api = _install(monkeypatch, crawler) + + asyncio.run( + api.handle_stream_crawl_request( + urls=["https://example.com/", "https://example.org/"], + browser_config={}, + crawler_config={}, + config=server_module.config, + crawler_configs=PER_URL, + ) + ) + + [(method, urls, config)] = crawler.calls + assert method == "arun_many" + assert [c.word_count_threshold for c in config] == [7] + # arun_many takes the stream flag from the first config of a list. + assert all(c.stream for c in config) + + +def test_stream_endpoint_forwards_the_list(server_module, monkeypatch): + captured = {} + + async def fake_handle_stream(**kwargs): + captured.update(kwargs) + raise RuntimeError("stop after capture") + + monkeypatch.setattr( + server_module, "handle_stream_crawl_request", fake_handle_stream + ) + request = server_module.CrawlRequestWithHooks( + urls=["https://example.com/"], crawler_configs=PER_URL + ) + + try: + asyncio.run(server_module.stream_process(crawl_request=request)) + except RuntimeError: + pass + + assert captured["crawler_configs"] == PER_URL diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py index 45150e589..4a888e070 100644 --- a/tests/test_issue_1837_config_list.py +++ b/tests/test_issue_1837_config_list.py @@ -88,11 +88,11 @@ class TestConfigListLogic: """Verify the branching logic for single vs list configs.""" def test_api_uses_config_list_when_provided(self): - """When crawler_configs is provided with multiple URLs, it should be used.""" + """When crawler_configs is provided, it should be used, for one URL or many.""" with open("deploy/docker/api.py") as f: source = f.read() # Should check crawler_configs and build a list - assert "if crawler_configs and len(urls) > 1:" in source + assert "if crawler_configs:" in source assert "config_list" in source def test_api_falls_back_to_single_config(self): @@ -125,12 +125,14 @@ def test_server_passes_crawler_configs(self): class TestBackwardCompatibility: """Ensure existing single-config requests still work.""" - def test_single_url_ignores_crawler_configs(self): - """With a single URL, crawler_configs should be ignored (uses arun, not arun_many).""" + def test_single_url_with_config_list_uses_arun_many(self): + """A config list goes to arun_many even for a single URL (arun takes one config). + + Behavioral coverage: deploy/docker/tests/test_crawler_configs_routing.py. + """ with open("deploy/docker/api.py") as f: source = f.read() - # Single URL uses arun which only takes one config - assert '"arun" if len(urls) == 1 else "arun_many"' in source + assert "use_many = len(urls) > 1 or isinstance(effective_config, list)" in source def test_no_crawler_configs_uses_single(self): """When crawler_configs is None, the original single config path is used.""" From 5ce402f9df60897896c77ba45e3901bf5e2c0595 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Fri, 25 Sep 2026 11:28:36 +0100 Subject: [PATCH 2/5] fix(docker): let a per-URL config list decide the PDF crawler With crawler_configs applied, the crawler was still chosen from the top-level crawler_config alone, so a list entry using PDFContentScrapingStrategy ran on the pooled browser crawler and failed before the PDF scraper could run. _needs_pdf_crawler() decides from the list when there is one: a list of PDF entries runs on PDFCrawlerStrategy, and since one crawler serves every URL of a request, a list mixing PDF and browser entries is refused with 400 instead of failing mid-crawl. The non-streaming handler now loads the list before it picks a crawler, as the streaming one already did. --- deploy/docker/api.py | 44 +++++++--- .../tests/test_crawler_configs_routing.py | 82 +++++++++++++++++++ tests/test_issue_1837_config_list.py | 2 +- 3 files changed, 117 insertions(+), 11 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index ab1f1aac1..9259efee7 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -674,6 +674,26 @@ def _load_crawler_configs( return config_list +def _needs_pdf_crawler( + crawler_config: CrawlerRunConfig, + config_list: Optional[List[CrawlerRunConfig]] = None, +) -> bool: + """Whether the crawl runs on PDFCrawlerStrategy. A per-URL list decides when + present, since it is what arun_many runs. One crawler serves every URL of a + request, so a list mixing PDF and browser entries is refused rather than run + on the wrong one.""" + from crawl4ai.processors.pdf import PDFContentScrapingStrategy + + configs = config_list or [crawler_config] + pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs] + if any(pdf) and not all(pdf): + raise HTTPException( + status_code=400, + detail="crawler_configs cannot mix PDFContentScrapingStrategy with other scraping strategies in one request", + ) + return all(pdf) + + def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]: """Prefix bare hosts with https:// and SSRF-validate every seed URL's destination. Shared by the streaming and non-streaming crawl handlers so a @@ -738,33 +758,36 @@ async def handle_crawl_request( ) if config["crawler"]["rate_limiter"]["enabled"] else None ) + base_config = config["crawler"]["base_config"] + # Per-URL config list: deserialize each and apply base_config. Loaded + # before a crawler is chosen, because the list decides which one runs. + config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None + from crawler_pool import get_crawler, release_crawler from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy - is_pdf_crawl = isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy) + is_pdf_crawl = _needs_pdf_crawler(crawler_config, config_list) if is_pdf_crawl: if hooks_config: # PDFCrawlerStrategy has no browser page, so hooks can't attach to it raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") # SSRF protection: vet the PDF download URL and every redirect hop - crawler_config.scraping_strategy.url_validator = validate_url_destination + if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy): + crawler_config.scraping_strategy.url_validator = validate_url_destination # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) await crawler.start() else: crawler = await get_crawler(browser_config) - + # Attach declarative hooks if provided hooks_status = {} if hooks_config: hooks_status = _attach_declarative_hooks(crawler, hooks_config) logger.info(f"Hooks attachment status: {hooks_status['status']}") - - base_config = config["crawler"]["base_config"] # Build the config(s) to pass to arun/arun_many - if crawler_configs: - # Per-URL config list: deserialize each and apply base_config - effective_config = _load_crawler_configs(crawler_configs, base_config) + if config_list: + effective_config = config_list else: # Single config (original behavior) for key, value in base_config.items(): @@ -972,12 +995,13 @@ async def handle_stream_crawl_request( from crawler_pool import get_crawler from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy - if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy): + if _needs_pdf_crawler(crawler_config, config_list): if hooks_config: # PDFCrawlerStrategy has no browser page, so hooks can't attach to it raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") # SSRF protection: vet the PDF download URL and every redirect hop - crawler_config.scraping_strategy.url_validator = validate_url_destination + if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy): + crawler_config.scraping_strategy.url_validator = validate_url_destination # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) await crawler.start() diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.py index b97093dfc..65303b3dc 100644 --- a/deploy/docker/tests/test_crawler_configs_routing.py +++ b/deploy/docker/tests/test_crawler_configs_routing.py @@ -4,10 +4,17 @@ carried two or more URLs. With one URL the handler called arun(), which takes a single config, and /crawl/stream never passed the list down, so both requests ran with crawler_config alone and still answered HTTP 200. + +With the list applied, it also has to decide the crawler: a list of PDF entries +runs on PDFCrawlerStrategy, and one crawler cannot serve a list mixing PDF and +browser entries, so that is refused instead of failing mid-crawl. """ import asyncio +import pytest +from fastapi import HTTPException + from crawl4ai import CrawlerRunConfig from crawl4ai.models import CrawlResult @@ -129,3 +136,78 @@ async def fake_handle_stream(**kwargs): pass assert captured["crawler_configs"] == PER_URL + + +PDF = {"type": "PDFContentScrapingStrategy", "params": {}} +ALL_PDF = [ + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*.pdf", "scraping_strategy": PDF}, + }, +] +MIXED = ALL_PDF + [{"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}}] + + +class RecordingPdfCrawler(RecordingCrawler): + """Stands in for AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()).""" + + def __init__(self, crawler_strategy=None, **kwargs): + super().__init__() + self.crawler_strategy = crawler_strategy + + async def start(self): + return self + + async def close(self): + return None + + +def test_a_list_of_pdf_entries_runs_on_the_pdf_crawler(server_module, monkeypatch): + from crawl4ai.processors.pdf import PDFCrawlerStrategy + + api = _install(monkeypatch, RecordingCrawler()) + made = [] + + def pdf_crawler(**kwargs): + made.append(RecordingPdfCrawler(**kwargs)) + return made[-1] + + monkeypatch.setattr(api, "AsyncWebCrawler", pdf_crawler) + + asyncio.run( + api.handle_crawl_request( + urls=["https://example.com/a.pdf"], + browser_config={}, + crawler_config={}, + config=server_module.config, + crawler_configs=ALL_PDF, + ) + ) + + [crawler] = made + assert isinstance(crawler.crawler_strategy, PDFCrawlerStrategy) + [(method, urls, config)] = crawler.calls + assert method == "arun_many" + # SSRF: every per-URL PDF entry still gets the destination check. + assert config[0].scraping_strategy.url_validator is api.validate_url_destination + + +@pytest.mark.parametrize("stream", [False, True]) +def test_a_list_mixing_pdf_and_browser_entries_is_refused( + server_module, monkeypatch, stream +): + api = _install(monkeypatch, RecordingCrawler()) + handler = api.handle_stream_crawl_request if stream else api.handle_crawl_request + + with pytest.raises(HTTPException) as refused: + asyncio.run( + handler( + urls=["https://example.com/a.pdf", "https://example.com/"], + browser_config={}, + crawler_config={}, + config=server_module.config, + crawler_configs=MIXED, + ) + ) + + assert refused.value.status_code == 400 diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py index 4a888e070..0e5463bba 100644 --- a/tests/test_issue_1837_config_list.py +++ b/tests/test_issue_1837_config_list.py @@ -92,7 +92,7 @@ def test_api_uses_config_list_when_provided(self): with open("deploy/docker/api.py") as f: source = f.read() # Should check crawler_configs and build a list - assert "if crawler_configs:" in source + assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in source assert "config_list" in source def test_api_falls_back_to_single_config(self): From 4d645a41334d3e628f5eca066683d70852fdffa3 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Sat, 26 Sep 2026 16:59:11 +0100 Subject: [PATCH 3/5] style(docker): black-format the new crawler_configs code Wraps the new lines over black's limit. The source assertion in test_issue_1837_config_list.py now matches the call rather than the whole one-line conditional, so it no longer depends on how the line is wrapped. No behaviour change. --- deploy/docker/api.py | 19 +++++++++++++++---- tests/test_issue_1837_config_list.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 9259efee7..d9ce3ade2 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -658,7 +658,10 @@ def _load_crawler_configs( same boundary and wire the PDF URL validator into every entry.""" from crawl4ai.processors.pdf import PDFContentScrapingStrategy - config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs] + config_list = [ + CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) + for cc in crawler_configs + ] for cfg in config_list: for key, value in (base_config or {}).items(): if hasattr(cfg, key): @@ -685,7 +688,9 @@ def _needs_pdf_crawler( from crawl4ai.processors.pdf import PDFContentScrapingStrategy configs = config_list or [crawler_config] - pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs] + pdf = [ + isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs + ] if any(pdf) and not all(pdf): raise HTTPException( status_code=400, @@ -761,7 +766,11 @@ async def handle_crawl_request( base_config = config["crawler"]["base_config"] # Per-URL config list: deserialize each and apply base_config. Loaded # before a crawler is chosen, because the list decides which one runs. - config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None + config_list = ( + _load_crawler_configs(crawler_configs, base_config) + if crawler_configs + else None + ) from crawler_pool import get_crawler, release_crawler from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy @@ -980,7 +989,9 @@ async def handle_stream_crawl_request( clamp_deep_crawl(crawler_config) crawler_config.stream = True config_list = ( - _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None + _load_crawler_configs(crawler_configs, stream=True) + if crawler_configs + else None ) # Deep crawl streaming supports exactly one start URL diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py index 0e5463bba..0e017be92 100644 --- a/tests/test_issue_1837_config_list.py +++ b/tests/test_issue_1837_config_list.py @@ -92,7 +92,7 @@ def test_api_uses_config_list_when_provided(self): with open("deploy/docker/api.py") as f: source = f.read() # Should check crawler_configs and build a list - assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in source + assert "_load_crawler_configs(crawler_configs, base_config)" in source assert "config_list" in source def test_api_falls_back_to_single_config(self): From 974275d0675cf662d2668449eefa66010c618c6c Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Mon, 12 Oct 2026 12:05:14 +0100 Subject: [PATCH 4/5] fix(docker): apply base_config to the crawler_configs list on the streaming path handle_stream_crawl_request loaded the per-URL list without base_config, so the same list got different server defaults on /crawl/stream than on /crawl. It now passes config["crawler"]["base_config"], the way handle_crawl_request does, so both paths build the list the same way. test_base_config_fills_the_list_on_both_paths runs one list through both handlers: an entry that leaves a field unset gets base_config's value, one that sets it keeps its own. It fails on the streaming path without this change. --- deploy/docker/api.py | 4 +- .../tests/test_crawler_configs_routing.py | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index d9ce3ade2..244b5046f 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -989,7 +989,9 @@ async def handle_stream_crawl_request( clamp_deep_crawl(crawler_config) crawler_config.stream = True config_list = ( - _load_crawler_configs(crawler_configs, stream=True) + _load_crawler_configs( + crawler_configs, config["crawler"]["base_config"], stream=True + ) if crawler_configs else None ) diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.py index 65303b3dc..9f5336cfa 100644 --- a/deploy/docker/tests/test_crawler_configs_routing.py +++ b/deploy/docker/tests/test_crawler_configs_routing.py @@ -116,6 +116,46 @@ def test_stream_passes_the_config_list(server_module, monkeypatch): assert all(c.stream for c in config) +@pytest.mark.parametrize("stream", [False, True]) +def test_base_config_fills_the_list_on_both_paths(server_module, monkeypatch, stream): + """/crawl and /crawl/stream give the list the same server defaults.""" + crawler = RecordingCrawler() + api = _install(monkeypatch, crawler) + config = { + **server_module.config, + "crawler": { + **server_module.config["crawler"], + "base_config": {"css_selector": "main"}, + }, + } + handler = api.handle_stream_crawl_request if stream else api.handle_crawl_request + + asyncio.run( + handler( + urls=["https://example.com/", "https://example.org/"], + browser_config={}, + crawler_config={}, + config=config, + crawler_configs=[ + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*example.com*"}, + }, + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*", "css_selector": "#own"}, + }, + ], + ) + ) + + [(method, urls, configs)] = crawler.calls + assert method == "arun_many" + # An entry that leaves the field unset gets base_config's value; one that + # sets it keeps its own. + assert [c.css_selector for c in configs] == ["main", "#own"] + + def test_stream_endpoint_forwards_the_list(server_module, monkeypatch): captured = {} From 82769644429334009745b75d7d69b5b5f275b7a3 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Mon, 12 Oct 2026 12:05:15 +0100 Subject: [PATCH 5/5] docs(docker): document crawler_configs Adds a "Per-URL Configs" section after the Streaming Results example in the Docker README and the self-hosting guide. It covers: the list is used for one URL or many, on /crawl and /crawl/stream; crawler_config is not a fallback, so the list should end with a catch-all entry (url_matcher "*"); base_config is applied to each entry; and a list mixing PDF and non-PDF entries returns 400. --- deploy/docker/README.md | 23 +++++++++++++++++++++++ docs/md_v2/core/self-hosting.md | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index a57c857b9..9216de6ce 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -658,6 +658,29 @@ async def test_stream_crawl(token: str = None): # Made token optional # asyncio.run(test_stream_crawl()) ``` +#### Per-URL Configs (`crawler_configs`) + +To crawl different URLs with different settings in one request, send `crawler_configs`: a list of `CrawlerRunConfig` payloads, each with a `url_matcher`. Every URL uses the first entry whose `url_matcher` matches it, and an entry without a `url_matcher` matches every URL. + +```python +payload = { + "urls": ["https://docs.example.com/guide", "https://example.com/"], + "crawler_configs": [ + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*docs.example.com*", "css_selector": "main"}, + }, + # Catch-all: decides every URL the entries above did not match + {"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}}, + ], +} +``` + +- The list is used for one URL or many, on both `/crawl` and `/crawl/stream`. +- When a list is sent, `crawler_config` is ignored. It is not a fallback: a URL that no entry matches comes back as a failed result (`No matching configuration found for URL: ...`), so end the list with a catch-all entry (`"url_matcher": "*"`). +- The server's `crawler.base_config` defaults from `config.yml` are applied to each entry on both endpoints. +- A list that mixes `PDFContentScrapingStrategy` entries with other scraping strategies is refused with `400`, because one crawler serves every URL of a request. + ### Asynchronous Jobs with Webhooks For long-running crawls or when you want to avoid keeping connections open, use the job queue endpoints. Instead of polling for results, configure a webhook to receive notifications when jobs complete. diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index 68a8a1065..8d0e54f2b 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -1481,6 +1481,29 @@ async def test_stream_crawl(token: str = None): # Made token optional # asyncio.run(test_stream_crawl()) ``` +#### Per-URL Configs (`crawler_configs`) + +To crawl different URLs with different settings in one request, send `crawler_configs`: a list of `CrawlerRunConfig` payloads, each with a `url_matcher`. Every URL uses the first entry whose `url_matcher` matches it, and an entry without a `url_matcher` matches every URL. + +```python +payload = { + "urls": ["https://docs.example.com/guide", "https://example.com/"], + "crawler_configs": [ + { + "type": "CrawlerRunConfig", + "params": {"url_matcher": "*docs.example.com*", "css_selector": "main"}, + }, + # Catch-all: decides every URL the entries above did not match + {"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}}, + ], +} +``` + +- The list is used for one URL or many, on both `/crawl` and `/crawl/stream`. +- When a list is sent, `crawler_config` is ignored. It is not a fallback: a URL that no entry matches comes back as a failed result (`No matching configuration found for URL: ...`), so end the list with a catch-all entry (`"url_matcher": "*"`). +- The server's `crawler.base_config` defaults from `config.yml` are applied to each entry on both endpoints. +- A list that mixes `PDFContentScrapingStrategy` entries with other scraping strategies is refused with `400`, because one crawler serves every URL of a request. + --- ## Real-time Monitoring & Operations