Merge pull request #2289 from talelboussetta/bugfix/crawlresult-failed-shape

fix(models): default CrawlResult links and media to the scraped shape

Author: ntohidi

crawl4ai/models.py

@@ -133,8 +133,15 @@ class CrawlResult(BaseModel):
     fit_html: Optional[str] = None
     success: bool
     cleaned_html: Optional[str] = None
-    media: Dict[str, List[Dict]] = {}
-    links: Dict[str, List[Dict]] = {}
+    # The keys a scraped page gets (arun moves media's "tables" to .tables), so
+    # a result built without them -- robots.txt refusal, a failed fetch --
+    # keeps the shape.
+    media: Dict[str, List[Dict]] = Field(
+        default_factory=lambda: {"images": [], "videos": [], "audios": []}
+    )
+    links: Dict[str, List[Dict]] = Field(
+        default_factory=lambda: {"internal": [], "external": []}
+    )
     downloaded_files: Optional[List[str]] = None
     js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None

deploy/docker/server.py

@@ -842,8 +842,8 @@ class CrawlResult(BaseModel):
             html: str
             success: bool
             cleaned_html: Optional[str] = None
-            media: Dict[str, List[Dict]] = {}
-            links: Dict[str, List[Dict]] = {}
+            media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []}
+            links: Dict[str, List[Dict]] = {"internal": [], "external": []}
             downloaded_files: Optional[List[str]] = None
             js_execution_result: Optional[Dict[str, Any]] = None
             screenshot: Optional[str] = None

docs/md_v2/api/crawl-result.md

@@ -11,8 +11,8 @@ class CrawlResult(BaseModel):
     success: bool
     cleaned_html: Optional[str] = None
     fit_html: Optional[str] = None  # Preprocessed HTML optimized for extraction
-    media: Dict[str, List[Dict]] = {}
-    links: Dict[str, List[Dict]] = {}
+    media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []}
+    links: Dict[str, List[Dict]] = {"internal": [], "external": []}
     downloaded_files: Optional[List[str]] = None
     screenshot: Optional[str] = None
     pdf : Optional[bytes] = None

docs/md_v2/complete-sdk-reference.md

@@ -1077,8 +1077,8 @@ class CrawlResult(BaseModel):
     success: bool
     cleaned_html: Optional[str] = None
     fit_html: Optional[str] = None  # Preprocessed HTML optimized for extraction
-    media: Dict[str, List[Dict]] = {}
-    links: Dict[str, List[Dict]] = {}
+    media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []}
+    links: Dict[str, List[Dict]] = {"internal": [], "external": []}
     downloaded_files: Optional[List[str]] = None
     screenshot: Optional[str] = None
     pdf : Optional[bytes] = None

docs/md_v2/core/crawler-result.md

@@ -22,8 +22,8 @@ class CrawlResult(BaseModel):
     fit_html: Optional[str] = None
     success: bool
     cleaned_html: Optional[str] = None
-    media: Dict[str, List[Dict]] = {}
-    links: Dict[str, List[Dict]] = {}
+    media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []}
+    links: Dict[str, List[Dict]] = {"internal": [], "external": []}
     downloaded_files: Optional[List[str]] = None
     js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None

tests/unit/test_crawl_result_default_shape.py

@@ -0,0 +1,66 @@
+"""A CrawlResult built without links/media must still have their keys.
+
+Successful crawls fill `links` and `media` from the scraper's result (arun moves
+media's "tables" to CrawlResult.tables). Results built without them -- the
+robots.txt refusal, the "all proxies failed" and exception paths in
+AsyncWebCrawler.arun -- used to default to `{}`, so `result.links["internal"]`
+raised KeyError on exactly the results a batch caller has to handle, and the
+Docker API returned `"links": {}` / `"media": {}` for them.
+"""
+
+from crawl4ai.models import CrawlResult
+
+
+def links_shape():
+    return {"internal": [], "external": []}
+
+
+def media_shape():
+    return {"images": [], "videos": [], "audios": []}
+
+
+def _refused(url="https://example.com/private"):
+    # The shape AsyncWebCrawler.arun returns when robots.txt disallows a URL.
+    return CrawlResult(
+        url=url,
+        html="",
+        success=False,
+        status_code=403,
+        error_message="Access denied by robots.txt",
+    )
+
+
+def test_result_without_links_or_media_has_the_scraped_shape():
+    result = _refused()
+
+    assert result.links == links_shape()
+    assert result.media == media_shape()
+
+
+def test_default_is_not_shared_between_results():
+    first, second = _refused(), _refused("https://example.com/other")
+
+    assert first.links["internal"] is not second.links["internal"]
+    assert first.media["images"] is not second.media["images"]
+
+    first.links["internal"].append({"href": "https://example.com/a"})
+    first.media["images"].append({"src": "https://example.com/a.png"})
+
+    assert second.links == links_shape()
+    assert second.media == media_shape()
+
+
+def test_serialized_result_keeps_the_keys():
+    dumped = _refused().model_dump()
+
+    assert dumped["links"] == links_shape()
+    assert dumped["media"] == media_shape()
+
+
+def test_explicit_values_are_kept():
+    links = {"internal": [{"href": "https://example.com/a"}], "external": []}
+    result = CrawlResult(
+        url="https://example.com/", html="<p></p>", success=True, links=links
+    )
+
+    assert result.links == links