fix(fetch): surface HTTP errors and missing content instead of answering NA

A page that could not be scraped as intended was indistinguishable from one
that could. FetchNode's default path (ChromiumLoader -> ascrape_playwright)
dropped the Response returned by page.goto(), so a 404, 403, 500, captcha wall
or login redirect reached the LLM as ordinary content and the model answered
"NA" with nothing in the logs to explain why. Reported in #1102, where
en.wikipedia.org/wiki/Timpson_(company) 404s (the article is at
Timpson_(retailer)) and the run still looked clean.

Two deterministic, LLM-free guards, both warnings so existing behaviour is
unchanged for anyone deliberately scraping error pages:

- ChromiumLoader keeps the Response from every page.goto() call site
  (ascrape_playwright, ascrape_playwright_scroll, ascrape_with_js_support) and
  warns on status >= 400. This mirrors what the opt-in use_soup=True path in
  FetchNode has always done.
- ParseNode warns when the parsed content contains none of the terms the user
  asked about — schema field names plus the significant words of the prompt.
  A 200 response can still reach the LLM without the requested data: content
  behind JavaScript that never rendered, a field inside a <script> blob the
  parser drops, or a document truncated beyond the model window. Zero matches
  is a deliberately conservative bar, so the warning stays quiet when the page
  simply phrases the answer differently. The graphs now pass their schema to
  ParseNode so it has the field names available.

Verified against the URLs from the issue: the 404 now logs "Received HTTP 404
for .../Timpson_(company); the scraped content is likely an error page" before
returning NA, while the corrected URL stays silent and answers 1865.

Also drops three dead imports from smart_scraper_multi_batch_graph.py, which
ruff blocks on now that the file is touched.

Fixes #1102

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: VinciGit00

.github/workflows/test-suite.yml

@@ -42,6 +42,7 @@ jobs:
           tests/test_batch_api.py
           tests/test_csv_scraper_multi_graph.py
           tests/test_depth_search_graph.py
+          tests/test_error_page_detection.py
           tests/test_json_scraper_graph.py
           tests/test_minimax_models.py
           tests/test_scrape_do.py

scrapegraphai/docloaders/chromium.py

@@ -10,6 +10,32 @@
 logger = get_logger("web-loader")
 
 
+def _warn_on_error_status(response: Any, url: str) -> None:
+    """Log a warning when a navigation returned an HTTP error status.
+
+    Playwright's ``page.goto()`` returns the main-frame ``Response``, but the
+    scrapers only keep ``page.content()``. Without this check an error page
+    (404, 403, 500, a captcha wall, a login redirect) is indistinguishable
+    from the intended document once it reaches the LLM, which then produces a
+    confidently wrong answer with no signal that anything went wrong.
+
+    This mirrors the behaviour of the ``use_soup=True`` path in ``FetchNode``:
+    it warns rather than raising, so scraping error pages on purpose keeps
+    working.
+
+    Args:
+        response: The ``Response`` returned by ``page.goto()``; may be ``None``
+            (for example on a same-document navigation) or lack a usable status.
+        url: The URL that was requested, used in the warning message.
+    """
+    status = getattr(response, "status", None)
+    if isinstance(status, int) and status >= 400:
+        logger.warning(
+            f"Received HTTP {status} for {url}; the scraped content is likely "
+            "an error page, not the intended document."
+        )
+
+
 class ChromiumLoader:
     """Scrapes HTML pages from URLs using a (headless) instance of the
     Chromium web driver with proxy protection.
@@ -251,7 +277,8 @@ async def ascrape_playwright_scroll(
                     context = await browser.new_context()
                     await Malenia.apply_stealth(context)
                     page = await context.new_page()
-                    await page.goto(url, wait_until="domcontentloaded")
+                    response = await page.goto(url, wait_until="domcontentloaded")
+                    _warn_on_error_status(response, url)
                     await page.wait_for_load_state(self.load_state)
 
                     previous_height = None
@@ -364,7 +391,8 @@ async def ascrape_playwright(self, url: str, browser_name: str = "chromium") ->
                     )
                     await Malenia.apply_stealth(context)
                     page = await context.new_page()
-                    await page.goto(url, wait_until="domcontentloaded")
+                    response = await page.goto(url, wait_until="domcontentloaded")
+                    _warn_on_error_status(response, url)
                     await page.wait_for_load_state(self.load_state)
                     results = await page.content()
                     logger.info("Content scraped")
@@ -421,7 +449,8 @@ async def ascrape_with_js_support(
                         storage_state=self.storage_state
                     )
                     page = await context.new_page()
-                    await page.goto(url, wait_until="networkidle")
+                    response = await page.goto(url, wait_until="networkidle")
+                    _warn_on_error_status(response, url)
                     results = await page.content()
                     logger.info("Content scraped after JavaScript rendering")
                     return results

scrapegraphai/graphs/code_generator_graph.py

@@ -93,7 +93,11 @@ def _create_graph(self) -> BaseGraph:
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         generate_validation_answer_node = GenerateAnswerNode(

scrapegraphai/graphs/document_scraper_graph.py

@@ -76,6 +76,7 @@ def _create_graph(self) -> BaseGraph:
                 "parse_html": False,
                 "chunk_size": self.model_token,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
         generate_answer_node = GenerateAnswerNode(

scrapegraphai/graphs/omni_scraper_graph.py

@@ -83,6 +83,7 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token,
                 "parse_urls": True,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
 

scrapegraphai/graphs/script_creator_graph.py

@@ -82,6 +82,7 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token,
                 "parse_html": False,
                 "llm_model": self.llm_model,
+                "schema": self.schema,
             },
         )
 

scrapegraphai/graphs/smart_scraper_graph.py

@@ -110,7 +110,11 @@ def _create_graph(self) -> BaseGraph:
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         generate_answer_node = GenerateAnswerNode(
@@ -152,6 +156,7 @@ def _create_graph(self) -> BaseGraph:
                 node_config={
                     "llm_model": self.llm_model,
                     "chunk_size": self.model_token,
+                    "schema": self.schema,
                 },
             )
 

scrapegraphai/graphs/smart_scraper_lite_graph.py

@@ -74,7 +74,11 @@ def _create_graph(self) -> BaseGraph:
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"llm_model": self.llm_model, "chunk_size": self.model_token},
+            node_config={
+                "llm_model": self.llm_model,
+                "chunk_size": self.model_token,
+                "schema": self.schema,
+            },
         )
 
         return BaseGraph(

scrapegraphai/graphs/smart_scraper_multi_batch_graph.py

@@ -5,9 +5,8 @@
 providing 50% cost savings compared to real-time API calls.
 """
 
-import asyncio
 from copy import deepcopy
-from typing import Dict, List, Optional, Type
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -17,7 +16,6 @@
 from ..utils.copy import safe_deepcopy
 from .abstract_graph import AbstractGraph
 from .base_graph import BaseGraph
-from .smart_scraper_graph import SmartScraperGraph
 
 
 class _FetchParseOnlyGraph(AbstractGraph):
@@ -57,6 +55,7 @@ def _create_graph(self) -> BaseGraph:
             node_config={
                 "llm_model": self.llm_model,
                 "chunk_size": self.model_token,
+                "schema": self.schema,
             },
         )
 

scrapegraphai/graphs/speech_graph.py

@@ -70,7 +70,11 @@ def _create_graph(self) -> BaseGraph:
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token, "llm_model": self.llm_model},
+            node_config={
+                "chunk_size": self.model_token,
+                "llm_model": self.llm_model,
+                "schema": self.schema,
+            },
         )
 
         generate_answer_node = GenerateAnswerNode(