Fix blocked webpage fetches with a requests fallback

scripts/search/bing_search.py

@@ -14,7 +14,7 @@
 from typing import Optional, Tuple
 from nltk.tokenize import sent_tokenize
 from typing import List, Dict, Union
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
 import aiohttp
 import asyncio
 import chardet
@@ -37,6 +37,21 @@
     'Upgrade-Insecure-Requests': '1'
 }
 
+WIKIMEDIA_USER_AGENT = (
+    'WebThinkerResearchBot/0.1 '
+    '(https://github.com/RUC-NLPIR/WebThinker; academic research)'
+)
+
+
+def request_headers_for_url(url: str) -> Dict[str, str]:
+    request_headers = dict(headers)
+    hostname = (urlparse(url).hostname or '').lower()
+    if hostname.endswith(('.wikipedia.org', '.wikimedia.org', '.wikimediafoundation.org')):
+        request_headers['User-Agent'] = WIKIMEDIA_USER_AGENT
+        request_headers.pop('Referer', None)
+    return request_headers
+
+
 # Initialize session
 session = requests.Session()
 session.headers.update(headers)
@@ -179,7 +194,11 @@ def extract_text_from_url(url, use_jina=False, jina_api_key=None, snippet: Optio
                 return extract_pdf_text(url)
 
             try:
-                response = session.get(url, timeout=30)
+                response = session.get(
+                    url,
+                    timeout=30,
+                    headers=request_headers_for_url(url),
+                )
                 response.raise_for_status()
 
                 # 添加编码检测和处理
@@ -196,8 +215,7 @@ def extract_text_from_url(url, use_jina=False, jina_api_key=None, snippet: Optio
                 has_error = (any(indicator.lower() in response.text.lower() for indicator in error_indicators) and len(response.text.split()) < 64) or response.text == ''
                 if has_error:
                     if WebParserClient_url is None:
-                        # If WebParserClient is not available, return error message
-                        return f"Error extracting content: {str(e)}"
+                        return f"Error extracting content: empty or blocked response from {url}"
                     # If content has error, use WebParserClient as fallback
                     client = WebParserClient(WebParserClient_url)
                     results = client.parse_urls([url])
@@ -502,6 +520,23 @@ async def acquire(self):
 # 创建全局速率限制器实例
 jina_rate_limiter = RateLimiter(rate_limit=130)  # 每分钟xxx次，避免报错
 
+async def fetch_with_requests_fallback(
+    url: str,
+    use_jina: bool = False,
+    jina_api_key: Optional[str] = None,
+    snippet: Optional[str] = None,
+    keep_links: bool = False,
+) -> str:
+    return await asyncio.to_thread(
+        extract_text_from_url,
+        url,
+        use_jina,
+        jina_api_key,
+        snippet,
+        keep_links,
+    )
+
+
 async def extract_text_from_url_async(url: str, session: aiohttp.ClientSession, use_jina: bool = False, 
                                     jina_api_key: Optional[str] = None, snippet: Optional[str] = None, 
                                     keep_links: bool = False) -> str:
@@ -527,7 +562,16 @@ async def extract_text_from_url_async(url: str, session: aiohttp.ClientSession,
                 text = await extract_pdf_text_async(url, session)
                 return text[:10000]
 
-            async with session.get(url) as response:
+            async with session.get(url, headers=request_headers_for_url(url)) as response:
+                if response.status >= 400:
+                    return await fetch_with_requests_fallback(
+                        url,
+                        use_jina=use_jina,
+                        jina_api_key=jina_api_key,
+                        snippet=snippet,
+                        keep_links=keep_links,
+                    )
+
                 # 检测和处理编码
                 content_type = response.headers.get('content-type', '').lower()
                 if 'charset' in content_type:
@@ -546,8 +590,13 @@ async def extract_text_from_url_async(url: str, session: aiohttp.ClientSession,
                 # has_error = len(html.split()) < 64
                 if has_error:
                     if WebParserClient_url is None:
-                        # If WebParserClient is not available, return error message
-                        return f"Error extracting content: {str(e)}"
+                        return await fetch_with_requests_fallback(
+                            url,
+                            use_jina=use_jina,
+                            jina_api_key=jina_api_key,
+                            snippet=snippet,
+                            keep_links=keep_links,
+                        )
                     # If content has error, use WebParserClient as fallback
                     client = WebParserClient(WebParserClient_url)
                     results = client.parse_urls([url])
@@ -596,8 +645,14 @@ async def extract_text_from_url_async(url: str, session: aiohttp.ClientSession,
         else:
             return text[:50000]
 
-    except Exception as e:
-        return f"Error fetching {url}: {str(e)}"
+    except Exception:
+        return await fetch_with_requests_fallback(
+            url,
+            use_jina=use_jina,
+            jina_api_key=jina_api_key,
+            snippet=snippet,
+            keep_links=keep_links,
+        )
 
 async def fetch_page_content_async(urls: List[str], use_jina: bool = False, jina_api_key: Optional[str] = None, 
                                  snippets: Optional[Dict[str, str]] = None, show_progress: bool = False,

scripts/tests/test_web_fetch_fallback.py

@@ -0,0 +1,83 @@
+import asyncio
+import os
+import sys
+import unittest
+from unittest.mock import AsyncMock, Mock, patch
+
+
+SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+from search.bing_search import (  # noqa: E402
+    WIKIMEDIA_USER_AGENT,
+    extract_text_from_url_async,
+    fetch_with_requests_fallback,
+    request_headers_for_url,
+)
+
+
+class WebFetchFallbackTests(unittest.TestCase):
+    def test_wikimedia_uses_descriptive_user_agent_without_google_referer(self):
+        request_headers = request_headers_for_url(
+            "https://en.wikipedia.org/wiki/Mercedes_Sosa"
+        )
+        self.assertEqual(request_headers["User-Agent"], WIKIMEDIA_USER_AGENT)
+        self.assertNotIn("Referer", request_headers)
+
+    def test_other_domains_keep_existing_browser_headers(self):
+        request_headers = request_headers_for_url("https://example.com/article")
+        self.assertIn("Mozilla/5.0", request_headers["User-Agent"])
+        self.assertEqual(request_headers["Referer"], "https://www.google.com/")
+
+    def test_requests_fallback_runs_off_the_async_event_loop(self):
+        with patch(
+            "search.bing_search.extract_text_from_url",
+            return_value="fallback page text",
+        ) as fetch:
+            result = asyncio.run(
+                fetch_with_requests_fallback(
+                    "https://example.com/article",
+                    snippet="relevant snippet",
+                )
+            )
+
+        self.assertEqual(result, "fallback page text")
+        fetch.assert_called_once_with(
+            "https://example.com/article",
+            False,
+            None,
+            "relevant snippet",
+            False,
+        )
+
+    def test_async_fetch_failure_uses_requests_fallback(self):
+        async def run_test():
+            session = Mock()
+            session.get.side_effect = asyncio.TimeoutError()
+
+            with patch(
+                "search.bing_search.fetch_with_requests_fallback",
+                new_callable=AsyncMock,
+                return_value="fallback page text",
+            ) as fallback:
+                result = await extract_text_from_url_async(
+                    "https://example.com/article",
+                    session,
+                    snippet="relevant snippet",
+                )
+
+            self.assertEqual(result, "fallback page text")
+            fallback.assert_awaited_once_with(
+                "https://example.com/article",
+                use_jina=False,
+                jina_api_key=None,
+                snippet="relevant snippet",
+                keep_links=False,
+            )
+
+        asyncio.run(run_test())
+
+
+if __name__ == "__main__":
+    unittest.main()