Skip to content

Commit 44968c5

Browse files
committed
chore: remove dead normalize_url definitions from utils
Two unreachable functions in utils.py: - The first `normalize_url` (plain urljoin wrapper) was silently shadowed by the extended `normalize_url` defined ~20 lines later. Python last-write wins, so the first definition was never callable. - `normalize_url_tmp` was a hand-rolled URL joiner (string split on "/") with no callers outside utils.py itself. `urllib.parse.urljoin` already covers this correctly.
1 parent 6a181da commit 44968c5

1 file changed

Lines changed: 0 additions & 54 deletions

File tree

‎crawl4ai/utils.py‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2212,25 +2212,6 @@ def fast_format_html(html_string):
22122212
return "\n".join(formatted)
22132213

22142214

2215-
def normalize_url(href, base_url):
2216-
"""Normalize URLs to ensure consistent format"""
2217-
from urllib.parse import urljoin, urlparse
2218-
2219-
# Parse base URL to get components
2220-
parsed_base = urlparse(base_url)
2221-
if not parsed_base.scheme or not parsed_base.netloc:
2222-
raise ValueError(f"Invalid base URL format: {base_url}")
2223-
2224-
if parsed_base.scheme.lower() not in ["http", "https"]:
2225-
# Handle special protocols
2226-
raise ValueError(f"Invalid base URL format: {base_url}")
2227-
cleaned_href = href.strip()
2228-
2229-
# Use urljoin to handle all cases
2230-
return urljoin(base_url, cleaned_href)
2231-
2232-
2233-
22342215

22352216
def normalize_url(
22362217
href: str,
@@ -2431,41 +2412,6 @@ def efficient_normalize_url_for_deep_crawl(href, base_url, preserve_https=False,
24312412
return normalized
24322413

24332414

2434-
def normalize_url_tmp(href, base_url):
2435-
"""Normalize URLs to ensure consistent format"""
2436-
# Extract protocol and domain from base URL
2437-
try:
2438-
base_parts = base_url.split("/")
2439-
protocol = base_parts[0]
2440-
domain = base_parts[2]
2441-
except IndexError:
2442-
raise ValueError(f"Invalid base URL format: {base_url}")
2443-
2444-
# Handle special protocols
2445-
special_protocols = {"mailto:", "tel:", "ftp:", "file:", "data:", "javascript:"}
2446-
if any(href.lower().startswith(proto) for proto in special_protocols):
2447-
return href.strip()
2448-
2449-
# Handle anchor links
2450-
if href.startswith("#"):
2451-
return f"{base_url}{href}"
2452-
2453-
# Handle protocol-relative URLs
2454-
if href.startswith("//"):
2455-
return f"{protocol}{href}"
2456-
2457-
# Handle root-relative URLs
2458-
if href.startswith("/"):
2459-
return f"{protocol}//{domain}{href}"
2460-
2461-
# Handle relative URLs
2462-
if not href.startswith(("http://", "https://")):
2463-
# Remove leading './' if present
2464-
href = href.lstrip("./")
2465-
return f"{protocol}//{domain}/{href}"
2466-
2467-
return href.strip()
2468-
24692415

24702416
def quick_extract_links(html: str, base_url: str) -> Dict[str, List[Dict[str, str]]]:
24712417
"""

0 commit comments

Comments
 (0)