Merge pull request #1131 from primorLee/codex/fix-relative-markdown-links

fix: resolve markdown links from the document URL

Author: VinciGit00

scrapegraphai/utils/convert_to_md.py

@@ -2,8 +2,6 @@
 convert_to_md module
 """
 
-from urllib.parse import urlparse
-
 import html2text
 
 
@@ -29,8 +27,6 @@ def convert_to_md(html: str, url: str = None) -> str:
     h.body_width = 0
 
     if url is not None:
-        parsed_url = urlparse(url)
-        domain = f"{parsed_url.scheme}://{parsed_url.netloc}"
-        h.baseurl = domain
+        h.baseurl = url
 
     return h.handle(html)

tests/utils/convert_to_md_test.py

@@ -11,6 +11,15 @@ def test_html_with_links_and_images():
     assert convert_to_md(html) is not None
 
 
+def test_relative_links_use_document_url_as_base():
+    html = '<a href="guide.html">Guide</a><img src="images/logo.png" alt="Logo">'
+
+    markdown = convert_to_md(html, "https://example.com/docs/index.html")
+
+    assert "[Guide](https://example.com/docs/guide.html)" in markdown
+    assert "![Logo](https://example.com/docs/images/logo.png)" in markdown
+
+
 def test_html_with_tables():
     html = """
     <table>