fix: resolve relative links against the page's <base href> (#157)

* fix: resolve relative links against the page's `<base href>`

* chore: remove base URL edge cases

Author: nikitachapovskii-dev

src/request-handler.ts

@@ -15,6 +15,7 @@ import {
     extractJsonLd,
     extractOpenGraphProperties,
     extractTitle,
+    getDocumentBaseUrl,
     processHtml,
 } from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
@@ -188,9 +189,12 @@ async function handleContent(
     const { request } = context;
     const { contentScraperSettings: settings } = request.userData;
 
+    const pageUrl = request.loadedUrl ?? request.url;
+    const baseUrl = getDocumentBaseUrl($, pageUrl);
+
     const $html = $('html');
     const html = $html.html()!;
-    const processedHtml = await processHtml(html, request.url, settings, $);
+    const processedHtml = await processHtml(html, baseUrl, settings, $);
     addTimeMeasureEvent(request.userData, `${crawlerType}-process-html`);
 
     const isTooLarge = processedHtml.length > settings.maxHtmlCharsToProcess;
@@ -211,17 +215,17 @@ async function handleContent(
             description: $('meta[name=description]').first().attr('content') ?? undefined,
             keywords: $('meta[name=keywords]').first().attr('content') ?? undefined,
             languageCode: $html.first().attr('lang') ?? undefined,
-            url: request.loadedUrl ?? request.url,
+            url: pageUrl,
             redirectedUrl: request.loadedUrl,
-            canonicalUrl: extractCanonicalUrl($, request.loadedUrl ?? request.url),
+            canonicalUrl: extractCanonicalUrl($, baseUrl),
             openGraph: extractOpenGraphProperties($),
             jsonLd: extractJsonLd($),
             headers,
         },
         query: request.userData.query,
         text: settings.outputFormats.includes('text') ? text : undefined,
         markdown: settings.outputFormats.includes('markdown')
-            ? htmlToMarkdown(processedHtml, request.loadedUrl ?? request.url)
+            ? htmlToMarkdown(processedHtml, pageUrl, baseUrl)
             : undefined,
         html: settings.outputFormats.includes('html') ? processedHtml : undefined,
     };

src/website-content-crawler/html-processing.ts

@@ -29,6 +29,17 @@ export function extractTitle($: CheerioAPI): string {
     return '';
 }
 
+export function getDocumentBaseUrl($: CheerioAPI, pageUrl: string): string {
+    const href = $('base[href]').first().attr('href');
+    if (!href) return pageUrl;
+
+    try {
+        return new URL(href, pageUrl).href;
+    } catch {
+        return pageUrl;
+    }
+}
+
 export function extractCanonicalUrl($: CheerioAPI, baseUrl: string): string | undefined {
     const href = $('html > head > link[rel="canonical"]').first().attr('href');
     if (!href) return undefined;

src/website-content-crawler/markdown.ts

@@ -13,12 +13,13 @@ function cleanAttribute(attribute: string | null): string {
     return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '';
 }
 
-function resolveHref(href: string, baseUrl: string): string {
-    if (!baseUrl) return href;
+function resolveHref(href: string, pageUrl: string, baseUrl: string): string {
+    if (!pageUrl) return href;
     try {
-        return new URL(href, baseUrl).toString();
+        // Unlike a browser, keep in-page anchors on the page, even when a `<base>` points elsewhere.
+        return new URL(href, href.startsWith('#') ? pageUrl : baseUrl).toString();
     } catch {
-        log.warning(`Failed to resolve link against the page URL: ${href}`);
+        log.warning(`Failed to resolve link: ${href}`);
         return href;
     }
 }
@@ -28,31 +29,31 @@ function resolveHref(href: string, baseUrl: string): string {
  * links such as `/docs` remain valid once the markdown is used outside of the source page.
  * @see https://github.com/mixmark-io/turndown/blob/master/src/commonmark-rules.js
  */
-const inlineLinkRule = (baseUrl: string): Rule => ({
+const inlineLinkRule = (pageUrl: string, baseUrl: string): Rule => ({
     filter: (node, options) => options.linkStyle === 'inlined'
         && node.nodeName === 'A'
         && Boolean(node.getAttribute('href')),
     replacement: (content, node) => {
         const element = node as HTMLElement;
-        const href = resolveHref(element.getAttribute('href')!, baseUrl).replace(/([()])/g, '\\$1');
+        const href = resolveHref(element.getAttribute('href')!, pageUrl, baseUrl).replace(/([()])/g, '\\$1');
         const title = cleanAttribute(element.getAttribute('title'));
         return `[${content}](${href}${title ? ` "${title.replace(/"/g, '\\"')}"` : ''})`;
     },
 });
 
 /**
  * Converts HTML to markdown using Turndown (source: Website Content Crawler).
- * Relative links are resolved against `url`, when provided.
+ * Relative links are resolved against `baseUrl`, and in-page anchors against `pageUrl`, when provided.
  */
-export const htmlToMarkdown = (html: string | null, url?: string): string | null => {
+export const htmlToMarkdown = (html: string | null, pageUrl?: string, baseUrl = pageUrl): string | null => {
     try {
         if (!html?.length) return null;
 
         const processor = new TurndownService(turndownSettings);
         if (html.length <= GFM_MAX_HTML_LENGTH) {
             processor.use(plugin.gfm); // Use GitHub Flavored Markdown
         }
-        processor.addRule('inlineLink', inlineLinkRule(url ?? ''));
+        processor.addRule('inlineLink', inlineLinkRule(pageUrl ?? '', baseUrl ?? ''));
 
         return processor.turndown(html);
     } catch (err: unknown) {

tests/helpers/server.ts

@@ -53,6 +53,26 @@ export function createTestServer(): express.Express {
         sendHtml('with-image.html', res);
     });
 
+    const sendLinksPage = (base: string, res: express.Response) => {
+        // Enough text for Readability to extract the content.
+        const text = `<p>${'Some readable text. '.repeat(20)}</p>`.repeat(8);
+        res.send(`<html><head>${base}<link rel="canonical" href="canonical-page"></head><body>
+            <p><a href="article">relative link</a> <a href="#section">in-page anchor</a></p>${text}
+        </body></html>`);
+    };
+
+    app.get('/with-base', (_req, res) => {
+        sendLinksPage('<base href="https://cdn.example.org/sub/">', res);
+    });
+
+    app.get('/redirect/page', (_req, res) => {
+        res.redirect('/redirected/page');
+    });
+
+    app.get('/redirected/page', (_req, res) => {
+        sendLinksPage('', res);
+    });
+
     // A minimal stand-in for a Google result page, holding a single organic result that points back
     // at this server, so that the search crawler can be exercised without reaching Google.
     app.get('/serp', (req, res) => {

tests/html-processing.test.ts

@@ -2,7 +2,7 @@ import { load } from 'cheerio';
 import type { CheerioAPI } from 'crawlee';
 import { describe, expect, it } from 'vitest';
 
-import { extractTitle } from '../src/website-content-crawler/html-processing.js';
+import { extractTitle, getDocumentBaseUrl } from '../src/website-content-crawler/html-processing.js';
 
 // The `cheerio` version bundled with Crawlee differs from the top-level one, so the types don't match.
 const parse = (html: string) => load(html) as unknown as CheerioAPI;
@@ -45,3 +45,10 @@ describe('extractTitle', () => {
         expect(extractTitle($)).toBe('Test Title');
     });
 });
+
+describe('getDocumentBaseUrl', () => {
+    it('should resolve a relative base against the page URL', () => {
+        const $ = parse('<html><head><base href="/docs/"></head><body></body></html>');
+        expect(getDocumentBaseUrl($, 'https://example.com/a/b')).toBe('https://example.com/docs/');
+    });
+});

tests/standby.test.ts

@@ -136,6 +136,31 @@ describe('Standby RAG tests', () => {
         expect(getImageRequestCount()).toBe(0);
     });
 
+    it.each([
+        { htmlTransformer: 'none' },
+        { htmlTransformer: 'readableText' },
+    ])('standby request with $htmlTransformer resolves links against the base URL', async (params) => {
+        const pageUrl = `${baseUrl}/with-base`;
+        const query = new URLSearchParams({ query: pageUrl, ...params });
+        const response = await fetch(`http://localhost:${browserServerPort}/search?${query}`);
+        const data = await response.json();
+
+        expect(response.status).toBe(200);
+        expect(data[0].metadata.canonicalUrl).toBe('https://cdn.example.org/sub/canonical-page');
+        expect(data[0].markdown).toContain('[relative link](https://cdn.example.org/sub/article)');
+        expect(data[0].markdown).toContain(`[in-page anchor](${pageUrl}#section)`);
+    });
+
+    it('standby request with readableText resolves links against the URL redirected to', async () => {
+        const query = new URLSearchParams({ query: `${baseUrl}/redirect/page`, htmlTransformer: 'readableText' });
+        const response = await fetch(`http://localhost:${browserServerPort}/search?${query}`);
+        const data = await response.json();
+
+        expect(response.status).toBe(200);
+        expect(data[0].metadata.url).toBe(`${baseUrl}/redirected/page`);
+        expect(data[0].markdown).toContain(`[relative link](${baseUrl}/redirected/article)`);
+    });
+
     it('standby request playwright does not download media files of the page', async () => {
         resetImageRequestCount();