|
| 1 | +/** |
| 2 | + * High-performance, edge-compatible HTML to Markdown converter. |
| 3 | + * Converts rendered Astro HTML pages into clean, token-efficient Markdown |
| 4 | + * for AI agents requesting `Accept: text/markdown` or appending `.md`. |
| 5 | + */ |
| 6 | + |
| 7 | +export function htmlToMarkdown(html: string, pageUrl?: string): string { |
| 8 | + if (!html) { |
| 9 | + return ""; |
| 10 | + } |
| 11 | + |
| 12 | + // 1. Extract page title if available before stripping head |
| 13 | + const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i); |
| 14 | + const pageTitle = titleMatch ? decodeEntities(titleMatch[1].trim()) : ""; |
| 15 | + |
| 16 | + // 2. Strip noise elements: scripts, styles, SVGs, modals, nav, header, footer |
| 17 | + let text = html |
| 18 | + .replace(/<!--[\s\S]*?-->/g, "") |
| 19 | + .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "") |
| 20 | + .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "") |
| 21 | + .replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, "") |
| 22 | + .replace(/<dialog\b[^<]*(?:(?!<\/dialog>)<[^<]*)*<\/dialog>/gi, "") |
| 23 | + .replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "") |
| 24 | + .replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, "") |
| 25 | + .replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, ""); |
| 26 | + |
| 27 | + // 3. Focus on <main> or <article> if available |
| 28 | + const mainMatch = text.match(/<main[^>]*>([\s\S]*?)<\/main>/i); |
| 29 | + if (mainMatch) { |
| 30 | + text = mainMatch[1]; |
| 31 | + } else { |
| 32 | + const articleMatch = text.match(/<article[^>]*>([\s\S]*?)<\/article>/i); |
| 33 | + if (articleMatch) { |
| 34 | + text = articleMatch[1]; |
| 35 | + } else { |
| 36 | + const bodyMatch = text.match(/<body[^>]*>([\s\S]*?)<\/body>/i); |
| 37 | + if (bodyMatch) { |
| 38 | + text = bodyMatch[1]; |
| 39 | + } |
| 40 | + } |
| 41 | + } |
| 42 | + |
| 43 | + // 4. Code blocks (<pre><code ...>...</code></pre>) |
| 44 | + text = text.replace( |
| 45 | + /<pre[^>]*><code[^>]*class=["'][^"']*language-([a-z0-9_-]+)[^"']*["'][^>]*>([\s\S]*?)<\/code><\/pre>/gi, |
| 46 | + (_match, lang, code) => { |
| 47 | + return `\n\n\`\`\`${lang}\n${decodeEntities(stripTags(code)).trim()}\n\`\`\`\n\n`; |
| 48 | + }, |
| 49 | + ); |
| 50 | + text = text.replace( |
| 51 | + /<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, |
| 52 | + (_match, code) => { |
| 53 | + return `\n\n\`\`\`\n${decodeEntities(stripTags(code)).trim()}\n\`\`\`\n\n`; |
| 54 | + }, |
| 55 | + ); |
| 56 | + text = text.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_match, code) => { |
| 57 | + return `\`${decodeEntities(stripTags(code))}\``; |
| 58 | + }); |
| 59 | + |
| 60 | + // 5. Headings |
| 61 | + text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n\n# $1\n\n"); |
| 62 | + text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n\n## $1\n\n"); |
| 63 | + text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n\n### $1\n\n"); |
| 64 | + text = text.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, "\n\n#### $1\n\n"); |
| 65 | + text = text.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, "\n\n##### $1\n\n"); |
| 66 | + text = text.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, "\n\n###### $1\n\n"); |
| 67 | + |
| 68 | + // 6. Blockquotes |
| 69 | + text = text.replace( |
| 70 | + /<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, |
| 71 | + (_match, quote) => { |
| 72 | + const lines = stripTags(quote) |
| 73 | + .trim() |
| 74 | + .split("\n") |
| 75 | + .map((l) => `> ${l.trim()}`) |
| 76 | + .join("\n"); |
| 77 | + return `\n\n${lines}\n\n`; |
| 78 | + }, |
| 79 | + ); |
| 80 | + |
| 81 | + // 7. Bold, Italic, Strikethrough |
| 82 | + text = text.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, "**$2**"); |
| 83 | + text = text.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, "*$2*"); |
| 84 | + text = text.replace(/<(s|del|strike)[^>]*>([\s\S]*?)<\/\1>/gi, "~~$2~~"); |
| 85 | + |
| 86 | + // 8. Images |
| 87 | + text = text.replace( |
| 88 | + /<img\b[^>]*src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*>/gi, |
| 89 | + "", |
| 90 | + ); |
| 91 | + text = text.replace( |
| 92 | + /<img\b[^>]*alt=["']([^"']*)["'][^>]*src=["']([^"']+)["'][^>]*>/gi, |
| 93 | + "", |
| 94 | + ); |
| 95 | + text = text.replace(/<img\b[^>]*src=["']([^"']+)["'][^>]*>/gi, ""); |
| 96 | + |
| 97 | + // 9. Links |
| 98 | + text = text.replace( |
| 99 | + /<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, |
| 100 | + (_match, href, label) => { |
| 101 | + const cleanLabel = stripTags(label).trim(); |
| 102 | + if (!cleanLabel) { |
| 103 | + return ""; |
| 104 | + } |
| 105 | + return `[${cleanLabel}](${href})`; |
| 106 | + }, |
| 107 | + ); |
| 108 | + |
| 109 | + // 10. Lists |
| 110 | + text = text.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_match, item) => { |
| 111 | + return `\n- ${stripTags(item).trim()}`; |
| 112 | + }); |
| 113 | + text = text.replace(/<\/(ul|ol)>/gi, "\n\n"); |
| 114 | + |
| 115 | + // 11. Paragraphs, breaks, dividers |
| 116 | + text = text.replace(/<hr\s*[/]?>/gi, "\n\n---\n\n"); |
| 117 | + text = text.replace(/<br\s*[/]?>/gi, "\n"); |
| 118 | + text = text.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, "\n\n$1\n\n"); |
| 119 | + |
| 120 | + // 12. Strip remaining tags |
| 121 | + text = stripTags(text); |
| 122 | + |
| 123 | + // 13. Decode HTML entities |
| 124 | + text = decodeEntities(text); |
| 125 | + |
| 126 | + // 14. Clean up whitespace |
| 127 | + text = text |
| 128 | + .split("\n") |
| 129 | + .map((line) => line.trimEnd()) |
| 130 | + .join("\n") |
| 131 | + .replace(/\n{3,}/g, "\n\n") |
| 132 | + .trim(); |
| 133 | + |
| 134 | + // Add header prefix if page title exists and wasn't already in H1 |
| 135 | + if (pageTitle && !text.startsWith("# ")) { |
| 136 | + text = `# ${pageTitle}\n\n${text}`; |
| 137 | + } |
| 138 | + |
| 139 | + if (pageUrl) { |
| 140 | + text = `> Canonical URL: ${pageUrl}\n\n${text}`; |
| 141 | + } |
| 142 | + |
| 143 | + return text; |
| 144 | +} |
| 145 | + |
| 146 | +function stripTags(str: string): string { |
| 147 | + return str.replace(/<[^>]+>/g, ""); |
| 148 | +} |
| 149 | + |
| 150 | +function decodeEntities(str: string): string { |
| 151 | + return str |
| 152 | + .replace(/&/g, "&") |
| 153 | + .replace(/</g, "<") |
| 154 | + .replace(/>/g, ">") |
| 155 | + .replace(/"/g, '"') |
| 156 | + .replace(/'/g, "'") |
| 157 | + .replace(/'/g, "'") |
| 158 | + .replace(/ /g, " ") |
| 159 | + .replace(/&#x([0-9a-fA-F]+);/g, (_m, hex) => |
| 160 | + String.fromCharCode(parseInt(hex, 16)), |
| 161 | + ) |
| 162 | + .replace(/&#([0-9]+);/g, (_m, dec) => |
| 163 | + String.fromCharCode(parseInt(dec, 10)), |
| 164 | + ); |
| 165 | +} |
0 commit comments